Frontend: Use VirtualHost to separate from custom config, listens to plain http port...
[slapos.git] / slapos / recipe / apache_frontend / __init__.py
1 ##############################################################################
2 #
3 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
4 #
5 # WARNING: This program as such is intended to be used by professional
6 # programmers who take the whole responsibility of assessing all potential
7 # consequences resulting from its eventual inadequacies and bugs
8 # End users who are looking for a ready-to-use solution with commercial
9 # guarantees and support are strongly adviced to contract a Free Software
10 # Service Company
11 #
12 # This program is Free Software; you can redistribute it and/or
13 # modify it under the terms of the GNU General Public License
14 # as published by the Free Software Foundation; either version 3
15 # of the License, or (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
25 #
26 ##############################################################################
27 from slapos.recipe.librecipe import BaseSlapRecipe
28 import os
29 import pkg_resources
30 import hashlib
31 import sys
32 import zc.buildout
33 import zc.recipe.egg
34 import ConfigParser
35 import re
36
37
38 class Recipe(BaseSlapRecipe):
39
40 def getTemplateFilename(self, template_name):
41 return pkg_resources.resource_filename(__name__,
42 'template/%s' % template_name)
43
44 def _install(self):
45 # Check for mandatory arguments
46 frontend_domain_name = self.parameter_dict.get("domain")
47 if frontend_domain_name is None:
48 raise zc.buildout.UserError('No domain name specified. Please define '
49 'the "domain" instance parameter.')
50
51 # Define optional arguments
52 frontend_port_number = self.parameter_dict.get("port", 4443)
53 frontend_plain_http_port_number = self.parameter_dict.get(
54 "plain_http_port", 8080)
55 base_varnish_port = 26009
56 slave_instance_list = self.parameter_dict.get("slave_instance_list", [])
57
58 self.path_list = []
59 self.requirements, self.ws = self.egg.working_set()
60
61 # self.cron_d is a directory, where cron jobs can be registered
62 self.cron_d = self.installCrond()
63 self.logrotate_d, self.logrotate_backup = self.installLogrotate()
64 self.killpidfromfile = zc.buildout.easy_install.scripts(
65 [('killpidfromfile', 'slapos.recipe.erp5.killpidfromfile',
66 'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
67 self.path_list.append(self.killpidfromfile)
68
69 rewrite_rule_list = []
70 rewrite_rule_zope_list = []
71 slave_dict = {}
72 service_dict = {}
73
74 # Check if default port
75 if frontend_port_number is 443 or frontend_port_number is 80:
76 port_snippet = ""
77 else:
78 port_snippet = ":%s" % frontend_port_number
79
80 for slave_instance in slave_instance_list:
81 backend_url = slave_instance.get("url", None)
82 reference = slave_instance.get("slave_reference")
83 # Set scheme (http? https?)
84 # Future work may allow to choose between http and https (or both?)
85 scheme = 'https://'
86
87 self.logger.info('processing slave instance: %s' % reference)
88
89 # Check for mandatory slave fields
90 if backend_url is None:
91 self.logger.warn('No "url" parameter is defined for %s slave'\
92 'instance. Ignoring it.' % reference)
93 continue
94
95 # Check for custom domain (like mypersonaldomain.com)
96 # If no custom domain, use generated one
97 domain = slave_instance.get('custom_domain',
98 "%s.%s" % (reference.replace("-", "").lower(), frontend_domain_name))
99 slave_dict[reference] = "%s%s%s/" % (scheme, domain, port_snippet)
100
101 # Check if we want varnish+stunnel cache.
102 if slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE'):
103 # XXX-Cedric : need to refactor to clean code? (to many variables)
104 rewrite_rule = self.configureVarnishSlave(
105 base_varnish_port, backend_url, reference, service_dict, domain)
106 base_varnish_port += 2
107 else:
108 rewrite_rule = "%s %s" % (domain, backend_url)
109
110 # Finally, if successful, we add the rewrite rule to our list of rules
111 if rewrite_rule:
112 # We check if we have a zope slave. It requires different rewrite
113 # rule structure.
114 # So we will have one RewriteMap for normal websites, and one
115 # RewriteMap for Zope Virtual Host Monster websites.
116 if slave_instance.get("type", "").lower() in ('zope'):
117 rewrite_rule_zope_list.append(rewrite_rule)
118 else:
119 rewrite_rule_list.append(rewrite_rule)
120
121 # Certificate stuff
122 valid_certificate_str = self.parameter_dict.get("domain_ssl_ca_cert")
123 valid_key_str = self.parameter_dict.get("domain_ssl_ca_key")
124 if valid_certificate_str is None and valid_key_str is None:
125 ca_conf = self.installCertificateAuthority()
126 key, certificate = self.requestCertificate(frontend_domain_name)
127 else:
128 ca_conf = self.installValidCertificateAuthority(
129 frontend_domain_name, valid_certificate_str, valid_key_str)
130 key = ca_conf.pop("key")
131 certificate = ca_conf.pop("certificate")
132 if service_dict != {}:
133 if valid_certificate_str is not None and valid_key_str is not None:
134 self.installCertificateAuthority()
135 stunnel_key, stunnel_certificate = \
136 self.requestCertificate(frontend_domain_name)
137 else:
138 stunnel_key, stunnel_certificate = key, certificate
139 self.installStunnel(service_dict,
140 stunnel_certificate, stunnel_key,
141 ca_conf["ca_crl"],
142 ca_conf["certificate_authority_path"])
143
144 apache_parameter_dict = self.installFrontendApache(
145 ip_list=["[%s]" % self.getGlobalIPv6Address(),
146 self.getLocalIPv4Address()],
147 port=frontend_port_number,
148 plain_http_port=frontend_plain_http_port_number,
149 name=frontend_domain_name,
150 rewrite_rule_list=rewrite_rule_list,
151 rewrite_rule_zope_list=rewrite_rule_zope_list,
152 key=key, certificate=certificate)
153
154 # Send connection informations about each slave
155 for reference, url in slave_dict.iteritems():
156 self.setConnectionDict(dict(site_url=url), reference)
157
158 # Then set it for master instance
159 self.setConnectionDict(
160 dict(site_url=apache_parameter_dict["site_url"],
161 domain_ipv6_address=self.getGlobalIPv6Address(),
162 domain_ipv4_address=self.getLocalIPv4Address()))
163
164 # Promises
165 promise_config = dict(
166 hostname=self.getGlobalIPv6Address(),
167 port=frontend_port_number,
168 python_path=sys.executable,
169 )
170 promise_v6 = self.createPromiseWrapper(
171 'apache_ipv6',
172 self.substituteTemplate(
173 pkg_resources.resource_filename(
174 'slapos.recipe.check_port_listening',
175 'template/socket_connection_attempt.py.in'),
176 promise_config))
177 self.path_list.append(promise_v6)
178
179 promise_config = dict(
180 hostname=self.getLocalIPv4Address(),
181 port=frontend_port_number,
182 python_path=sys.executable,
183 )
184 promise_v4 = self.createPromiseWrapper(
185 'apache_ipv4',
186 self.substituteTemplate(
187 pkg_resources.resource_filename(
188 'slapos.recipe.check_port_listening',
189 'template/socket_connection_attempt.py.in'),
190 promise_config))
191 self.path_list.append(promise_v4)
192
193 return self.path_list
194
195 def configureVarnishSlave(self, base_varnish_port, url, reference,
196 service_dict, domain):
197 # Varnish should use stunnel to connect to the backend
198 base_varnish_control_port = base_varnish_port
199 base_varnish_port += 1
200 # Use regex
201 host_regex = "((\[\w*|[0-9]+\.)(\:|)).*(\]|\.[0-9]+)"
202 slave_host = re.search(host_regex, url).group(0)
203 port_regex = "\w+(\/|)$"
204 matcher = re.search(port_regex, url)
205 if matcher is not None:
206 slave_port = matcher.group(0)
207 slave_port = slave_port.replace("/", "")
208 elif url.startswith("https://"):
209 slave_port = 443
210 else:
211 slave_port = 80
212 service_name = "varnish_%s" % reference
213 varnish_ip = self.getLocalIPv4Address()
214 stunnel_port = base_varnish_port + 1
215 self.installVarnishCache(service_name,
216 ip=varnish_ip,
217 port=base_varnish_port,
218 control_port=base_varnish_control_port,
219 backend_host=varnish_ip,
220 backend_port=stunnel_port,
221 size="1G")
222 service_dict[service_name] = dict(public_ip=varnish_ip,
223 public_port=stunnel_port,
224 private_ip=slave_host.replace("[", "").replace("]", ""),
225 private_port=slave_port)
226 return "%s http://%s:%s" % \
227 (domain, varnish_ip, base_varnish_port)
228
229 def installLogrotate(self):
230 """Installs logortate main configuration file and registers its to cron"""
231 logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
232 'logrotate.d'))
233 self._createDirectory(logrotate_d)
234 logrotate_backup = self.createBackupDirectory('logrotate')
235 logrotate_conf = self.createConfigurationFile("logrotate.conf",
236 "include %s" % logrotate_d)
237 logrotate_cron = os.path.join(self.cron_d, 'logrotate')
238 state_file = os.path.join(self.data_root_directory, 'logrotate.status')
239 open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
240 (self.options['logrotate_binary'], state_file, logrotate_conf))
241 self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
242 return logrotate_d, logrotate_backup
243
244 def registerLogRotation(self, name, log_file_list, postrotate_script):
245 """Register new log rotation requirement"""
246 open(os.path.join(self.logrotate_d, name), 'w').write(
247 self.substituteTemplate(self.getTemplateFilename(
248 'logrotate_entry.in'),
249 dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
250 postrotate=postrotate_script, olddir=self.logrotate_backup)))
251
252 def requestCertificate(self, name):
253 hash = hashlib.sha512(name).hexdigest()
254 key = os.path.join(self.ca_private, hash + self.ca_key_ext)
255 certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
256 parser = ConfigParser.RawConfigParser()
257 parser.add_section('certificate')
258 parser.set('certificate', 'name', name)
259 parser.set('certificate', 'key_file', key)
260 parser.set('certificate', 'certificate_file', certificate)
261 parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
262 return key, certificate
263
264 def installCrond(self):
265 timestamps = self.createDataDirectory('cronstamps')
266 cron_output = os.path.join(self.log_directory, 'cron-output')
267 self._createDirectory(cron_output)
268 catcher = zc.buildout.easy_install.scripts([('catchcron',
269 __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
270 self.bin_directory, arguments=[cron_output])[0]
271 self.path_list.append(catcher)
272 cron_d = os.path.join(self.etc_directory, 'cron.d')
273 crontabs = os.path.join(self.etc_directory, 'crontabs')
274 self._createDirectory(cron_d)
275 self._createDirectory(crontabs)
276 wrapper = zc.buildout.easy_install.scripts([('crond',
277 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
278 self.wrapper_directory, arguments=[
279 self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
280 '-t', timestamps, '-f', '-l', '5', '-M', catcher]
281 )[0]
282 self.path_list.append(wrapper)
283 return cron_d
284
285 def installValidCertificateAuthority(self, domain_name, certificate, key):
286 ca_dir = os.path.join(self.data_root_directory, 'ca')
287 ca_private = os.path.join(ca_dir, 'private')
288 ca_certs = os.path.join(ca_dir, 'certs')
289 ca_crl = os.path.join(ca_dir, 'crl')
290 self._createDirectory(ca_dir)
291 for path in (ca_private, ca_certs, ca_crl):
292 self._createDirectory(path)
293 key_path = os.path.join(ca_private, domain_name + ".key")
294 certificate_path = os.path.join(ca_certs, domain_name + ".crt")
295 self._writeFile(key_path, key)
296 self._writeFile(certificate_path, certificate)
297 return dict(certificate_authority_path=ca_dir,
298 ca_crl=ca_crl,
299 certificate=certificate_path,
300 key=key_path)
301
302 def installCertificateAuthority(self, ca_country_code='XX',
303 ca_email='xx@example.com', ca_state='State', ca_city='City',
304 ca_company='Company'):
305 backup_path = self.createBackupDirectory('ca')
306 self.ca_dir = os.path.join(self.data_root_directory, 'ca')
307 self._createDirectory(self.ca_dir)
308 self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
309 self._createDirectory(self.ca_request_dir)
310 config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
311 self.ca_private = os.path.join(self.ca_dir, 'private')
312 self.ca_certs = os.path.join(self.ca_dir, 'certs')
313 self.ca_crl = os.path.join(self.ca_dir, 'crl')
314 self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
315 self.ca_key_ext = '.key'
316 self.ca_crt_ext = '.crt'
317 for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
318 self._createDirectory(d)
319 for f in ['crlnumber', 'serial']:
320 if not os.path.exists(os.path.join(self.ca_dir, f)):
321 open(os.path.join(self.ca_dir, f), 'w').write('01')
322 if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
323 open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
324 openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
325 config.update(
326 working_directory=self.ca_dir,
327 country_code=ca_country_code,
328 state=ca_state,
329 city=ca_city,
330 company=ca_company,
331 email_address=ca_email,
332 )
333 self._writeFile(openssl_configuration, pkg_resources.resource_string(
334 __name__, 'template/openssl.cnf.ca.in') % config)
335 self.path_list.extend(zc.buildout.easy_install.scripts([
336 ('certificate_authority', __name__ + '.certificate_authority',
337 'runCertificateAuthority')],
338 self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
339 openssl_configuration=openssl_configuration,
340 openssl_binary=self.options['openssl_binary'],
341 certificate=os.path.join(self.ca_dir, 'cacert.pem'),
342 key=os.path.join(self.ca_private, 'cakey.pem'),
343 crl=os.path.join(self.ca_crl),
344 request_dir=self.ca_request_dir
345 )]))
346
347 # configure backup
348 backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
349 open(backup_cron, 'w').write(
350 '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
351 rdiff_backup=self.options['rdiff_backup_binary'],
352 source=self.ca_dir,
353 destination=backup_path))
354 self.path_list.append(backup_cron)
355
356 return dict(
357 ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
358 ca_crl=os.path.join(config['ca_dir'], 'crl'),
359 certificate_authority_path=config['ca_dir']
360 )
361
362 def _getApacheConfigurationDict(self, name, ip_list, port):
363 apache_conf = dict()
364 apache_conf['server_name'] = name
365 apache_conf['pid_file'] = os.path.join(self.run_directory,
366 name + '.pid')
367 apache_conf['lock_file'] = os.path.join(self.run_directory,
368 name + '.lock')
369 apache_conf['document_root'] = os.path.join(self.data_root_directory,
370 'htdocs')
371 apache_conf['ip_list'] = ip_list
372 apache_conf['port'] = port
373 apache_conf['server_admin'] = 'admin@'
374 apache_conf['error_log'] = os.path.join(self.log_directory,
375 'frontend-apache-error.log')
376 apache_conf['access_log'] = os.path.join(self.log_directory,
377 'frontend-apache-access.log')
378 self.registerLogRotation(name, [apache_conf['error_log'],
379 apache_conf['access_log']], self.killpidfromfile + ' ' +
380 apache_conf['pid_file'] + ' SIGUSR1')
381 return apache_conf
382
383 def installVarnishCache(self, name, ip, port, control_port, backend_host,
384 backend_port, size="1G"):
385 """
386 Install a varnish daemon for a certain address
387 """
388 directory = self.createDataDirectory(name)
389 varnish_config = dict(
390 directory=directory,
391 pid = "%s/varnish.pid" % directory,
392 port="%s:%s" % (ip, port),
393 varnishd_binary=self.options["varnishd_binary"],
394 control_port="%s:%s" % (ip, control_port),
395 storage="file,%s/storage.bin,%s" % (directory, size))
396
397 config_file = self.createConfigurationFile("%s.conf" % name,
398 self.substituteTemplate(self.getTemplateFilename('varnish.vcl.in'),
399 dict(backend_host=backend_host, backend_port=backend_port)))
400
401 varnish_argument_list = [varnish_config['varnishd_binary'].strip(),
402 "-F", "-n", directory, "-P", varnish_config["pid"], "-p",
403 "cc_command=exec %s " % self.options["gcc_binary"] +\
404 "-fpic -shared -o %o %s",
405 "-f", config_file,
406 "-a", varnish_config["port"], "-T", varnish_config["control_port"],
407 "-s", varnish_config["storage"]]
408 environment = dict(PATH=self.options["binutils_directory"])
409 wrapper = zc.buildout.easy_install.scripts([(name,
410 'slapos.recipe.librecipe.execute', 'executee')], self.ws,
411 sys.executable, self.wrapper_directory, arguments=[varnish_argument_list,
412 environment])[0]
413 self.path_list.append(wrapper)
414
415 return varnish_config
416
417 def installStunnel(self, service_dict, certificate,
418 key, ca_crl, ca_path):
419 """Installs stunnel
420 service_dict =
421 { name: (public_ip, private_ip, public_port, private_port),}
422 """
423 template_filename = self.getTemplateFilename('stunnel.conf.in')
424 template_entry_filename = self.getTemplateFilename('stunnel.conf.entry.in')
425
426 log = os.path.join(self.log_directory, 'stunnel.log')
427 pid_file = os.path.join(self.run_directory, 'stunnel.pid')
428 stunnel_conf = dict(
429 pid_file=pid_file,
430 log=log,
431 cert = certificate,
432 key = key,
433 ca_crl = ca_crl,
434 ca_path = ca_path,
435 entry_str=''
436 )
437 entry_list = []
438 for name, parameter_dict in service_dict.iteritems():
439 parameter_dict["name"] = name
440 entry_str = self.substituteTemplate(template_entry_filename,
441 parameter_dict)
442 entry_list.append(entry_str)
443
444 stunnel_conf["entry_str"] = "\n".join(entry_list)
445 stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
446 self.substituteTemplate(template_filename,
447 stunnel_conf))
448 wrapper = zc.buildout.easy_install.scripts([('stunnel',
449 'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
450 sys.executable, self.wrapper_directory, arguments=[
451 [self.options['stunnel_binary'].strip(), stunnel_conf_path],
452 [certificate, key]]
453 )[0]
454 self.path_list.append(wrapper)
455 return stunnel_conf
456
457 def installFrontendApache(self, ip_list, key, certificate, name,
458 port, plain_http_port=8080,
459 rewrite_rule_list=[], rewrite_rule_zope_list=[],
460 access_control_string=None):
461 # Create htdocs, populate it with default 404 document
462 htdocs_location = os.path.join(self.data_root_directory, 'htdocs')
463 self._createDirectory(htdocs_location)
464 notfound_file_location = os.path.join(htdocs_location, 'notfound.html')
465 notfound_template_file_location = self.getTemplateFilename(
466 'notfound.html')
467 notfound_file_content = open(notfound_template_file_location, 'r').read()
468 self._writeFile(notfound_file_location, notfound_file_content)
469
470 # Create mod_ssl cache directory
471 cache_directory_location = os.path.join(self.var_directory, 'cache')
472 mod_ssl_cache_location = os.path.join(cache_directory_location,
473 'httpd_mod_ssl')
474 self._createDirectory(cache_directory_location)
475 self._createDirectory(mod_ssl_cache_location)
476
477 # Create "custom" apache configuration file if it does not exist.
478 # Note : This file won't be erased or changed when slapgrid is ran.
479 # It can be freely customized by node admin.
480 custom_apache_configuration_directory = os.path.join(
481 self.data_root_directory, 'apache-conf.d')
482 self._createDirectory(custom_apache_configuration_directory)
483 custom_apache_configuration_file_location = os.path.join(
484 custom_apache_configuration_directory, 'apache_frontend.custom.conf')
485 f = open(custom_apache_configuration_file_location, 'a')
486 f.close()
487
488 # Create backup of custom apache configuration
489 backup_path = self.createBackupDirectory('custom_apache_conf_backup')
490 backup_cron = os.path.join(self.cron_d, 'custom_apache_conf_backup')
491 open(backup_cron, 'w').write(
492 '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
493 rdiff_backup=self.options['rdiff_backup_binary'],
494 source=custom_apache_configuration_directory,
495 destination=backup_path))
496 self.path_list.append(backup_cron)
497
498 # Create configuration file and rewritemaps
499 apachemap_name = "apachemap.txt"
500 apachemapzope_name = "apachemapzope.txt"
501 self.createConfigurationFile(apachemap_name, "\n".join(rewrite_rule_list))
502 self.createConfigurationFile(apachemapzope_name,
503 "\n".join(rewrite_rule_zope_list))
504 apache_conf = self._getApacheConfigurationDict(name, ip_list, port)
505 apache_conf['ssl_snippet'] = self.substituteTemplate(
506 self.getTemplateFilename('apache.ssl-snippet.conf.in'),
507 dict(login_certificate=certificate,
508 login_key=key,
509 httpd_mod_ssl_cache_directory=mod_ssl_cache_location,
510 )
511 )
512
513 apache_conf["listen"] = "\n".join([
514 "Listen %s:%s" % (ip, port)
515 for port in (plain_http_port, port)
516 for ip in ip_list
517 ])
518
519 path = self.substituteTemplate(
520 self.getTemplateFilename('apache.conf.path-protected.in'),
521 dict(path='/', access_control_string='none'))
522
523 apache_conf.update(**dict(
524 path_enable=path,
525 apachemap_path=os.path.join(self.etc_directory, apachemap_name),
526 apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name),
527 apache_domain=name,
528 https_port=port,
529 plain_http_port=plain_http_port,
530 custom_apache_conf=custom_apache_configuration_file_location,
531 ))
532
533 apache_conf_string = self.substituteTemplate(
534 self.getTemplateFilename('apache.conf.in'), apache_conf)
535
536 apache_config_file = self.createConfigurationFile('apache_frontend.conf',
537 apache_conf_string)
538 self.path_list.append(apache_config_file)
539
540 self.path_list.extend(zc.buildout.easy_install.scripts([(
541 'frontend_apache', 'slapos.recipe.erp5.apache', 'runApache')], self.ws,
542 sys.executable, self.wrapper_directory, arguments=[
543 dict(
544 required_path_list=[key, certificate],
545 binary=self.options['httpd_binary'],
546 config=apache_config_file)
547 ]))
548
549 return dict(site_url="https://%s:%s/" % (name, port))