Remove Zope2 dependency.
[slapos.git] / slapos / recipe / erp5 / __init__.py
1 ##############################################################################
2 #
3 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
4 #
5 # WARNING: This program as such is intended to be used by professional
6 # programmers who take the whole responsibility of assessing all potential
7 # consequences resulting from its eventual inadequacies and bugs
8 # End users who are looking for a ready-to-use solution with commercial
9 # guarantees and support are strongly adviced to contract a Free Software
10 # Service Company
11 #
12 # This program is Free Software; you can redistribute it and/or
13 # modify it under the terms of the GNU General Public License
14 # as published by the Free Software Foundation; either version 3
15 # of the License, or (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
25 #
26 ##############################################################################
27 from slapos.recipe.librecipe import BaseSlapRecipe
28 import binascii
29 import os
30 import pkg_resources
31 import pprint
32 import hashlib
33 import sys
34 import zc.buildout
35 import zc.recipe.egg
36 import ConfigParser
37
38 # based on Zope2.utilities.mkzopeinstance.write_inituser
39 def Zope2InitUser(path, username, password):
40 open(path, 'w').write('')
41 os.chmod(path, 0600)
42 open(path, "w").write('%s:{SHA}%s\n' % (
43 username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
44
45 class Recipe(BaseSlapRecipe):
46 def getTemplateFilename(self, template_name):
47 return pkg_resources.resource_filename(__name__,
48 'template/%s' % template_name)
49
50 site_id = 'erp5'
51
52 def _install(self):
53 self.path_list = []
54 self.requirements, self.ws = self.egg.working_set()
55 # self.cron_d is a directory, where cron jobs can be registered
56 self.cron_d = self.installCrond()
57 self.logrotate_d, self.logrotate_backup = self.installLogrotate()
58 self.killpidfromfile = zc.buildout.easy_install.scripts(
59 [('killpidfromfile', __name__ + '.killpidfromfile',
60 'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
61 self.path_list.append(self.killpidfromfile)
62 ca_conf = self.installCertificateAuthority()
63
64 memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
65 port=11000)
66 kumo_conf = self.installKumo(self.getLocalIPv4Address())
67 conversion_server_conf = self.installConversionServer(
68 self.getLocalIPv4Address(), 23000, 23060)
69 mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678)
70 user, password = self.installERP5()
71 zodb_dir = os.path.join(self.data_root_directory, 'zodb')
72 self._createDirectory(zodb_dir)
73 zodb_root_path = os.path.join(zodb_dir, 'root.fs')
74 zope_access = self.installZope(ip=self.getLocalIPv4Address(),
75 port=12000 + 1, name='zope_%s' % 1,
76 zodb_configuration_string=self.substituteTemplate(
77 self.getTemplateFilename('zope-zodb-snippet.conf.in'),
78 dict(zodb_root_path=zodb_root_path)), with_timerservice=True)
79 key, certificate = self.requestCertificate('Login Based Access')
80 apache_conf = dict(
81 apache_login=self.installBackendApache(ip=self.getGlobalIPv6Address(),
82 port=13000, backend=zope_access, key=key, certificate=certificate))
83 self.installERP5Site(user, password, zope_access, mysql_conf,
84 conversion_server_conf, memcached_conf, kumo_conf, self.site_id)
85 self.installTestRunner(ca_conf, mysql_conf, conversion_server_conf,
86 memcached_conf, kumo_conf)
87 self.installTestSuiteRunner(ca_conf, mysql_conf, conversion_server_conf,
88 memcached_conf, kumo_conf)
89 self.linkBinary()
90 self.setConnectionDict(dict(
91 site_url=apache_conf['apache_login'],
92 site_user=user,
93 site_password=password,
94 memcached_url=memcached_conf['memcached_url'],
95 kumo_url=kumo_conf['kumo_address']
96 ))
97 return self.path_list
98
99 def _requestZeoFileStorage(self, server_name, storage_name):
100 """Local, slap.request compatible, call to ask for filestorage on Zeo
101
102 filter_kw can be used to select specific Zeo server
103
104 Someday in future it will be possible to invoke:
105
106 self.request(
107 software_release=self.computer_partition.getSoftwareRelease().getURI(),
108 software_type='Zeo Server',
109 partition_reference=storage_name,
110 filter_kw={'server_name': server_name},
111 shared=True
112 )
113
114 Thanks to this it will be possible to select precisely on which server
115 which storage will be placed.
116 """
117 base_port = 35001
118 if getattr(self, '_zeo_storage_dict', None) is None:
119 self._zeo_storage_dict = {}
120 if getattr(self, '_zeo_storage_port_dict', None) is None:
121 self._zeo_storage_port_dict = {}
122 self._zeo_storage_port_dict.setdefault(server_name,
123 base_port+len(self._zeo_storage_port_dict))
124 self._zeo_storage_dict[server_name] = self._zeo_storage_dict.get(
125 server_name, []) + [storage_name]
126 return dict(
127 ip=self.getLocalIPv4Address(),
128 port=self._zeo_storage_port_dict[server_name],
129 storage_name=storage_name
130 )
131
132 def installLogrotate(self):
133 """Installs logortate main configuration file and registers its to cron"""
134 logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
135 'logrotate.d'))
136 self._createDirectory(logrotate_d)
137 logrotate_backup = self.createBackupDirectory('logrotate')
138 logrotate_conf = self.createConfigurationFile("logrotate.conf",
139 "include %s" % logrotate_d)
140 logrotate_cron = os.path.join(self.cron_d, 'logrotate')
141 state_file = os.path.join(self.data_root_directory, 'logrotate.status')
142 open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
143 (self.options['logrotate_binary'], state_file, logrotate_conf))
144 self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
145 return logrotate_d, logrotate_backup
146
147 def registerLogRotation(self, name, log_file_list, postrotate_script):
148 """Register new log rotation requirement"""
149 open(os.path.join(self.logrotate_d, name), 'w').write(
150 self.substituteTemplate(self.getTemplateFilename(
151 'logrotate_entry.in'),
152 dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
153 postrotate=postrotate_script, olddir=self.logrotate_backup)))
154
155 def linkBinary(self):
156 """Links binaries to instance's bin directory for easier exposal"""
157 for linkline in self.options.get('link_binary_list', '').splitlines():
158 if not linkline:
159 continue
160 target = linkline.split()
161 if len(target) == 1:
162 target = target[0]
163 path, linkname = os.path.split(target)
164 else:
165 linkname = target[1]
166 target = target[0]
167 link = os.path.join(self.bin_directory, linkname)
168 if os.path.lexists(link):
169 if not os.path.islink(link):
170 raise zc.buildout.UserError(
171 'Target link already %r exists but it is not link' % link)
172 os.unlink(link)
173 os.symlink(target, link)
174 self.logger.debug('Created link %r -> %r' % (link, target))
175 self.path_list.append(link)
176
177 def installKumo(self, ip, kumo_manager_port=13101, kumo_server_port=13201,
178 kumo_server_listen_port=13202, kumo_gateway_port=13301):
179 # XXX: kumo is not storing pid in file, unless it is not running as daemon
180 # but running daemons is incompatible with SlapOS, so there is currently
181 # no way to have Kumo's pid files to rotate logs and send signals to them
182 config = dict(
183 kumo_gateway_binary=self.options['kumo_gateway_binary'],
184 kumo_gateway_ip=ip,
185 kumo_gateway_log=os.path.join(self.log_directory, "kumo-gateway.log"),
186 kumo_manager_binary=self.options['kumo_manager_binary'],
187 kumo_manager_ip=ip,
188 kumo_manager_log=os.path.join(self.log_directory, "kumo-manager.log"),
189 kumo_server_binary=self.options['kumo_server_binary'],
190 kumo_server_ip=ip,
191 kumo_server_log=os.path.join(self.log_directory, "kumo-server.log"),
192 kumo_server_storage=os.path.join(self.data_root_directory, "kumodb.tch"),
193 kumo_manager_port=kumo_manager_port,
194 kumo_server_port=kumo_server_port,
195 kumo_server_listen_port=kumo_server_listen_port,
196 kumo_gateway_port=kumo_gateway_port
197 )
198
199 self.path_list.append(self.createRunningWrapper('kumo_gateway',
200 self.substituteTemplate(self.getTemplateFilename('kumo_gateway.in'),
201 config)))
202
203 self.path_list.append(self.createRunningWrapper('kumo_manager',
204 self.substituteTemplate(self.getTemplateFilename('kumo_manager.in'),
205 config)))
206
207 self.path_list.append(self.createRunningWrapper('kumo_server',
208 self.substituteTemplate(self.getTemplateFilename('kumo_server.in'),
209 config)))
210
211 return dict(
212 kumo_address = '%s:%s' % (config['kumo_gateway_ip'],
213 config['kumo_gateway_port']),
214 kumo_gateway_ip=config['kumo_gateway_ip'],
215 kumo_gateway_port=config['kumo_gateway_port'],
216 )
217
218 def installMemcached(self, ip, port):
219 config = dict(
220 memcached_binary=self.options['memcached_binary'],
221 memcached_ip=ip,
222 memcached_port=port,
223 )
224 self.path_list.append(self.createRunningWrapper('memcached',
225 self.substituteTemplate(self.getTemplateFilename('memcached.in'),
226 config)))
227 return dict(memcached_url='%s:%s' %
228 (config['memcached_ip'], config['memcached_port']),
229 memcached_ip=config['memcached_ip'],
230 memcached_port=config['memcached_port'])
231
232 def installTestRunner(self, ca_conf, mysql_conf, conversion_server_conf,
233 memcached_conf, kumo_conf):
234 """Installs bin/runUnitTest executable to run all tests using
235 bin/runUnitTest"""
236 testinstance = self.createDataDirectory('testinstance')
237 # workaround wrong assumptions of ERP5Type.tests.runUnitTest about
238 # directory existence
239 unit_test = os.path.join(testinstance, 'unit_test')
240 if not os.path.isdir(unit_test):
241 os.mkdir(unit_test)
242 runUnitTest = zc.buildout.easy_install.scripts([
243 ('runUnitTest', __name__ + '.testrunner', 'runUnitTest')],
244 self.ws, sys.executable, self.bin_directory, arguments=[dict(
245 instance_home=testinstance,
246 prepend_path=self.bin_directory,
247 openssl_binary=self.options['openssl_binary'],
248 test_ca_path=ca_conf['certificate_authority_path'],
249 call_list=[self.options['runUnitTest_binary'],
250 '--erp5_sql_connection_string', '%(mysql_test_database)s@%'
251 '(ip)s:%(tcp_port)s %(mysql_test_user)s '
252 '%(mysql_test_password)s' % mysql_conf,
253 '--conversion_server_hostname=%(conversion_server_ip)s' % \
254 conversion_server_conf,
255 '--conversion_server_port=%(conversion_server_port)s' % \
256 conversion_server_conf,
257 '--volatile_memcached_server_hostname=%(memcached_ip)s' % memcached_conf,
258 '--volatile_memcached_server_port=%(memcached_port)s' % memcached_conf,
259 '--persistent_memcached_server_hostname=%(kumo_gateway_ip)s' % kumo_conf,
260 '--persistent_memcached_server_port=%(kumo_gateway_port)s' % kumo_conf,
261 ]
262 )])[0]
263 self.path_list.append(runUnitTest)
264
265 def installTestSuiteRunner(self, ca_conf, mysql_conf, conversion_server_conf,
266 memcached_conf, kumo_conf):
267 """Installs bin/runTestSuite executable to run all tests using
268 bin/runUnitTest"""
269 testinstance = self.createDataDirectory('test_suite_instance')
270 # workaround wrong assumptions of ERP5Type.tests.runUnitTest about
271 # directory existence
272 unit_test = os.path.join(testinstance, 'unit_test')
273 if not os.path.isdir(unit_test):
274 os.mkdir(unit_test)
275 connection_string_list = []
276 for test_database, test_user, test_password in \
277 mysql_conf['mysql_parallel_test_dict']:
278 connection_string_list.append(
279 '%s@%s:%s %s %s' % (test_database, mysql_conf['ip'],
280 mysql_conf['tcp_port'], test_user, test_password))
281 command = zc.buildout.easy_install.scripts([
282 ('runTestSuite', __name__ + '.test_suite_runner', 'runTestSuite')],
283 self.ws, sys.executable, self.bin_directory, arguments=[dict(
284 instance_home=testinstance,
285 prepend_path=self.bin_directory,
286 openssl_binary=self.options['openssl_binary'],
287 test_ca_path=ca_conf['certificate_authority_path'],
288 call_list=[self.options['runTestSuite_binary'],
289 '--db_list', ','.join(connection_string_list),
290 '--conversion_server_hostname=%(conversion_server_ip)s' % \
291 conversion_server_conf,
292 '--conversion_server_port=%(conversion_server_port)s' % \
293 conversion_server_conf,
294 '--volatile_memcached_server_hostname=%(memcached_ip)s' % memcached_conf,
295 '--volatile_memcached_server_port=%(memcached_port)s' % memcached_conf,
296 '--persistent_memcached_server_hostname=%(kumo_gateway_ip)s' % kumo_conf,
297 '--persistent_memcached_server_port=%(kumo_gateway_port)s' % kumo_conf,
298 ]
299 )])[0]
300 self.path_list.append(command)
301
302 def installCrond(self):
303 timestamps = self.createDataDirectory('cronstamps')
304 cron_output = os.path.join(self.log_directory, 'cron-output')
305 self._createDirectory(cron_output)
306 catcher = zc.buildout.easy_install.scripts([('catchcron',
307 __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
308 self.bin_directory, arguments=[cron_output])[0]
309 self.path_list.append(catcher)
310 cron_d = os.path.join(self.etc_directory, 'cron.d')
311 crontabs = os.path.join(self.etc_directory, 'crontabs')
312 self._createDirectory(cron_d)
313 self._createDirectory(crontabs)
314 wrapper = zc.buildout.easy_install.scripts([('crond',
315 __name__ + '.execute', 'execute')], self.ws, sys.executable,
316 self.wrapper_directory, arguments=[
317 self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
318 '-t', timestamps, '-f', '-l', '5', '-M', catcher]
319 )[0]
320 self.path_list.append(wrapper)
321 return cron_d
322
323 def requestCertificate(self, name):
324 hash = hashlib.sha512(name).hexdigest()
325 key = os.path.join(self.ca_private, hash + self.ca_key_ext)
326 certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
327 parser = ConfigParser.RawConfigParser()
328 parser.add_section('certificate')
329 parser.set('certificate', 'name', name)
330 parser.set('certificate', 'key_file', key)
331 parser.set('certificate', 'certificate_file', certificate)
332 parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
333 return key, certificate
334
335 def installCertificateAuthority(self, ca_country_code='XX',
336 ca_email='xx@example.com', ca_state='State', ca_city='City',
337 ca_company='Company'):
338 backup_path = self.createBackupDirectory('ca')
339 self.ca_dir = os.path.join(self.data_root_directory, 'ca')
340 self._createDirectory(self.ca_dir)
341 self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
342 self._createDirectory(self.ca_request_dir)
343 config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
344 self.ca_private = os.path.join(self.ca_dir, 'private')
345 self.ca_certs = os.path.join(self.ca_dir, 'certs')
346 self.ca_crl = os.path.join(self.ca_dir, 'crl')
347 self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
348 self.ca_key_ext = '.key'
349 self.ca_crt_ext = '.crt'
350 for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
351 self._createDirectory(d)
352 for f in ['crlnumber', 'serial']:
353 if not os.path.exists(os.path.join(self.ca_dir, f)):
354 open(os.path.join(self.ca_dir, f), 'w').write('01')
355 if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
356 open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
357 openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
358 config.update(
359 working_directory=self.ca_dir,
360 country_code=ca_country_code,
361 state=ca_state,
362 city=ca_city,
363 company=ca_company,
364 email_address=ca_email,
365 )
366 self._writeFile(openssl_configuration, pkg_resources.resource_string(
367 __name__, 'template/openssl.cnf.ca.in') % config)
368 self.path_list.extend(zc.buildout.easy_install.scripts([
369 ('certificate_authority',
370 __name__ + '.certificate_authority', 'runCertificateAuthority')],
371 self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
372 openssl_configuration=openssl_configuration,
373 openssl_binary=self.options['openssl_binary'],
374 certificate=os.path.join(self.ca_dir, 'cacert.pem'),
375 key=os.path.join(self.ca_private, 'cakey.pem'),
376 crl=os.path.join(self.ca_crl),
377 request_dir=self.ca_request_dir
378 )]))
379 # configure backup
380 backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
381 open(backup_cron, 'w').write(
382 '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
383 rdiff_backup=self.options['rdiff_backup_binary'],
384 source=self.ca_dir,
385 destination=backup_path))
386 self.path_list.append(backup_cron)
387
388 return dict(
389 ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
390 ca_crl=os.path.join(config['ca_dir'], 'crl'),
391 certificate_authority_path=config['ca_dir']
392 )
393
394 def installConversionServer(self, ip, port, openoffice_port):
395 name = 'conversion_server'
396 working_directory = self.createDataDirectory(name)
397 conversion_server_dict = dict(
398 working_path=working_directory,
399 uno_path=self.options['ooo_uno_path'],
400 office_binary_path=self.options['ooo_binary_path'],
401 ip=ip,
402 port=port,
403 openoffice_port=openoffice_port,
404 )
405 for env_line in self.options['environment'].splitlines():
406 env_line = env_line.strip()
407 if not env_line:
408 continue
409 if '=' in env_line:
410 env_key, env_value = env_line.split('=')
411 conversion_server_dict[env_key.strip()] = env_value.strip()
412 else:
413 raise zc.buildout.UserError('Line %r in environment parameter is '
414 'incorrect' % env_line)
415 config_file = self.createConfigurationFile(name + '.cfg',
416 self.substituteTemplate(self.getTemplateFilename('cloudooo.cfg.in'),
417 conversion_server_dict))
418 self.path_list.append(config_file)
419 self.path_list.extend(zc.buildout.easy_install.scripts([(name,
420 __name__ + '.execute', 'execute_with_signal_translation')], self.ws,
421 sys.executable, self.wrapper_directory,
422 arguments=[self.options['ooo_paster'].strip(), 'serve', config_file]))
423 return {
424 name + '_port': conversion_server_dict['port'],
425 name + '_ip': conversion_server_dict['ip']
426 }
427
428 def installHaproxy(self, ip, port, name, server_check_path, url_list):
429 server_template = """ server %(name)s %(address)s cookie %(name)s check inter 20s rise 2 fall 4"""
430 config = dict(name=name, ip=ip, port=port,
431 server_check_path=server_check_path,)
432 i = 1
433 server_list = []
434 for url in url_list:
435 server_list.append(server_template % dict(name='%s_%s' % (name, i),
436 address=url))
437 i += 1
438 config['server_text'] = '\n'.join(server_list)
439 haproxy_conf_path = self.createConfigurationFile('haproxy_%s.cfg' % name,
440 self.substituteTemplate(self.getTemplateFilename('haproxy.cfg.in'),
441 config))
442 self.path_list.append(haproxy_conf_path)
443 wrapper = zc.buildout.easy_install.scripts([('haproxy_%s' % name,
444 __name__ + '.execute', 'execute')], self.ws, sys.executable,
445 self.wrapper_directory, arguments=[
446 self.options['haproxy_binary'].strip(), '-f', haproxy_conf_path]
447 )[0]
448 self.path_list.append(wrapper)
449 return '%s:%s' % (ip, port)
450
451 def installERP5(self):
452 """
453 All zope have to share file created by portal_classes
454 (until everything is integrated into the ZODB).
455 So, do not request zope instance and create multiple in the same partition.
456 """
457 # Create instance directories
458 self.erp5_directory = self.createDataDirectory('erp5shared')
459 # Create init user
460 password = self.generatePassword()
461 # XXX Unhardcoded me please
462 user = 'zope'
463 Zope2InitUser(
464 os.path.join(self.erp5_directory, "inituser"), user, password)
465
466 self._createDirectory(self.erp5_directory)
467 for directory in (
468 'Constraint',
469 'Document',
470 'Extensions',
471 'PropertySheet',
472 'import',
473 'lib',
474 'tests',
475 'Products',
476 'etc',
477 ):
478 self._createDirectory(os.path.join(self.erp5_directory, directory))
479 self._createDirectory(os.path.join(self.erp5_directory, 'etc',
480 'package-includes'))
481 return user, password
482
483 def installERP5Site(self, user, password, zope_access, mysql_conf,
484 conversion_server_conf=None, memcached_conf=None, kumo_conf=None, erp5_site_id='erp5'):
485 """ Create a script controlled by supervisor, which creates a erp5
486 site on current available zope and mysql environment"""
487 conversion_server = None
488 if conversion_server_conf is not None:
489 conversion_server = "%s:%s" % (conversion_server_conf['conversion_server_ip'],
490 conversion_server_conf['conversion_server_port'])
491 if memcached_conf is None:
492 memcached_conf = {}
493 if kumo_conf is None:
494 kumo_conf = {}
495 # XXX Conversion server and memcache server coordinates are not relevant
496 # for pure site creation.
497 https_connection_url = "http://%s:%s@%s/" % (user, password, zope_access)
498 mysql_connection_string = "%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s" % mysql_conf
499
500 # XXX URL list vs. repository + list of bt5 names?
501 bt5_list = self.parameter_dict.get("bt5_list", "").split()
502 bt5_repository_list = self.parameter_dict.get("bt5_repository_list", "").split()
503
504 self.path_list.extend(zc.buildout.easy_install.scripts([('erp5_update',
505 __name__ + '.erp5', 'updateERP5')], self.ws,
506 sys.executable, self.wrapper_directory,
507 arguments=[erp5_site_id,
508 mysql_connection_string,
509 https_connection_url,
510 memcached_conf.get('memcached_url'),
511 conversion_server,
512 kumo_conf.get("kumo_address"),
513 bt5_list,
514 bt5_repository_list]))
515 return []
516
517 def installZeo(self, ip):
518 zodb_dir = os.path.join(self.data_root_directory, 'zodb')
519 self._createDirectory(zodb_dir)
520 zeo_configuration_dict = {}
521 zeo_number = 0
522 for zeo_server in sorted(self._zeo_storage_dict.iterkeys()):
523 zeo_number += 1
524 zeo_event_log = os.path.join(self.log_directory, 'zeo-%s.log'% zeo_number)
525 zeo_pid = os.path.join(self.run_directory, 'zeo-%s.pid'% zeo_number)
526 self.registerLogRotation('zeo-%s' % zeo_number, [zeo_event_log],
527 self.killpidfromfile + ' ' + zeo_pid + ' SIGUSR2')
528 config = dict(
529 zeo_ip=ip,
530 zeo_port=self._zeo_storage_port_dict[zeo_server],
531 zeo_event_log=zeo_event_log,
532 zeo_pid=zeo_pid,
533 )
534 storage_definition_list = []
535 for storage_name in sorted(self._zeo_storage_dict[zeo_server]):
536 path = os.path.join(zodb_dir, '%s.fs' % storage_name)
537 storage_definition_list.append("""<filestorage %(storage_name)s>
538 path %(path)s
539 </filestorage>"""% dict(storage_name=storage_name, path=path))
540 zeo_configuration_dict[storage_name] = dict(
541 ip=ip,
542 port=config['zeo_port'],
543 path=path
544 )
545 config['zeo_filestorage_snippet'] = '\n'.join(storage_definition_list)
546 zeo_conf_path = self.createConfigurationFile('zeo-%s.conf' % zeo_number,
547 self.substituteTemplate(self.getTemplateFilename('zeo.conf.in'), config))
548 self.path_list.append(zeo_conf_path)
549 wrapper = zc.buildout.easy_install.scripts([('zeo_%s' % zeo_number,
550 __name__ + '.execute', 'execute')], self.ws, sys.executable,
551 self.wrapper_directory, arguments=[
552 self.options['runzeo_binary'].strip(), '-C', zeo_conf_path]
553 )[0]
554 self.path_list.append(wrapper)
555 return zeo_configuration_dict
556
557 def installTidStorage(self, ip, port, known_tid_storage_identifier_dict,
558 access_url):
559 """Install TidStorage with all required backup tools
560
561 known_tid_storage_identifier_dict is a dictionary of:
562 (((ip, port),), storagename): (filestorage path, url for serialize)
563 url for serialize will be merged with access_url by internal tidstorage
564
565 """
566 backup_base_path = self.createBackupDirectory('zodb')
567 # it is time to fill known_tid_storage_identifier_dict with backup
568 # destination
569 formatted_storage_dict = dict()
570 for key, v in known_tid_storage_identifier_dict.copy().iteritems():
571 # generate unique name for each backup
572 storage_name = key[1]
573 destination = os.path.join(backup_base_path, storage_name)
574 self._createDirectory(destination)
575 formatted_storage_dict[str(key)] = (v[0], destination, v[1])
576 logfile = os.path.join(self.log_directory, 'tidstorage.log')
577 pidfile = os.path.join(self.run_directory, 'tidstorage.pid')
578 statusfile = os.path.join(self.log_directory, 'tidstorage.tid')
579 timestamp_file_path = os.path.join(self.log_directory,
580 'repozo_tidstorage_timestamp.log')
581 # shared configuration file
582 tidstorage_config = self.createConfigurationFile('tidstorage.py',
583 self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'),
584 dict(
585 known_tid_storage_identifier_dict=pprint.pformat(formatted_storage_dict),
586 base_url='%s/%%s/serialize' % access_url,
587 host=ip,
588 port=port,
589 timestamp_file_path=timestamp_file_path,
590 logfile=logfile,
591 pidfile=pidfile,
592 statusfile=statusfile
593 )))
594 # TID server
595 tidstorage_server = zc.buildout.easy_install.scripts([('tidstoraged',
596 __name__ + '.execute', 'execute')], self.ws, sys.executable,
597 self.wrapper_directory, arguments=[
598 self.options['tidstoraged_binary'], '--nofork', '--config',
599 tidstorage_config])[0]
600 self.registerLogRotation('tidsorage', [logfile, timestamp_file_path],
601 self.killpidfromfile + ' ' + pidfile + ' SIGHUP')
602 self.path_list.append(tidstorage_config)
603 self.path_list.append(tidstorage_server)
604
605 # repozo wrapper
606 tidstorage_repozo = zc.buildout.easy_install.scripts([('tidstorage_repozo',
607 __name__ + '.execute', 'execute')], self.ws, sys.executable,
608 self.bin_directory, arguments=[
609 self.options['tidstorage_repozo_binary'], '--config', tidstorage_config,
610 '--repozo', self.options['repozo_binary'], '-z'])[0]
611 self.path_list.append(tidstorage_repozo)
612
613 # and backup configuration
614 tidstorage_repozo_cron = os.path.join(self.cron_d, 'tidstorage_repozo')
615 open(tidstorage_repozo_cron, 'w').write('''0 0 * * * %(tidstorage_repozo)s
616 0 0 * * * cp -f %(tidstorage_tid)s %(tidstorage_tid_backup)s'''%dict(
617 tidstorage_repozo=tidstorage_repozo,
618 tidstorage_tid=statusfile,
619 tidstorage_tid_backup=os.path.join(backup_base_path, 'tidstorage.tid')))
620 self.path_list.append(tidstorage_repozo_cron)
621 return dict(host=ip, port=port)
622
623 def installZope(self, ip, port, name, zodb_configuration_string,
624 with_timerservice=False, tidstorage_config=None, thread_amount=1,
625 with_deadlockdebugger=True, zope_environment=None):
626 default_zope_environment = dict(
627 TMP=self.tmp_directory,
628 TMPDIR=self.tmp_directory,
629 HOME=self.tmp_directory,
630 PATH=self.bin_directory
631 )
632 if zope_environment is None:
633 zope_environment = default_zope_environment.copy()
634 else:
635 for envk, envv in default_zope_environment.iteritems():
636 if envk not in zope_environment:
637 zope_environment[envk] = envv
638 # Create zope configuration file
639 zope_config = dict(
640 products=self.options['products'],
641 thread_amount=thread_amount
642 )
643 # configure default Zope2 zcml
644 open(os.path.join(self.erp5_directory, 'etc', 'site.zcml'), 'w').write(
645 pkg_resources.resource_string(__name__, 'template/site.zcml'))
646 zope_config['zodb_configuration_string'] = zodb_configuration_string
647 zope_config['instance'] = self.erp5_directory
648 zope_config['event_log'] = os.path.join(self.log_directory,
649 '%s-event.log' % name)
650 zope_config['z2_log'] = os.path.join(self.log_directory,
651 '%s-Z2.log' % name)
652 zope_config['pid-filename'] = os.path.join(self.run_directory,
653 '%s.pid' % name)
654 zope_config['lock-filename'] = os.path.join(self.run_directory,
655 '%s.lock' % name)
656 self.registerLogRotation(name, [zope_config['event_log'],
657 zope_config['z2_log']], self.killpidfromfile + ' ' +
658 zope_config['pid-filename'] + ' SIGUSR2')
659
660 prefixed_products = []
661 for product in reversed(zope_config['products'].split()):
662 product = product.strip()
663 if product:
664 prefixed_products.append('products %s' % product)
665 prefixed_products.insert(0, 'products %s' % os.path.join(
666 self.erp5_directory, 'Products'))
667 zope_config['products'] = '\n'.join(prefixed_products)
668 zope_config['address'] = '%s:%s' % (ip, port)
669 zope_environment_list = []
670 for envk, envv in zope_environment.iteritems():
671 zope_environment_list.append('%s %s' % (envk, envv))
672 zope_config['environment'] = "\n".join(zope_environment_list)
673
674 zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in')
675 zope_conf_content = self.substituteTemplate(
676 zope_wrapper_template_location, zope_config)
677 if with_timerservice:
678 zope_conf_content += self.substituteTemplate(
679 self.getTemplateFilename('zope.conf.timerservice.in'), zope_config)
680 if tidstorage_config is not None:
681 zope_conf_content += self.substituteTemplate(
682 self.getTemplateFilename('zope-tidstorage-snippet.conf.in'),
683 tidstorage_config)
684 if with_deadlockdebugger:
685 zope_conf_content += self.substituteTemplate(
686 self.getTemplateFilename('zope-deadlockdebugger-snippet.conf.in'),
687 dict(dump_url='/manage_debug_threads',
688 secret=self.generatePassword()))
689
690 zope_conf_path = self.createConfigurationFile("%s.conf" % name,
691 zope_conf_content)
692 self.path_list.append(zope_conf_path)
693 # Create init script
694 wrapper = zc.buildout.easy_install.scripts([(name,
695 __name__ + '.execute', 'execute')], self.ws, sys.executable,
696 self.wrapper_directory, arguments=[
697 self.options['runzope_binary'].strip(), '-C', zope_conf_path]
698 )[0]
699 self.path_list.append(wrapper)
700 return zope_config['address']
701
702 def _getApacheConfigurationDict(self, prefix, ip, port):
703 apache_conf = dict()
704 apache_conf['pid_file'] = os.path.join(self.run_directory,
705 prefix + '.pid')
706 apache_conf['lock_file'] = os.path.join(self.run_directory,
707 prefix + '.lock')
708 apache_conf['ip'] = ip
709 apache_conf['port'] = port
710 apache_conf['server_admin'] = 'admin@'
711 apache_conf['error_log'] = os.path.join(self.log_directory,
712 prefix + '-error.log')
713 apache_conf['access_log'] = os.path.join(self.log_directory,
714 prefix + '-access.log')
715 self.registerLogRotation(prefix, [apache_conf['error_log'],
716 apache_conf['access_log']], self.killpidfromfile + ' ' +
717 apache_conf['pid_file'] + ' SIGUSR1')
718 return apache_conf
719
720 def _writeApacheConfiguration(self, prefix, apache_conf, backend,
721 access_control_string=None):
722 rewrite_rule_template = \
723 "RewriteRule (.*) http://%(backend)s$1 [L,P]"
724 if access_control_string is None:
725 path_template = pkg_resources.resource_string(__name__,
726 'template/apache.zope.conf.path.in')
727 path = path_template % dict(path='/')
728 else:
729 path_template = pkg_resources.resource_string(__name__,
730 'template/apache.zope.conf.path-protected.in')
731 path = path_template % dict(path='/',
732 access_control_string=access_control_string)
733 d = dict(
734 path=path,
735 backend=backend,
736 backend_path='/',
737 port=apache_conf['port'],
738 vhname=path.replace('/', ''),
739 )
740 rewrite_rule = rewrite_rule_template % d
741 apache_conf.update(**dict(
742 path_enable=path,
743 rewrite_rule=rewrite_rule
744 ))
745 apache_conf_string = pkg_resources.resource_string(__name__,
746 'template/apache.zope.conf.in') % apache_conf
747 return self.createConfigurationFile(prefix + '.conf', apache_conf_string)
748
749 def installFrontendZopeApache(self, ip, port, name, frontend_path, backend_url,
750 backend_path, key, certificate, access_control_string=None):
751 ident = 'frontend_' + name
752 apache_conf = self._getApacheConfigurationDict(ident, ip, port)
753 apache_conf['server_name'] = name
754 apache_conf['ssl_snippet'] = pkg_resources.resource_string(__name__,
755 'template/apache.ssl-snippet.conf.in') % dict(
756 login_certificate=certificate, login_key=key)
757
758 rewrite_rule_template = \
759 "RewriteRule ^%(path)s($|/.*) %(backend_url)s/VirtualHostBase/https/%(server_name)s:%(port)s%(backend_path)s/VirtualHostRoot/_vh_%(vhname)s$1 [L,P]\n"
760 path = pkg_resources.resource_string(__name__, 'template/apache.zope.conf.path-protected.in') % dict(path='/', access_control_string='none')
761 if access_control_string is None:
762 path_template = pkg_resources.resource_string(__name__,
763 'template/apache.zope.conf.path.in')
764 path += path_template % dict(path=frontend_path)
765 else:
766 path_template = pkg_resources.resource_string(__name__,
767 'template/apache.zope.conf.path-protected.in')
768 path += path_template % dict(path=frontend_path,
769 access_control_string=access_control_string)
770 d = dict(
771 path=frontend_path,
772 backend_url=backend_url,
773 backend_path=backend_path,
774 port=apache_conf['port'],
775 vhname=frontend_path.replace('/', ''),
776 server_name=name
777 )
778 rewrite_rule = rewrite_rule_template % d
779 apache_conf.update(**dict(
780 path_enable=path,
781 rewrite_rule=rewrite_rule
782 ))
783 apache_conf_string = pkg_resources.resource_string(__name__,
784 'template/apache.zope.conf.in') % apache_conf
785 apache_config_file = self.createConfigurationFile(ident + '.conf',
786 apache_conf_string)
787 self.path_list.append(apache_config_file)
788 self.path_list.extend(zc.buildout.easy_install.scripts([(
789 ident, __name__ + '.apache', 'runApache')], self.ws,
790 sys.executable, self.wrapper_directory, arguments=[
791 dict(
792 required_path_list=[key, certificate],
793 binary=self.options['httpd_binary'],
794 config=apache_config_file
795 )
796 ]))
797
798 def installBackendApache(self, ip, port, backend, key, certificate,
799 suffix='', access_control_string=None):
800 apache_conf = self._getApacheConfigurationDict('backend_apache'+suffix, ip,
801 port)
802 apache_conf['server_name'] = '%s' % apache_conf['ip']
803 apache_conf['ssl_snippet'] = pkg_resources.resource_string(__name__,
804 'template/apache.ssl-snippet.conf.in') % dict(
805 login_certificate=certificate, login_key=key)
806 apache_config_file = self._writeApacheConfiguration('backend_apache'+suffix,
807 apache_conf, backend, access_control_string)
808 self.path_list.append(apache_config_file)
809 self.path_list.extend(zc.buildout.easy_install.scripts([(
810 'backend_apache'+suffix,
811 __name__ + '.apache', 'runApache')], self.ws,
812 sys.executable, self.wrapper_directory, arguments=[
813 dict(
814 required_path_list=[key, certificate],
815 binary=self.options['httpd_binary'],
816 config=apache_config_file
817 )
818 ]))
819 # Note: IPv6 is assumed always
820 return 'https://[%(ip)s]:%(port)s' % apache_conf
821
822 def installMysqlServer(self, ip, port, database='erp5', user='user',
823 test_database='test_erp5', test_user='test_user', template_filename=None,
824 parallel_test_database_amount=100, mysql_conf=None):
825 if mysql_conf is None:
826 mysql_conf = {}
827 backup_directory = self.createBackupDirectory('mysql')
828 if template_filename is None:
829 template_filename = self.getTemplateFilename('my.cnf.in')
830 error_log = os.path.join(self.log_directory, 'mysqld.log')
831 slow_query_log = os.path.join(self.log_directory, 'mysql-slow.log')
832 mysql_conf.update(
833 ip=ip,
834 data_directory=os.path.join(self.data_root_directory,
835 'mysql'),
836 tcp_port=port,
837 pid_file=os.path.join(self.run_directory, 'mysqld.pid'),
838 socket=os.path.join(self.run_directory, 'mysqld.sock'),
839 error_log=error_log,
840 slow_query_log=slow_query_log,
841 mysql_database=database,
842 mysql_user=user,
843 mysql_password=self.generatePassword(),
844 mysql_test_password=self.generatePassword(),
845 mysql_test_database=test_database,
846 mysql_test_user=test_user,
847 mysql_parallel_test_dict=[
848 ('test_%i' % x,)*2 + (self.generatePassword(),) \
849 for x in xrange(0,parallel_test_database_amount)],
850 )
851 self.registerLogRotation('mysql', [error_log, slow_query_log],
852 '%(mysql_binary)s --no-defaults -B --user=root '
853 '--socket=%(mysql_socket)s -e "FLUSH LOGS"' % dict(
854 mysql_binary=self.options['mysql_binary'],
855 mysql_socket=mysql_conf['socket']))
856 self._createDirectory(mysql_conf['data_directory'])
857
858 mysql_conf_path = self.createConfigurationFile("my.cnf",
859 self.substituteTemplate(template_filename,
860 mysql_conf))
861
862 mysql_script_list = []
863 for x_database, x_user, x_password in \
864 [(mysql_conf['mysql_database'],
865 mysql_conf['mysql_user'],
866 mysql_conf['mysql_password']),
867 (mysql_conf['mysql_test_database'],
868 mysql_conf['mysql_test_user'],
869 mysql_conf['mysql_test_password']),
870 ] + mysql_conf['mysql_parallel_test_dict']:
871 mysql_script_list.append(pkg_resources.resource_string(__name__,
872 'template/initmysql.sql.in') % {
873 'mysql_database': x_database,
874 'mysql_user': x_user,
875 'mysql_password': x_password})
876 mysql_script_list.append('EXIT')
877 mysql_script = '\n'.join(mysql_script_list)
878 self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update',
879 __name__ + '.mysql', 'updateMysql')], self.ws,
880 sys.executable, self.wrapper_directory, arguments=[dict(
881 mysql_script=mysql_script,
882 mysql_binary=self.options['mysql_binary'].strip(),
883 mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(),
884 socket=mysql_conf['socket'],
885 )]))
886 self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld',
887 __name__ + '.mysql', 'runMysql')], self.ws,
888 sys.executable, self.wrapper_directory, arguments=[dict(
889 mysql_install_binary=self.options['mysql_install_binary'].strip(),
890 mysqld_binary=self.options['mysqld_binary'].strip(),
891 data_directory=mysql_conf['data_directory'].strip(),
892 mysql_binary=self.options['mysql_binary'].strip(),
893 socket=mysql_conf['socket'].strip(),
894 configuration_file=mysql_conf_path,
895 )]))
896 self.path_list.extend([mysql_conf_path])
897
898 # backup configuration
899 backup_directory = self.createBackupDirectory('mysql')
900 full_backup = os.path.join(backup_directory, 'full')
901 incremental_backup = os.path.join(backup_directory, 'incremental')
902 self._createDirectory(full_backup)
903 self._createDirectory(incremental_backup)
904 innobackupex_argument_list = [self.options['perl_binary'],
905 self.options['innobackupex_binary'],
906 '--defaults-file=%s' % mysql_conf_path,
907 '--socket=%s' %mysql_conf['socket'].strip(), '--user=root']
908 environment = dict(PATH='%s' % self.bin_directory)
909 innobackupex_incremental = zc.buildout.easy_install.scripts([(
910 'innobackupex_incremental', __name__ + '.execute', 'executee')],
911 self.ws, sys.executable, self.bin_directory, arguments=[
912 innobackupex_argument_list + ['--incremental'],
913 environment])[0]
914 self.path_list.append(innobackupex_incremental)
915 innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full',
916 __name__ + '.execute', 'executee')], self.ws,
917 sys.executable, self.bin_directory, arguments=[
918 innobackupex_argument_list,
919 environment])[0]
920 self.path_list.append(innobackupex_full)
921 backup_controller = zc.buildout.easy_install.scripts([
922 ('innobackupex_controller', __name__ + '.innobackupex', 'controller')],
923 self.ws, sys.executable, self.bin_directory,
924 arguments=[innobackupex_incremental, innobackupex_full, full_backup,
925 incremental_backup])[0]
926 self.path_list.append(backup_controller)
927 mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup')
928 open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller)
929 self.path_list.append(mysql_backup_cron)
930 # The return could be more explicit database, user ...
931 return mysql_conf