slapos/recipe/erp5/__init__.py

   1 ##############################################################################
   2 #
   3 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
   4 #
   5 # WARNING: This program as such is intended to be used by professional
   6 # programmers who take the whole responsibility of assessing all potential
   7 # consequences resulting from its eventual inadequacies and bugs
   8 # End users who are looking for a ready-to-use solution with commercial
   9 # guarantees and support are strongly adviced to contract a Free Software
  10 # Service Company
  11 #
  12 # This program is Free Software; you can redistribute it and/or
  13 # modify it under the terms of the GNU General Public License
  14 # as published by the Free Software Foundation; either version 3
  15 # of the License, or (at your option) any later version.
  16 #
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  25 #
  26 ##############################################################################
  27 from slapos.recipe.librecipe import BaseSlapRecipe
  28 import binascii
  29 import os
  30 import pkg_resources
  31 import pprint
  32 import hashlib
  33 import sys
  34 import zc.buildout
  35 import zc.recipe.egg
  36 import ConfigParser
  37
  38 # based on Zope2.utilities.mkzopeinstance.write_inituser
  39 def Zope2InitUser(path, username, password):
  40   open(path, 'w').write('')
  41   os.chmod(path, 0600)
  42   open(path, "w").write('%s:{SHA}%s\n' % (
  43     username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
  44
  45 class Recipe(BaseSlapRecipe):
  46   def getTemplateFilename(self, template_name):
  47     return pkg_resources.resource_filename(__name__,
  48         'template/%s' % template_name)
  49
  50   site_id = 'erp5'
  51
  52   def _install(self):
  53     self.path_list = []
  54     self.requirements, self.ws = self.egg.working_set()
  55     # self.cron_d is a directory, where cron jobs can be registered
  56     self.cron_d = self.installCrond()
  57     self.logrotate_d, self.logrotate_backup = self.installLogrotate()
  58     self.killpidfromfile = zc.buildout.easy_install.scripts(
  59         [('killpidfromfile', __name__ + '.killpidfromfile',
  60           'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
  61     self.path_list.append(self.killpidfromfile)
  62     ca_conf = self.installCertificateAuthority()
  63
  64     memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
  65         port=11000)
  66     kumo_conf = self.installKumo(self.getLocalIPv4Address())
  67     conversion_server_conf = self.installConversionServer(
  68         self.getLocalIPv4Address(), 23000, 23060)
  69     mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678)
  70     user, password = self.installERP5()
  71     zodb_dir = os.path.join(self.data_root_directory, 'zodb')
  72     self._createDirectory(zodb_dir)
  73     zodb_root_path = os.path.join(zodb_dir, 'root.fs')
  74     zope_access = self.installZope(ip=self.getLocalIPv4Address(),
  75           port=12000 + 1, name='zope_%s' % 1,
  76           zodb_configuration_string=self.substituteTemplate(
  77             self.getTemplateFilename('zope-zodb-snippet.conf.in'),
  78             dict(zodb_root_path=zodb_root_path)), with_timerservice=True)
  79     key, certificate = self.requestCertificate('Login Based Access')
  80     apache_conf = dict(
  81         apache_login=self.installBackendApache(ip=self.getGlobalIPv6Address(),
  82           port=13000, backend=zope_access, key=key, certificate=certificate))
  83     self.installERP5Site(user, password, zope_access, mysql_conf,
  84              conversion_server_conf, memcached_conf, kumo_conf, self.site_id)
  85     self.installTestRunner(ca_conf, mysql_conf, conversion_server_conf,
  86                            memcached_conf, kumo_conf)
  87     self.installTestSuiteRunner(ca_conf, mysql_conf, conversion_server_conf,
  88                            memcached_conf, kumo_conf)
  89     self.linkBinary()
  90     self.setConnectionDict(dict(
  91       site_url=apache_conf['apache_login'],
  92       site_user=user,
  93       site_password=password,
  94       memcached_url=memcached_conf['memcached_url'],
  95       kumo_url=kumo_conf['kumo_address']
  96     ))
  97     return self.path_list
  98
  99   def _requestZeoFileStorage(self, server_name, storage_name):
 100     """Local, slap.request compatible, call to ask for filestorage on Zeo
 101
 102     filter_kw can be used to select specific Zeo server
 103
 104     Someday in future it will be possible to invoke:
 105
 106     self.request(
 107       software_release=self.computer_partition.getSoftwareRelease().getURI(),
 108       software_type='Zeo Server',
 109       partition_reference=storage_name,
 110       filter_kw={'server_name': server_name},
 111       shared=True
 112     )
 113
 114     Thanks to this it will be possible to select precisely on which server
 115     which storage will be placed.
 116     """
 117     base_port = 35001
 118     if getattr(self, '_zeo_storage_dict', None) is None:
 119       self._zeo_storage_dict = {}
 120     if getattr(self, '_zeo_storage_port_dict', None) is None:
 121       self._zeo_storage_port_dict = {}
 122     self._zeo_storage_port_dict.setdefault(server_name,
 123         base_port+len(self._zeo_storage_port_dict))
 124     self._zeo_storage_dict[server_name] = self._zeo_storage_dict.get(
 125         server_name, []) + [storage_name]
 126     return dict(
 127       ip=self.getLocalIPv4Address(),
 128       port=self._zeo_storage_port_dict[server_name],
 129       storage_name=storage_name
 130     )
 131
 132   def installLogrotate(self):
 133     """Installs logortate main configuration file and registers its to cron"""
 134     logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
 135       'logrotate.d'))
 136     self._createDirectory(logrotate_d)
 137     logrotate_backup = self.createBackupDirectory('logrotate')
 138     logrotate_conf = self.createConfigurationFile("logrotate.conf",
 139         "include %s" % logrotate_d)
 140     logrotate_cron = os.path.join(self.cron_d, 'logrotate')
 141     state_file = os.path.join(self.data_root_directory, 'logrotate.status')
 142     open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
 143         (self.options['logrotate_binary'], state_file, logrotate_conf))
 144     self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
 145     return logrotate_d, logrotate_backup
 146
 147   def registerLogRotation(self, name, log_file_list, postrotate_script):
 148     """Register new log rotation requirement"""
 149     open(os.path.join(self.logrotate_d, name), 'w').write(
 150         self.substituteTemplate(self.getTemplateFilename(
 151           'logrotate_entry.in'),
 152           dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
 153             postrotate=postrotate_script, olddir=self.logrotate_backup)))
 154
 155   def linkBinary(self):
 156     """Links binaries to instance's bin directory for easier exposal"""
 157     for linkline in self.options.get('link_binary_list', '').splitlines():
 158       if not linkline:
 159         continue
 160       target = linkline.split()
 161       if len(target) == 1:
 162         target = target[0]
 163         path, linkname = os.path.split(target)
 164       else:
 165         linkname = target[1]
 166         target = target[0]
 167       link = os.path.join(self.bin_directory, linkname)
 168       if os.path.lexists(link):
 169         if not os.path.islink(link):
 170           raise zc.buildout.UserError(
 171               'Target link already %r exists but it is not link' % link)
 172         os.unlink(link)
 173       os.symlink(target, link)
 174       self.logger.debug('Created link %r -> %r' % (link, target))
 175       self.path_list.append(link)
 176
 177   def installKumo(self, ip, kumo_manager_port=13101, kumo_server_port=13201,
 178       kumo_server_listen_port=13202, kumo_gateway_port=13301):
 179     # XXX: kumo is not storing pid in file, unless it is not running as daemon
 180     #      but running daemons is incompatible with SlapOS, so there is currently
 181     #      no way to have Kumo's pid files to rotate logs and send signals to them
 182     config = dict(
 183       kumo_gateway_binary=self.options['kumo_gateway_binary'],
 184       kumo_gateway_ip=ip,
 185       kumo_gateway_log=os.path.join(self.log_directory, "kumo-gateway.log"),
 186       kumo_manager_binary=self.options['kumo_manager_binary'],
 187       kumo_manager_ip=ip,
 188       kumo_manager_log=os.path.join(self.log_directory, "kumo-manager.log"),
 189       kumo_server_binary=self.options['kumo_server_binary'],
 190       kumo_server_ip=ip,
 191       kumo_server_log=os.path.join(self.log_directory, "kumo-server.log"),
 192       kumo_server_storage=os.path.join(self.data_root_directory, "kumodb.tch"),
 193       kumo_manager_port=kumo_manager_port,
 194       kumo_server_port=kumo_server_port,
 195       kumo_server_listen_port=kumo_server_listen_port,
 196       kumo_gateway_port=kumo_gateway_port
 197     )
 198
 199     self.path_list.append(self.createRunningWrapper('kumo_gateway',
 200       self.substituteTemplate(self.getTemplateFilename('kumo_gateway.in'),
 201         config)))
 202
 203     self.path_list.append(self.createRunningWrapper('kumo_manager',
 204       self.substituteTemplate(self.getTemplateFilename('kumo_manager.in'),
 205         config)))
 206
 207     self.path_list.append(self.createRunningWrapper('kumo_server',
 208       self.substituteTemplate(self.getTemplateFilename('kumo_server.in'),
 209         config)))
 210
 211     return dict(
 212       kumo_address = '%s:%s' % (config['kumo_gateway_ip'],
 213         config['kumo_gateway_port']),
 214       kumo_gateway_ip=config['kumo_gateway_ip'],
 215       kumo_gateway_port=config['kumo_gateway_port'],
 216     )
 217
 218   def installMemcached(self, ip, port):
 219     config = dict(
 220         memcached_binary=self.options['memcached_binary'],
 221         memcached_ip=ip,
 222         memcached_port=port,
 223     )
 224     self.path_list.append(self.createRunningWrapper('memcached',
 225       self.substituteTemplate(self.getTemplateFilename('memcached.in'),
 226         config)))
 227     return dict(memcached_url='%s:%s' %
 228         (config['memcached_ip'], config['memcached_port']),
 229         memcached_ip=config['memcached_ip'],
 230         memcached_port=config['memcached_port'])
 231
 232   def installTestRunner(self, ca_conf, mysql_conf, conversion_server_conf,
 233                         memcached_conf, kumo_conf):
 234     """Installs bin/runUnitTest executable to run all tests using
 235        bin/runUnitTest"""
 236     testinstance = self.createDataDirectory('testinstance')
 237     # workaround wrong assumptions of ERP5Type.tests.runUnitTest about
 238     # directory existence
 239     unit_test = os.path.join(testinstance, 'unit_test')
 240     if not os.path.isdir(unit_test):
 241       os.mkdir(unit_test)
 242     runUnitTest = zc.buildout.easy_install.scripts([
 243       ('runUnitTest', __name__ + '.testrunner', 'runUnitTest')],
 244       self.ws, sys.executable, self.bin_directory, arguments=[dict(
 245         instance_home=testinstance,
 246         prepend_path=self.bin_directory,
 247         openssl_binary=self.options['openssl_binary'],
 248         test_ca_path=ca_conf['certificate_authority_path'],
 249         call_list=[self.options['runUnitTest_binary'],
 250           '--erp5_sql_connection_string', '%(mysql_test_database)s@%'
 251           '(ip)s:%(tcp_port)s %(mysql_test_user)s '
 252           '%(mysql_test_password)s' % mysql_conf,
 253           '--conversion_server_hostname=%(conversion_server_ip)s' % \
 254                                                          conversion_server_conf,
 255           '--conversion_server_port=%(conversion_server_port)s' % \
 256                                                          conversion_server_conf,
 257           '--volatile_memcached_server_hostname=%(memcached_ip)s' % memcached_conf,
 258           '--volatile_memcached_server_port=%(memcached_port)s' % memcached_conf,
 259           '--persistent_memcached_server_hostname=%(kumo_gateway_ip)s' % kumo_conf,
 260           '--persistent_memcached_server_port=%(kumo_gateway_port)s' % kumo_conf,
 261       ]
 262         )])[0]
 263     self.path_list.append(runUnitTest)
 264
 265   def installTestSuiteRunner(self, ca_conf, mysql_conf, conversion_server_conf,
 266                         memcached_conf, kumo_conf):
 267     """Installs bin/runTestSuite executable to run all tests using
 268        bin/runUnitTest"""
 269     testinstance = self.createDataDirectory('test_suite_instance')
 270     # workaround wrong assumptions of ERP5Type.tests.runUnitTest about
 271     # directory existence
 272     unit_test = os.path.join(testinstance, 'unit_test')
 273     if not os.path.isdir(unit_test):
 274       os.mkdir(unit_test)
 275     connection_string_list = []
 276     for test_database, test_user, test_password in \
 277         mysql_conf['mysql_parallel_test_dict']:
 278       connection_string_list.append(
 279           '%s@%s:%s %s %s' % (test_database, mysql_conf['ip'],
 280                               mysql_conf['tcp_port'], test_user, test_password))
 281     command = zc.buildout.easy_install.scripts([
 282       ('runTestSuite', __name__ + '.test_suite_runner', 'runTestSuite')],
 283       self.ws, sys.executable, self.bin_directory, arguments=[dict(
 284         instance_home=testinstance,
 285         prepend_path=self.bin_directory,
 286         openssl_binary=self.options['openssl_binary'],
 287         test_ca_path=ca_conf['certificate_authority_path'],
 288         call_list=[self.options['runTestSuite_binary'],
 289           '--db_list', ','.join(connection_string_list),
 290           '--conversion_server_hostname=%(conversion_server_ip)s' % \
 291                                                          conversion_server_conf,
 292           '--conversion_server_port=%(conversion_server_port)s' % \
 293                                                          conversion_server_conf,
 294           '--volatile_memcached_server_hostname=%(memcached_ip)s' % memcached_conf,
 295           '--volatile_memcached_server_port=%(memcached_port)s' % memcached_conf,
 296           '--persistent_memcached_server_hostname=%(kumo_gateway_ip)s' % kumo_conf,
 297           '--persistent_memcached_server_port=%(kumo_gateway_port)s' % kumo_conf,
 298       ]
 299         )])[0]
 300     self.path_list.append(command)
 301
 302   def installCrond(self):
 303     timestamps = self.createDataDirectory('cronstamps')
 304     cron_output = os.path.join(self.log_directory, 'cron-output')
 305     self._createDirectory(cron_output)
 306     catcher = zc.buildout.easy_install.scripts([('catchcron',
 307       __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
 308       self.bin_directory, arguments=[cron_output])[0]
 309     self.path_list.append(catcher)
 310     cron_d = os.path.join(self.etc_directory, 'cron.d')
 311     crontabs = os.path.join(self.etc_directory, 'crontabs')
 312     self._createDirectory(cron_d)
 313     self._createDirectory(crontabs)
 314     wrapper = zc.buildout.easy_install.scripts([('crond',
 315       __name__ + '.execute', 'execute')], self.ws, sys.executable,
 316       self.wrapper_directory, arguments=[
 317         self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
 318         '-t', timestamps, '-f', '-l', '5', '-M', catcher]
 319       )[0]
 320     self.path_list.append(wrapper)
 321     return cron_d
 322
 323   def requestCertificate(self, name):
 324     hash = hashlib.sha512(name).hexdigest()
 325     key = os.path.join(self.ca_private, hash + self.ca_key_ext)
 326     certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
 327     parser = ConfigParser.RawConfigParser()
 328     parser.add_section('certificate')
 329     parser.set('certificate', 'name', name)
 330     parser.set('certificate', 'key_file', key)
 331     parser.set('certificate', 'certificate_file', certificate)
 332     parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
 333     return key, certificate
 334
 335   def installCertificateAuthority(self, ca_country_code='XX',
 336       ca_email='xx@example.com', ca_state='State', ca_city='City',
 337       ca_company='Company'):
 338     backup_path = self.createBackupDirectory('ca')
 339     self.ca_dir = os.path.join(self.data_root_directory, 'ca')
 340     self._createDirectory(self.ca_dir)
 341     self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
 342     self._createDirectory(self.ca_request_dir)
 343     config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
 344     self.ca_private = os.path.join(self.ca_dir, 'private')
 345     self.ca_certs = os.path.join(self.ca_dir, 'certs')
 346     self.ca_crl = os.path.join(self.ca_dir, 'crl')
 347     self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
 348     self.ca_key_ext = '.key'
 349     self.ca_crt_ext = '.crt'
 350     for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
 351       self._createDirectory(d)
 352     for f in ['crlnumber', 'serial']:
 353       if not os.path.exists(os.path.join(self.ca_dir, f)):
 354         open(os.path.join(self.ca_dir, f), 'w').write('01')
 355     if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
 356       open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
 357     openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
 358     config.update(
 359         working_directory=self.ca_dir,
 360         country_code=ca_country_code,
 361         state=ca_state,
 362         city=ca_city,
 363         company=ca_company,
 364         email_address=ca_email,
 365     )
 366     self._writeFile(openssl_configuration, pkg_resources.resource_string(
 367       __name__, 'template/openssl.cnf.ca.in') % config)
 368     self.path_list.extend(zc.buildout.easy_install.scripts([
 369       ('certificate_authority',
 370         __name__ + '.certificate_authority', 'runCertificateAuthority')],
 371         self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
 372           openssl_configuration=openssl_configuration,
 373           openssl_binary=self.options['openssl_binary'],
 374           certificate=os.path.join(self.ca_dir, 'cacert.pem'),
 375           key=os.path.join(self.ca_private, 'cakey.pem'),
 376           crl=os.path.join(self.ca_crl),
 377           request_dir=self.ca_request_dir
 378           )]))
 379     # configure backup
 380     backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
 381     open(backup_cron, 'w').write(
 382         '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
 383           rdiff_backup=self.options['rdiff_backup_binary'],
 384           source=self.ca_dir,
 385           destination=backup_path))
 386     self.path_list.append(backup_cron)
 387
 388     return dict(
 389       ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
 390       ca_crl=os.path.join(config['ca_dir'], 'crl'),
 391       certificate_authority_path=config['ca_dir']
 392     )
 393
 394   def installConversionServer(self, ip, port, openoffice_port):
 395     name = 'conversion_server'
 396     working_directory = self.createDataDirectory(name)
 397     conversion_server_dict = dict(
 398       working_path=working_directory,
 399       uno_path=self.options['ooo_uno_path'],
 400       office_binary_path=self.options['ooo_binary_path'],
 401       ip=ip,
 402       port=port,
 403       openoffice_port=openoffice_port,
 404     )
 405     for env_line in self.options['environment'].splitlines():
 406       env_line = env_line.strip()
 407       if not env_line:
 408         continue
 409       if '=' in env_line:
 410         env_key, env_value = env_line.split('=')
 411         conversion_server_dict[env_key.strip()] = env_value.strip()
 412       else:
 413         raise zc.buildout.UserError('Line %r in environment parameter is '
 414             'incorrect' % env_line)
 415     config_file = self.createConfigurationFile(name + '.cfg',
 416         self.substituteTemplate(self.getTemplateFilename('cloudooo.cfg.in'),
 417           conversion_server_dict))
 418     self.path_list.append(config_file)
 419     self.path_list.extend(zc.buildout.easy_install.scripts([(name,
 420       __name__ + '.execute', 'execute_with_signal_translation')], self.ws,
 421       sys.executable, self.wrapper_directory,
 422       arguments=[self.options['ooo_paster'].strip(), 'serve', config_file]))
 423     return {
 424       name + '_port': conversion_server_dict['port'],
 425       name + '_ip': conversion_server_dict['ip']
 426       }
 427
 428   def installHaproxy(self, ip, port, name, server_check_path, url_list):
 429     server_template = """  server %(name)s %(address)s cookie %(name)s check inter 20s rise 2 fall 4"""
 430     config = dict(name=name, ip=ip, port=port,
 431         server_check_path=server_check_path,)
 432     i = 1
 433     server_list = []
 434     for url in url_list:
 435       server_list.append(server_template % dict(name='%s_%s' % (name, i),
 436         address=url))
 437       i += 1
 438     config['server_text'] = '\n'.join(server_list)
 439     haproxy_conf_path = self.createConfigurationFile('haproxy_%s.cfg' % name,
 440       self.substituteTemplate(self.getTemplateFilename('haproxy.cfg.in'),
 441         config))
 442     self.path_list.append(haproxy_conf_path)
 443     wrapper = zc.buildout.easy_install.scripts([('haproxy_%s' % name,
 444       __name__ + '.execute', 'execute')], self.ws, sys.executable,
 445       self.wrapper_directory, arguments=[
 446         self.options['haproxy_binary'].strip(), '-f', haproxy_conf_path]
 447       )[0]
 448     self.path_list.append(wrapper)
 449     return '%s:%s' % (ip, port)
 450
 451   def installERP5(self):
 452     """
 453     All zope have to share file created by portal_classes
 454     (until everything is integrated into the ZODB).
 455     So, do not request zope instance and create multiple in the same partition.
 456     """
 457     # Create instance directories
 458     self.erp5_directory = self.createDataDirectory('erp5shared')
 459     # Create init user
 460     password = self.generatePassword()
 461     # XXX Unhardcoded me please
 462     user = 'zope'
 463     Zope2InitUser(
 464         os.path.join(self.erp5_directory, "inituser"), user, password)
 465
 466     self._createDirectory(self.erp5_directory)
 467     for directory in (
 468       'Constraint',
 469       'Document',
 470       'Extensions',
 471       'PropertySheet',
 472       'import',
 473       'lib',
 474       'tests',
 475       'Products',
 476       'etc',
 477       ):
 478       self._createDirectory(os.path.join(self.erp5_directory, directory))
 479     self._createDirectory(os.path.join(self.erp5_directory, 'etc',
 480       'package-includes'))
 481     return user, password
 482
 483   def installERP5Site(self, user, password, zope_access, mysql_conf,
 484           conversion_server_conf=None, memcached_conf=None, kumo_conf=None, erp5_site_id='erp5'):
 485     """ Create a script controlled by supervisor, which creates a erp5
 486     site on current available zope and mysql environment"""
 487     conversion_server = None
 488     if conversion_server_conf is not None:
 489       conversion_server = "%s:%s" % (conversion_server_conf['conversion_server_ip'],
 490              conversion_server_conf['conversion_server_port'])
 491     if memcached_conf is None:
 492       memcached_conf = {}
 493     if kumo_conf is None:
 494       kumo_conf = {}
 495     # XXX Conversion server and memcache server coordinates are not relevant
 496     # for pure site creation.
 497     https_connection_url = "http://%s:%s@%s/" % (user, password, zope_access)
 498     mysql_connection_string = "%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s" % mysql_conf
 499
 500     # XXX URL list vs. repository + list of bt5 names?
 501     bt5_list = self.parameter_dict.get("bt5_list", "").split()
 502     bt5_repository_list = self.parameter_dict.get("bt5_repository_list", "").split()
 503
 504     self.path_list.extend(zc.buildout.easy_install.scripts([('erp5_update',
 505             __name__ + '.erp5', 'updateERP5')], self.ws,
 506                   sys.executable, self.wrapper_directory,
 507                   arguments=[erp5_site_id,
 508                              mysql_connection_string,
 509                              https_connection_url,
 510                              memcached_conf.get('memcached_url'),
 511                              conversion_server,
 512                              kumo_conf.get("kumo_address"),
 513                              bt5_list,
 514                              bt5_repository_list]))
 515     return []
 516
 517   def installZeo(self, ip):
 518     zodb_dir = os.path.join(self.data_root_directory, 'zodb')
 519     self._createDirectory(zodb_dir)
 520     zeo_configuration_dict = {}
 521     zeo_number = 0
 522     for zeo_server in sorted(self._zeo_storage_dict.iterkeys()):
 523       zeo_number += 1
 524       zeo_event_log = os.path.join(self.log_directory, 'zeo-%s.log'% zeo_number)
 525       zeo_pid = os.path.join(self.run_directory, 'zeo-%s.pid'% zeo_number)
 526       self.registerLogRotation('zeo-%s' % zeo_number, [zeo_event_log],
 527           self.killpidfromfile + ' ' + zeo_pid + ' SIGUSR2')
 528       config = dict(
 529         zeo_ip=ip,
 530         zeo_port=self._zeo_storage_port_dict[zeo_server],
 531         zeo_event_log=zeo_event_log,
 532         zeo_pid=zeo_pid,
 533       )
 534       storage_definition_list = []
 535       for storage_name in sorted(self._zeo_storage_dict[zeo_server]):
 536         path = os.path.join(zodb_dir, '%s.fs' % storage_name)
 537         storage_definition_list.append("""<filestorage %(storage_name)s>
 538   path %(path)s
 539 </filestorage>"""% dict(storage_name=storage_name, path=path))
 540         zeo_configuration_dict[storage_name] = dict(
 541           ip=ip,
 542           port=config['zeo_port'],
 543           path=path
 544           )
 545       config['zeo_filestorage_snippet'] = '\n'.join(storage_definition_list)
 546       zeo_conf_path = self.createConfigurationFile('zeo-%s.conf' % zeo_number,
 547         self.substituteTemplate(self.getTemplateFilename('zeo.conf.in'), config))
 548       self.path_list.append(zeo_conf_path)
 549       wrapper = zc.buildout.easy_install.scripts([('zeo_%s' % zeo_number,
 550         __name__ + '.execute', 'execute')], self.ws, sys.executable,
 551         self.wrapper_directory, arguments=[
 552           self.options['runzeo_binary'].strip(), '-C', zeo_conf_path]
 553         )[0]
 554       self.path_list.append(wrapper)
 555     return zeo_configuration_dict
 556
 557   def installTidStorage(self, ip, port, known_tid_storage_identifier_dict,
 558       access_url):
 559     """Install TidStorage with all required backup tools
 560
 561       known_tid_storage_identifier_dict is a dictionary of:
 562       (((ip, port),), storagename): (filestorage path, url for serialize)
 563       url for serialize will be merged with access_url by internal tidstorage
 564
 565       """
 566     backup_base_path = self.createBackupDirectory('zodb')
 567     # it is time to fill known_tid_storage_identifier_dict with backup
 568     # destination
 569     formatted_storage_dict = dict()
 570     for key, v in known_tid_storage_identifier_dict.copy().iteritems():
 571       # generate unique name for each backup
 572       storage_name = key[1]
 573       destination = os.path.join(backup_base_path, storage_name)
 574       self._createDirectory(destination)
 575       formatted_storage_dict[str(key)] = (v[0], destination, v[1])
 576     logfile = os.path.join(self.log_directory, 'tidstorage.log')
 577     pidfile = os.path.join(self.run_directory, 'tidstorage.pid')
 578     statusfile = os.path.join(self.log_directory, 'tidstorage.tid')
 579     timestamp_file_path = os.path.join(self.log_directory,
 580           'repozo_tidstorage_timestamp.log')
 581     # shared configuration file
 582     tidstorage_config = self.createConfigurationFile('tidstorage.py',
 583         self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'),
 584           dict(
 585         known_tid_storage_identifier_dict=pprint.pformat(formatted_storage_dict),
 586         base_url='%s/%%s/serialize' % access_url,
 587         host=ip,
 588         port=port,
 589         timestamp_file_path=timestamp_file_path,
 590         logfile=logfile,
 591         pidfile=pidfile,
 592         statusfile=statusfile
 593       )))
 594     # TID server
 595     tidstorage_server = zc.buildout.easy_install.scripts([('tidstoraged',
 596       __name__ + '.execute', 'execute')], self.ws, sys.executable,
 597       self.wrapper_directory, arguments=[
 598         self.options['tidstoraged_binary'], '--nofork', '--config',
 599         tidstorage_config])[0]
 600     self.registerLogRotation('tidsorage', [logfile, timestamp_file_path],
 601           self.killpidfromfile + ' ' + pidfile + ' SIGHUP')
 602     self.path_list.append(tidstorage_config)
 603     self.path_list.append(tidstorage_server)
 604
 605     # repozo wrapper
 606     tidstorage_repozo = zc.buildout.easy_install.scripts([('tidstorage_repozo',
 607       __name__ + '.execute', 'execute')], self.ws, sys.executable,
 608       self.bin_directory, arguments=[
 609         self.options['tidstorage_repozo_binary'], '--config', tidstorage_config,
 610       '--repozo', self.options['repozo_binary'], '-z'])[0]
 611     self.path_list.append(tidstorage_repozo)
 612
 613     # and backup configuration
 614     tidstorage_repozo_cron = os.path.join(self.cron_d, 'tidstorage_repozo')
 615     open(tidstorage_repozo_cron, 'w').write('''0 0 * * * %(tidstorage_repozo)s
 616 0 0 * * * cp -f %(tidstorage_tid)s %(tidstorage_tid_backup)s'''%dict(
 617       tidstorage_repozo=tidstorage_repozo,
 618       tidstorage_tid=statusfile,
 619       tidstorage_tid_backup=os.path.join(backup_base_path, 'tidstorage.tid')))
 620     self.path_list.append(tidstorage_repozo_cron)
 621     return dict(host=ip, port=port)
 622
 623   def installZope(self, ip, port, name, zodb_configuration_string,
 624       with_timerservice=False, tidstorage_config=None, thread_amount=1,
 625       with_deadlockdebugger=True, zope_environment=None):
 626     default_zope_environment = dict(
 627       TMP=self.tmp_directory,
 628       TMPDIR=self.tmp_directory,
 629       HOME=self.tmp_directory,
 630       PATH=self.bin_directory
 631     )
 632     if zope_environment is None:
 633       zope_environment = default_zope_environment.copy()
 634     else:
 635       for envk, envv in default_zope_environment.iteritems():
 636         if envk not in zope_environment:
 637           zope_environment[envk] = envv
 638     # Create zope configuration file
 639     zope_config = dict(
 640         products=self.options['products'],
 641         thread_amount=thread_amount
 642     )
 643     # configure default Zope2 zcml
 644     open(os.path.join(self.erp5_directory, 'etc', 'site.zcml'), 'w').write(
 645         pkg_resources.resource_string(__name__, 'template/site.zcml'))
 646     zope_config['zodb_configuration_string'] = zodb_configuration_string
 647     zope_config['instance'] = self.erp5_directory
 648     zope_config['event_log'] = os.path.join(self.log_directory,
 649         '%s-event.log' % name)
 650     zope_config['z2_log'] = os.path.join(self.log_directory,
 651         '%s-Z2.log' % name)
 652     zope_config['pid-filename'] = os.path.join(self.run_directory,
 653         '%s.pid' % name)
 654     zope_config['lock-filename'] = os.path.join(self.run_directory,
 655         '%s.lock' % name)
 656     self.registerLogRotation(name, [zope_config['event_log'],
 657       zope_config['z2_log']], self.killpidfromfile + ' ' +
 658       zope_config['pid-filename'] + ' SIGUSR2')
 659
 660     prefixed_products = []
 661     for product in reversed(zope_config['products'].split()):
 662       product = product.strip()
 663       if product:
 664         prefixed_products.append('products %s' % product)
 665     prefixed_products.insert(0, 'products %s' % os.path.join(
 666                              self.erp5_directory, 'Products'))
 667     zope_config['products'] = '\n'.join(prefixed_products)
 668     zope_config['address'] = '%s:%s' % (ip, port)
 669     zope_environment_list = []
 670     for envk, envv in zope_environment.iteritems():
 671       zope_environment_list.append('%s %s' % (envk, envv))
 672     zope_config['environment'] = "\n".join(zope_environment_list)
 673
 674     zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in')
 675     zope_conf_content = self.substituteTemplate(
 676         zope_wrapper_template_location, zope_config)
 677     if with_timerservice:
 678       zope_conf_content += self.substituteTemplate(
 679           self.getTemplateFilename('zope.conf.timerservice.in'), zope_config)
 680     if tidstorage_config is not None:
 681       zope_conf_content += self.substituteTemplate(
 682           self.getTemplateFilename('zope-tidstorage-snippet.conf.in'),
 683           tidstorage_config)
 684     if with_deadlockdebugger:
 685       zope_conf_content += self.substituteTemplate(
 686           self.getTemplateFilename('zope-deadlockdebugger-snippet.conf.in'),
 687           dict(dump_url='/manage_debug_threads',
 688             secret=self.generatePassword()))
 689
 690     zope_conf_path = self.createConfigurationFile("%s.conf" % name,
 691         zope_conf_content)
 692     self.path_list.append(zope_conf_path)
 693     # Create init script
 694     wrapper = zc.buildout.easy_install.scripts([(name,
 695       __name__ + '.execute', 'execute')], self.ws, sys.executable,
 696       self.wrapper_directory, arguments=[
 697         self.options['runzope_binary'].strip(), '-C', zope_conf_path]
 698       )[0]
 699     self.path_list.append(wrapper)
 700     return zope_config['address']
 701
 702   def _getApacheConfigurationDict(self, prefix, ip, port):
 703     apache_conf = dict()
 704     apache_conf['pid_file'] = os.path.join(self.run_directory,
 705         prefix + '.pid')
 706     apache_conf['lock_file'] = os.path.join(self.run_directory,
 707         prefix + '.lock')
 708     apache_conf['ip'] = ip
 709     apache_conf['port'] = port
 710     apache_conf['server_admin'] = 'admin@'
 711     apache_conf['error_log'] = os.path.join(self.log_directory,
 712         prefix + '-error.log')
 713     apache_conf['access_log'] = os.path.join(self.log_directory,
 714         prefix + '-access.log')
 715     self.registerLogRotation(prefix, [apache_conf['error_log'],
 716       apache_conf['access_log']], self.killpidfromfile + ' ' +
 717       apache_conf['pid_file'] + ' SIGUSR1')
 718     return apache_conf
 719
 720   def _writeApacheConfiguration(self, prefix, apache_conf, backend,
 721       access_control_string=None):
 722     rewrite_rule_template = \
 723         "RewriteRule (.*) http://%(backend)s$1 [L,P]"
 724     if access_control_string is None:
 725       path_template = pkg_resources.resource_string(__name__,
 726         'template/apache.zope.conf.path.in')
 727       path = path_template % dict(path='/')
 728     else:
 729       path_template = pkg_resources.resource_string(__name__,
 730         'template/apache.zope.conf.path-protected.in')
 731       path = path_template % dict(path='/',
 732           access_control_string=access_control_string)
 733     d = dict(
 734           path=path,
 735           backend=backend,
 736           backend_path='/',
 737           port=apache_conf['port'],
 738           vhname=path.replace('/', ''),
 739     )
 740     rewrite_rule = rewrite_rule_template % d
 741     apache_conf.update(**dict(
 742       path_enable=path,
 743       rewrite_rule=rewrite_rule
 744     ))
 745     apache_conf_string = pkg_resources.resource_string(__name__,
 746           'template/apache.zope.conf.in') % apache_conf
 747     return self.createConfigurationFile(prefix + '.conf', apache_conf_string)
 748
 749   def installFrontendZopeApache(self, ip, port, name, frontend_path, backend_url,
 750       backend_path, key, certificate, access_control_string=None):
 751     ident = 'frontend_' + name
 752     apache_conf = self._getApacheConfigurationDict(ident, ip, port)
 753     apache_conf['server_name'] = name
 754     apache_conf['ssl_snippet'] = pkg_resources.resource_string(__name__,
 755         'template/apache.ssl-snippet.conf.in') % dict(
 756         login_certificate=certificate, login_key=key)
 757
 758     rewrite_rule_template = \
 759         "RewriteRule ^%(path)s($|/.*) %(backend_url)s/VirtualHostBase/https/%(server_name)s:%(port)s%(backend_path)s/VirtualHostRoot/_vh_%(vhname)s$1 [L,P]\n"
 760     path = pkg_resources.resource_string(__name__, 'template/apache.zope.conf.path-protected.in') % dict(path='/', access_control_string='none')
 761     if access_control_string is None:
 762       path_template = pkg_resources.resource_string(__name__,
 763         'template/apache.zope.conf.path.in')
 764       path += path_template % dict(path=frontend_path)
 765     else:
 766       path_template = pkg_resources.resource_string(__name__,
 767         'template/apache.zope.conf.path-protected.in')
 768       path += path_template % dict(path=frontend_path,
 769           access_control_string=access_control_string)
 770     d = dict(
 771           path=frontend_path,
 772           backend_url=backend_url,
 773           backend_path=backend_path,
 774           port=apache_conf['port'],
 775           vhname=frontend_path.replace('/', ''),
 776           server_name=name
 777     )
 778     rewrite_rule = rewrite_rule_template % d
 779     apache_conf.update(**dict(
 780       path_enable=path,
 781       rewrite_rule=rewrite_rule
 782     ))
 783     apache_conf_string = pkg_resources.resource_string(__name__,
 784           'template/apache.zope.conf.in') % apache_conf
 785     apache_config_file = self.createConfigurationFile(ident + '.conf',
 786         apache_conf_string)
 787     self.path_list.append(apache_config_file)
 788     self.path_list.extend(zc.buildout.easy_install.scripts([(
 789       ident, __name__ + '.apache', 'runApache')], self.ws,
 790           sys.executable, self.wrapper_directory, arguments=[
 791             dict(
 792               required_path_list=[key, certificate],
 793               binary=self.options['httpd_binary'],
 794               config=apache_config_file
 795             )
 796           ]))
 797
 798   def installBackendApache(self, ip, port, backend, key, certificate,
 799       suffix='', access_control_string=None):
 800     apache_conf = self._getApacheConfigurationDict('backend_apache'+suffix, ip,
 801         port)
 802     apache_conf['server_name'] = '%s' % apache_conf['ip']
 803     apache_conf['ssl_snippet'] = pkg_resources.resource_string(__name__,
 804         'template/apache.ssl-snippet.conf.in') % dict(
 805         login_certificate=certificate, login_key=key)
 806     apache_config_file = self._writeApacheConfiguration('backend_apache'+suffix,
 807         apache_conf, backend, access_control_string)
 808     self.path_list.append(apache_config_file)
 809     self.path_list.extend(zc.buildout.easy_install.scripts([(
 810       'backend_apache'+suffix,
 811         __name__ + '.apache', 'runApache')], self.ws,
 812           sys.executable, self.wrapper_directory, arguments=[
 813             dict(
 814               required_path_list=[key, certificate],
 815               binary=self.options['httpd_binary'],
 816               config=apache_config_file
 817             )
 818           ]))
 819     # Note: IPv6 is assumed always
 820     return 'https://[%(ip)s]:%(port)s' % apache_conf
 821
 822   def installMysqlServer(self, ip, port, database='erp5', user='user',
 823       test_database='test_erp5', test_user='test_user', template_filename=None,
 824       parallel_test_database_amount=100, mysql_conf=None):
 825     if mysql_conf is None:
 826       mysql_conf = {}
 827     backup_directory = self.createBackupDirectory('mysql')
 828     if template_filename is None:
 829       template_filename = self.getTemplateFilename('my.cnf.in')
 830     error_log = os.path.join(self.log_directory, 'mysqld.log')
 831     slow_query_log = os.path.join(self.log_directory, 'mysql-slow.log')
 832     mysql_conf.update(
 833         ip=ip,
 834         data_directory=os.path.join(self.data_root_directory,
 835           'mysql'),
 836         tcp_port=port,
 837         pid_file=os.path.join(self.run_directory, 'mysqld.pid'),
 838         socket=os.path.join(self.run_directory, 'mysqld.sock'),
 839         error_log=error_log,
 840         slow_query_log=slow_query_log,
 841         mysql_database=database,
 842         mysql_user=user,
 843         mysql_password=self.generatePassword(),
 844         mysql_test_password=self.generatePassword(),
 845         mysql_test_database=test_database,
 846         mysql_test_user=test_user,
 847         mysql_parallel_test_dict=[
 848             ('test_%i' % x,)*2 + (self.generatePassword(),) \
 849                  for x in xrange(0,parallel_test_database_amount)],
 850     )
 851     self.registerLogRotation('mysql', [error_log, slow_query_log],
 852         '%(mysql_binary)s --no-defaults -B --user=root '
 853         '--socket=%(mysql_socket)s -e "FLUSH LOGS"' % dict(
 854           mysql_binary=self.options['mysql_binary'],
 855           mysql_socket=mysql_conf['socket']))
 856     self._createDirectory(mysql_conf['data_directory'])
 857
 858     mysql_conf_path = self.createConfigurationFile("my.cnf",
 859         self.substituteTemplate(template_filename,
 860           mysql_conf))
 861
 862     mysql_script_list = []
 863     for x_database, x_user, x_password in \
 864           [(mysql_conf['mysql_database'],
 865             mysql_conf['mysql_user'],
 866             mysql_conf['mysql_password']),
 867            (mysql_conf['mysql_test_database'],
 868             mysql_conf['mysql_test_user'],
 869             mysql_conf['mysql_test_password']),
 870           ] + mysql_conf['mysql_parallel_test_dict']:
 871       mysql_script_list.append(pkg_resources.resource_string(__name__,
 872                      'template/initmysql.sql.in') % {
 873                         'mysql_database': x_database,
 874                         'mysql_user': x_user,
 875                         'mysql_password': x_password})
 876     mysql_script_list.append('EXIT')
 877     mysql_script = '\n'.join(mysql_script_list)
 878     self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update',
 879       __name__ + '.mysql', 'updateMysql')], self.ws,
 880       sys.executable, self.wrapper_directory, arguments=[dict(
 881         mysql_script=mysql_script,
 882         mysql_binary=self.options['mysql_binary'].strip(),
 883         mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(),
 884         socket=mysql_conf['socket'],
 885         )]))
 886     self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld',
 887       __name__ + '.mysql', 'runMysql')], self.ws,
 888         sys.executable, self.wrapper_directory, arguments=[dict(
 889         mysql_install_binary=self.options['mysql_install_binary'].strip(),
 890         mysqld_binary=self.options['mysqld_binary'].strip(),
 891         data_directory=mysql_conf['data_directory'].strip(),
 892         mysql_binary=self.options['mysql_binary'].strip(),
 893         socket=mysql_conf['socket'].strip(),
 894         configuration_file=mysql_conf_path,
 895        )]))
 896     self.path_list.extend([mysql_conf_path])
 897
 898     # backup configuration
 899     backup_directory = self.createBackupDirectory('mysql')
 900     full_backup = os.path.join(backup_directory, 'full')
 901     incremental_backup = os.path.join(backup_directory, 'incremental')
 902     self._createDirectory(full_backup)
 903     self._createDirectory(incremental_backup)
 904     innobackupex_argument_list = [self.options['perl_binary'],
 905         self.options['innobackupex_binary'],
 906         '--defaults-file=%s' % mysql_conf_path,
 907         '--socket=%s' %mysql_conf['socket'].strip(), '--user=root']
 908     environment = dict(PATH='%s' % self.bin_directory)
 909     innobackupex_incremental = zc.buildout.easy_install.scripts([(
 910       'innobackupex_incremental', __name__ + '.execute', 'executee')],
 911       self.ws, sys.executable, self.bin_directory, arguments=[
 912         innobackupex_argument_list + ['--incremental'],
 913         environment])[0]
 914     self.path_list.append(innobackupex_incremental)
 915     innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full',
 916       __name__ + '.execute', 'executee')], self.ws,
 917       sys.executable, self.bin_directory, arguments=[
 918         innobackupex_argument_list,
 919         environment])[0]
 920     self.path_list.append(innobackupex_full)
 921     backup_controller = zc.buildout.easy_install.scripts([
 922       ('innobackupex_controller', __name__ + '.innobackupex', 'controller')],
 923       self.ws, sys.executable, self.bin_directory,
 924       arguments=[innobackupex_incremental, innobackupex_full, full_backup,
 925         incremental_backup])[0]
 926     self.path_list.append(backup_controller)
 927     mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup')
 928     open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller)
 929     self.path_list.append(mysql_backup_cron)
 930     # The return could be more explicit database, user ...
 931     return mysql_conf