Fix indentation in pulse2 recipe
[slapos.git] / slapos / recipe / pulse2 / __init__.py
1 ##############################################################################
2 #
3 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
4 #
5 # WARNING: This program as such is intended to be used by professional
6 # programmers who take the whole responsibility of assessing all potential
7 # consequences resulting from its eventual inadequacies and bugs
8 # End users who are looking for a ready-to-use solution with commercial
9 # guarantees and support are strongly adviced to contract a Free Software
10 # Service Company
11 #
12 # This program is Free Software; you can redistribute it and/or
13 # modify it under the terms of the GNU General Public License
14 # as published by the Free Software Foundation; either version 3
15 # of the License, or (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
25 #
26 ##############################################################################
27 from slapos.recipe.librecipe import BaseSlapRecipe
28 import hashlib
29 import os
30 import pkg_resources
31 import sys
32 import zc.buildout
33 import ConfigParser
34
35 class Recipe(BaseSlapRecipe):
36 def getTemplateFilename(self, template_name):
37 return pkg_resources.resource_filename(__name__,
38 'template/%s' % template_name)
39
40 def _install(self):
41 # Apache
42 # TODO add in apache conf location of mmc.ini
43 document_root = self.createDataDirectory('htdocs')
44 self.createHtdocs(self.options['source'].strip(), document_root)
45 url = self.installApache(document_root)
46
47 # MySQL
48 mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678)
49
50 # LDAP
51 ldap_port = dict()
52
53 # Pulse
54 mmc_core_conf = self.installPulse2(ip=self.getLocalIPv4Address(),
55 port=11000, ldap_host=ldap_conf['host'], ldap_port=ldap_conf['port'])
56
57 ca_conf = self.installCertificateAuthority()
58 key, certificate = self.requestCertificate('Pulse')
59
60 stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(),
61 self.getLocalIPv4Address(), 12345, pulse_conf['inventory_port'],
62 certificate, key, ca_conf['ca_crl'],
63 ca_conf['certificate_authority_path'])
64
65 self.linkBinary()
66 self.setConnectionDict(dict(
67 stunnel_inventory_ip = stunnel_conf['public_ip'],
68 stunnel_inventory_port = stunnel_conf['public_port'],
69 url=url,
70 **mysql_conf
71 ))
72 return self.path_list
73
74 def linkBinary(self):
75 """Links binaries to instance's bin directory for easier exposal"""
76 for linkline in self.options.get('link_binary_list', '').splitlines():
77 if not linkline:
78 continue
79 target = linkline.split()
80 if len(target) == 1:
81 target = target[0]
82 path, linkname = os.path.split(target)
83 else:
84 linkname = target[1]
85 target = target[0]
86 link = os.path.join(self.bin_directory, linkname)
87 if os.path.lexists(link):
88 if not os.path.islink(link):
89 raise zc.buildout.UserError(
90 'Target link already %r exists but it is not link' % link)
91 os.unlink(link)
92 os.symlink(target, link)
93 self.logger.debug('Created link %r -> %r' % (link, target))
94 self.path_list.append(link)
95
96 def installCrond(self):
97 timestamps = self.createDataDirectory('cronstamps')
98 cron_output = os.path.join(self.log_directory, 'cron-output')
99 self._createDirectory(cron_output)
100 catcher = zc.buildout.easy_install.scripts([('catchcron',
101 __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
102 self.bin_directory, arguments=[cron_output])[0]
103 self.path_list.append(catcher)
104 cron_d = os.path.join(self.etc_directory, 'cron.d')
105 crontabs = os.path.join(self.etc_directory, 'crontabs')
106 self._createDirectory(cron_d)
107 self._createDirectory(crontabs)
108 # Use execute from erp5.
109 wrapper = zc.buildout.easy_install.scripts([('crond',
110 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
111 self.wrapper_directory, arguments=[
112 self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
113 '-t', timestamps, '-f', '-l', '5', '-M', catcher]
114 )[0]
115 self.path_list.append(wrapper)
116 return cron_d
117
118 def installLogrotate(self):
119 """Installs logortate main configuration file and registers its to cron"""
120 logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
121 'logrotate.d'))
122 self._createDirectory(logrotate_d)
123 logrotate_backup = self.createBackupDirectory('logrotate')
124 logrotate_conf = self.createConfigurationFile("logrotate.conf",
125 "include %s" % logrotate_d)
126 logrotate_cron = os.path.join(self.cron_d, 'logrotate')
127 state_file = os.path.join(self.data_root_directory, 'logrotate.status')
128 open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
129 (self.options['logrotate_binary'], state_file, logrotate_conf))
130 self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
131 return logrotate_d, logrotate_backup
132
133 def registerLogRotation(self, name, log_file_list, postrotate_script):
134 """Register new log rotation requirement"""
135 open(os.path.join(self.logrotate_d, name), 'w').write(
136 self.substituteTemplate(self.getTemplateFilename(
137 'logrotate_entry.in'),
138 dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
139 postrotate=postrotate_script, olddir=self.logrotate_backup)))
140
141 def installCertificateAuthority(self, ca_country_code='XX',
142 ca_email='xx@example.com', ca_state='State', ca_city='City',
143 ca_company='Company'):
144 backup_path = self.createBackupDirectory('ca')
145 self.ca_dir = os.path.join(self.data_root_directory, 'ca')
146 self._createDirectory(self.ca_dir)
147 self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
148 self._createDirectory(self.ca_request_dir)
149 config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
150 self.ca_private = os.path.join(self.ca_dir, 'private')
151 self.ca_certs = os.path.join(self.ca_dir, 'certs')
152 self.ca_crl = os.path.join(self.ca_dir, 'crl')
153 self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
154 self.ca_key_ext = '.key'
155 self.ca_crt_ext = '.crt'
156 for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
157 self._createDirectory(d)
158 for f in ['crlnumber', 'serial']:
159 if not os.path.exists(os.path.join(self.ca_dir, f)):
160 open(os.path.join(self.ca_dir, f), 'w').write('01')
161 if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
162 open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
163 openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
164 config.update(
165 working_directory=self.ca_dir,
166 country_code=ca_country_code,
167 state=ca_state,
168 city=ca_city,
169 company=ca_company,
170 email_address=ca_email,
171 )
172 self._writeFile(openssl_configuration, pkg_resources.resource_string(
173 __name__, 'template/openssl.cnf.ca.in') % config)
174 self.path_list.extend(zc.buildout.easy_install.scripts([
175 ('certificate_authority',
176 __name__ + '.certificate_authority', 'runCertificateAuthority')],
177 self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
178 openssl_configuration=openssl_configuration,
179 openssl_binary=self.options['openssl_binary'],
180 certificate=os.path.join(self.ca_dir, 'cacert.pem'),
181 key=os.path.join(self.ca_private, 'cakey.pem'),
182 crl=os.path.join(self.ca_crl),
183 request_dir=self.ca_request_dir
184 )]))
185 # configure backup
186 backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
187 open(backup_cron, 'w').write(
188 '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
189 rdiff_backup=self.options['rdiff_backup_binary'],
190 source=self.ca_dir,
191 destination=backup_path))
192 self.path_list.append(backup_cron)
193
194 return dict(
195 ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
196 ca_crl=os.path.join(config['ca_dir'], 'crl'),
197 certificate_authority_path=config['ca_dir']
198 )
199
200 def requestCertificate(self, name):
201 hash = hashlib.sha512(name).hexdigest()
202 key = os.path.join(self.ca_private, hash + self.ca_key_ext)
203 certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
204 parser = ConfigParser.RawConfigParser()
205 parser.add_section('certificate')
206 parser.set('certificate', 'name', name)
207 parser.set('certificate', 'key_file', key)
208 parser.set('certificate', 'certificate_file', certificate)
209 parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
210 return key, certificate
211
212 def installStunnel(self, public_ip, private_ip, public_port, private_port,
213 ca_certificate, key, ca_crl, ca_path):
214 """Installs stunnel"""
215 template_filename = self.getTemplateFilename('stunnel.conf.in')
216 log = os.path.join(self.log_directory, 'stunnel.log')
217 pid_file = os.path.join(self.run_directory, 'stunnel.pid')
218 stunnel_conf = dict(
219 public_ip=public_ip,
220 private_ip=private_ip,
221 public_port=public_port,
222 pid_file=pid_file,
223 log=log,
224 cert = ca_certificate,
225 key = key,
226 ca_crl = ca_crl,
227 ca_path = ca_path,
228 private_port = private_port,
229 )
230 stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
231 self.substituteTemplate(template_filename,
232 stunnel_conf))
233 wrapper = zc.buildout.easy_install.scripts([('stunnel',
234 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
235 self.wrapper_directory, arguments=[
236 self.options['stunnel_binary'].strip(), stunnel_conf_path]
237 )[0]
238 self.path_list.append(wrapper)
239 return stunnel_conf
240
241 def installPulse2(self, ip, port, ldap_host, ldap_port):
242 """Installs both mmc_core and pulse2"""
243 config = dict(
244 ldap_host=ldap_host,
245 ldap_port=ldap_port,
246 memcached_port=port,
247 ldap_logfile_path= os.path.join(self.log_directory, 'ldap.log'),
248 mmc_core_binary=self.options['mmc_core_binary']
249 )
250
251 #TODO write function that takes all templates in subdir and creates conf
252 # files, keeping same dir structure.
253 mmc_conf_path = self.createConfigurationFile(os.path.join("mmc",
254 "agent", "config.ini"), self.substituteTemplate(
255 self.getTemplateFilename(os.path.join("mmc_conf",
256 "agent", "config.ini.in")), config))
257 config['mmc_core_config_file'] = mysql_conf_path
258
259 self.path_list.append(self.createRunningWrapper('mmc-core',
260 self.substituteTemplate(self.getTemplateFilename('mmc-core.in'),
261 config)))
262
263
264
265 return dict(memcached_url='%s:%s' %
266 (config['memcached_ip'], config['memcached_port']),
267 memcached_ip=config['memcached_ip'],
268 memcached_port=config['memcached_port'])
269
270 def createHtdocs(self, source, document_root):
271 source = self.options['source'].strip()
272 document_root = self.createDataDirectory('htdocs')
273 for p in os.listdir(document_root):
274 path = os.path.join(document_root, p)
275 if os.path.isdir(path):
276 shutil.rmtree(path)
277 else:
278 os.unlink(path)
279 for p in os.listdir(source):
280 path = os.path.join(source, p)
281 if os.path.isdir(path):
282 shutil.copytree(path, os.path.join(document_root, p))
283 else:
284 shutil.copy2(path, os.path.join(document_root, p))
285
286 def installApache(self, document_root, ip=None, port=None):
287 if ip is None:
288 ip=self.getGlobalIPv6Address()
289 if port is None:
290 port = '9080'
291 apache_config = dict(
292 pid_file=os.path.join(self.run_directory, 'httpd.pid'),
293 lock_file=os.path.join(self.run_directory, 'httpd.lock'),
294 ip=ip,
295 port=port,
296 error_log=os.path.join(self.log_directory, 'httpd-error.log'),
297 access_log=os.path.join(self.log_directory, 'httpd-access.log'),
298 document_root=document_root,
299 php_ini_dir=self.etc_directory
300 )
301 config_file = self.createConfigurationFile('httpd.conf',
302 self.substituteTemplate(pkg_resources.resource_filename(__name__,
303 'template/apache.in'), apache_config))
304 self.path_list.append(config_file)
305 self.path_list.append(self.createConfigurationFile('php.ini',
306 self.substituteTemplate(pkg_resources.resource_filename(__name__,
307 'template/php.ini.in'), {})))
308 self.path_list.extend(zc.buildout.easy_install.scripts([(
309 'httpd',
310 __name__ + '.apache', 'runApache')], self.ws,
311 sys.executable, self.wrapper_directory, arguments=[
312 dict(
313 required_path_list=[],
314 binary=self.options['httpd_binary'],
315 config=config_file
316 )
317 ]))
318 return 'http://[%s]:%s' % (ip, port)
319
320 def installMysqlServer(self, ip, port, database='erp5', user='user',
321 test_database='test_erp5', test_user='test_user', template_filename=None,
322 parallel_test_database_amount=100, mysql_conf=None):
323 if mysql_conf is None:
324 mysql_conf = {}
325 backup_directory = self.createBackupDirectory('mysql')
326 if template_filename is None:
327 template_filename = self.getTemplateFilename('my.cnf.in')
328 error_log = os.path.join(self.log_directory, 'mysqld.log')
329 slow_query_log = os.path.join(self.log_directory, 'mysql-slow.log')
330 mysql_conf.update(
331 ip=ip,
332 data_directory=os.path.join(self.data_root_directory,
333 'mysql'),
334 tcp_port=port,
335 pid_file=os.path.join(self.run_directory, 'mysqld.pid'),
336 socket=os.path.join(self.run_directory, 'mysqld.sock'),
337 error_log=error_log,
338 slow_query_log=slow_query_log,
339 mysql_database=database,
340 mysql_user=user,
341 mysql_password=self.generatePassword(),
342 mysql_test_password=self.generatePassword(),
343 mysql_test_database=test_database,
344 mysql_test_user=test_user,
345 mysql_parallel_test_dict=[
346 ('test_%i' % x,)*2 + (self.generatePassword(),) \
347 for x in xrange(0,parallel_test_database_amount)],
348 )
349 self.registerLogRotation('mysql', [error_log, slow_query_log],
350 '%(mysql_binary)s --no-defaults -B --user=root '
351 '--socket=%(mysql_socket)s -e "FLUSH LOGS"' % dict(
352 mysql_binary=self.options['mysql_binary'],
353 mysql_socket=mysql_conf['socket']))
354 self._createDirectory(mysql_conf['data_directory'])
355
356 mysql_conf_path = self.createConfigurationFile("my.cnf",
357 self.substituteTemplate(template_filename,
358 mysql_conf))
359
360 mysql_script_list = []
361 for x_database, x_user, x_password in \
362 [(mysql_conf['mysql_database'],
363 mysql_conf['mysql_user'],
364 mysql_conf['mysql_password']),
365 (mysql_conf['mysql_test_database'],
366 mysql_conf['mysql_test_user'],
367 mysql_conf['mysql_test_password']),
368 ] + mysql_conf['mysql_parallel_test_dict']:
369 mysql_script_list.append(pkg_resources.resource_string(__name__,
370 'template/initmysql.sql.in') % {
371 'mysql_database': x_database,
372 'mysql_user': x_user,
373 'mysql_password': x_password})
374 mysql_script_list.append('EXIT')
375 mysql_script = '\n'.join(mysql_script_list)
376 self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update',
377 __name__ + '.mysql', 'updateMysql')], self.ws,
378 sys.executable, self.wrapper_directory, arguments=[dict(
379 mysql_script=mysql_script,
380 mysql_binary=self.options['mysql_binary'].strip(),
381 mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(),
382 socket=mysql_conf['socket'],
383 )]))
384 self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld',
385 __name__ + '.mysql', 'runMysql')], self.ws,
386 sys.executable, self.wrapper_directory, arguments=[dict(
387 mysql_install_binary=self.options['mysql_install_binary'].strip(),
388 mysqld_binary=self.options['mysqld_binary'].strip(),
389 data_directory=mysql_conf['data_directory'].strip(),
390 mysql_binary=self.options['mysql_binary'].strip(),
391 socket=mysql_conf['socket'].strip(),
392 configuration_file=mysql_conf_path,
393 )]))
394 self.path_list.extend([mysql_conf_path])
395
396 # backup configuration
397 backup_directory = self.createBackupDirectory('mysql')
398 full_backup = os.path.join(backup_directory, 'full')
399 incremental_backup = os.path.join(backup_directory, 'incremental')
400 self._createDirectory(full_backup)
401 self._createDirectory(incremental_backup)
402 innobackupex_argument_list = [self.options['perl_binary'],
403 self.options['innobackupex_binary'],
404 '--defaults-file=%s' % mysql_conf_path,
405 '--socket=%s' %mysql_conf['socket'].strip(), '--user=root',
406 '--ibbackup=%s'% self.options['xtrabackup_binary']]
407 environment = dict(PATH='%s' % self.bin_directory)
408 innobackupex_incremental = zc.buildout.easy_install.scripts([(
409 'innobackupex_incremental','slapos.recipe.librecipe.execute', 'executee')],
410 self.ws, sys.executable, self.bin_directory, arguments=[
411 innobackupex_argument_list + ['--incremental'],
412 environment])[0]
413 self.path_list.append(innobackupex_incremental)
414 innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full',
415 'slapos.recipe.librecipe.execute', 'executee')], self.ws,
416 sys.executable, self.bin_directory, arguments=[
417 innobackupex_argument_list,
418 environment])[0]
419 self.path_list.append(innobackupex_full)
420 backup_controller = zc.buildout.easy_install.scripts([
421 ('innobackupex_controller', __name__ + '.innobackupex', 'controller')],
422 self.ws, sys.executable, self.bin_directory,
423 arguments=[innobackupex_incremental, innobackupex_full, full_backup,
424 incremental_backup])[0]
425 self.path_list.append(backup_controller)
426 mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup')
427 open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller)
428 self.path_list.append(mysql_backup_cron)
429 # The return could be more explicit database, user ...
430 return mysql_conf
431