Add URL to view and download BOINC result
[slapos.git] / slapos / recipe / boinc / __init__.py
1 ##############################################################################
2 #
3 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
4 #
5 # WARNING: This program as such is intended to be used by professional
6 # programmers who take the whole responsibility of assessing all potential
7 # consequences resulting from its eventual inadequacies and bugs
8 # End users who are looking for a ready-to-use solution with commercial
9 # guarantees and support are strongly adviced to contract a Free Software
10 # Service Company
11 #
12 # This program is Free Software; you can redistribute it and/or
13 # modify it under the terms of the GNU General Public License
14 # as published by the Free Software Foundation; either version 3
15 # of the License, or (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
25 #
26 ##############################################################################
27 from slapos.recipe.librecipe import GenericBaseRecipe
28 import os
29 import subprocess
30 import pwd
31 import json
32 import signal
33 import zc.buildout
34
35 class Recipe(GenericBaseRecipe):
36 """Deploy a fully operational boinc architecture."""
37
38 def __init__(self, buildout, name, options):
39 #get current slapuser name
40 stat_info = os.stat(options['home'].strip())
41 options['user'] = pwd.getpwuid(stat_info.st_uid)[0]
42 url_base = options['url-base']
43 project = options['project'].strip()
44 root = options['installroot'].strip()
45 options['home_page'] = url_base + "/" + project
46 options['admin_page'] = url_base + "/" + project + "_ops/"
47 options['result_page'] = url_base + "/" + project + "_result/"
48 options['cronjob'] = os.path.join(root, project+'.cronjob')
49
50 return GenericBaseRecipe.__init__(self, buildout, name, options)
51
52 def _options(self, options):
53 #Path of boinc compiled package
54 self.package = options['boinc'].strip()
55 self.sourcedir = options['source'].strip()
56 self.home = options['home'].strip()
57 self.project = options['project'].strip()
58 self.fullname = options['fullname'].strip()
59 self.copyright = options['copyright'].strip()
60 self.installroot = options['installroot'].strip()
61 self.boinc_egg = os.path.join(self.package, 'lib/python2.7/site-packages')
62 self.developegg = options['develop-egg'].strip()
63 self.wrapperdir = options['wrapper-dir'].strip()
64 self.passwd = options['passwd'].strip()
65 #Get binary path
66 self.svn = options['svn-binary'].strip()
67 self.perl = options['perl-binary'].strip()
68 self.pythonbin = options['python-binary'].strip()
69
70 #Apache php informations
71 self.url_base =options['url-base'].strip()
72 self.htpasswd = options['htpasswd'].strip()
73 self.phpini = options['php-ini'].strip()
74 self.phpbin = options['php-bin'].strip()
75
76 #get Mysql parameters
77 self.username = options['mysql-username'].strip()
78 self.password = options['mysql-password'].strip()
79 self.database = options['mysql-database'].strip()
80 self.mysqlhost = options['mysql-host'].strip()
81 self.mysqlport = options['mysql-port'].strip()
82
83 def haschanges(self):
84 config_file = os.path.join(self.home, '.config')
85 current = [self.fullname, self.copyright,
86 self.password, self.mysqlhost, self.installroot,
87 self.project, self.passwd, self.url_base]
88 previous = []
89 result = False
90 if os.path.exists(config_file):
91 previous = open(config_file, 'r').read().split('#')
92 #Check if config has changed
93 if len(current) != len(set(current).intersection(set(previous))) or \
94 not os.path.exists(self.installroot) or \
95 not os.path.exists(os.path.join(self.home, '.start_service')):
96 result = True
97 open(config_file, 'w').write('#'.join(current))
98 return result
99
100 def install(self):
101 path_list = []
102 make_project = os.path.join(self.package, 'bin/make_project')
103 niceprojectname = self.project + "@Home"
104 slapuser = self.options['user']
105
106 #Check if given URL is not empty (case of URL request with frontend)
107 if not self.url_base:
108 raise Exception("URL_BASE is still empty. Can not use it")
109
110 #Define environment variable here
111 python = os.path.join(self.home, 'bin/python')
112 python_path = self.boinc_egg
113 if not os.path.exists(python):
114 os.symlink(self.pythonbin, python)
115 for f in os.listdir(self.developegg):
116 dir = os.path.join(self.developegg, f)
117 if os.path.isdir(dir):
118 python_path += ":" + dir
119 bin_dir = os.path.join(self.home, 'bin')
120 environment = dict(
121 PATH=os.pathsep.join([self.svn, bin_dir, self.perl, os.environ['PATH']]),
122 PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
123 )
124
125 #Generate wrapper for php
126 wrapperphp = os.path.join(self.home, 'bin/php')
127 php_wrapper = self.createPythonScript(wrapperphp,
128 'slapos.recipe.librecipe.execute.executee',
129 ([self.phpbin, '-c', self.phpini], os.environ)
130 )
131 path_list.append(php_wrapper)
132
133 #Generate python script for MySQL database test (starting)
134 file_status = os.path.join(self.home, '.boinc_config')
135 if os.path.exists(file_status):
136 os.unlink(file_status)
137 mysql_wrapper = self.createPythonScript(
138 os.path.join(self.wrapperdir, 'start_config'),
139 '%s.configure.checkMysql' % __name__,
140 dict(mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
141 mysql_user=self.username, mysql_password=self.password,
142 database=self.database,
143 file_status=file_status, environment=environment
144 )
145 )
146
147 # Generate make project wrapper file
148 readme_file = os.path.join(self.installroot, self.project+'.readme')
149 launch_args = [make_project, '--url_base', self.url_base, "--db_name",
150 self.database, "--db_user", self.username, "--db_passwd",
151 self.password, "--project_root", self.installroot, "--db_host",
152 self.mysqlhost, "--user_name", slapuser, "--srcdir",
153 self.sourcedir, "--no_query"]
154 drop_install = self.haschanges()
155 request_make_boinc = os.path.join(self.home, '.make_project')
156 if drop_install:
157 #Allow to restart Boinc installation from the begining
158 launch_args += ["--delete_prev_inst", "--drop_db_first"]
159 open(request_make_boinc, 'w').write('Make Requested')
160 if os.path.exists(readme_file):
161 os.unlink(readme_file)
162 launch_args += [self.project, niceprojectname]
163
164 install_wrapper = self.createPythonScript(
165 os.path.join(self.wrapperdir, 'make_project'),
166 '%s.configure.makeProject' % __name__,
167 dict(launch_args=launch_args, request_file=request_make_boinc,
168 make_sig=file_status, env=environment)
169 )
170 path_list.append(install_wrapper)
171
172 #generate sh script for project configuration
173 bash = os.path.join(self.home, 'bin', 'project_config.sh')
174 sh_script = self.createFile(bash,
175 self.substituteTemplate(self.getTemplateFilename('project_config.in'),
176 dict(dash=self.options['dash'].strip(),
177 uldl_pid=self.options['apache-pid'].strip(),
178 user=slapuser, fullname=self.fullname,
179 copyright=self.copyright, installroot=self.installroot))
180 )
181 path_list.append(sh_script)
182 os.chmod(bash , 0700)
183
184 #After make_project run configure_script to perform and restart apache php services
185 service_status = os.path.join(self.home, '.start_service')
186 parameter = dict(
187 readme=readme_file,
188 htpasswd=self.htpasswd,
189 installroot=self.installroot,
190 username=slapuser,
191 passwd=self.passwd,
192 xadd=os.path.join(self.installroot, 'bin/xadd'),
193 environment=environment,
194 service_status=service_status,
195 drop_install=drop_install,
196 sedconfig=bash
197 )
198 start_service = self.createPythonScript(
199 os.path.join(self.wrapperdir, 'config_project'),
200 '%s.configure.services' % __name__, parameter
201 )
202 path_list.append(start_service)
203
204 #Generate Boinc start project wrapper
205 start_args = [os.path.join(self.installroot, 'bin/start')]
206 start_boinc = os.path.join(self.home, '.start_boinc')
207 if os.path.exists(start_boinc):
208 os.unlink(start_boinc)
209 boinc_parameter = dict(service_status=service_status,
210 installroot=self.installroot, drop_install=drop_install,
211 mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
212 mysql_user=self.username, mysql_password=self.password,
213 database=self.database, environment=environment,
214 start_boinc=start_boinc)
215 start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
216 'start_boinc'),
217 '%s.configure.restart_boinc' % __name__,
218 boinc_parameter
219 )
220 path_list.append(start_wrapper)
221
222 return path_list
223
224 update = install
225
226
227 class App(GenericBaseRecipe):
228 """This recipe allow to deploy an scientific applications using boinc
229 Note that recipe use depend on boinc-server parameter"""
230
231 def downloadFiles(self, app):
232 """This is used to download app files if necessary and update options values"""
233 for key in ('input-file', 'template-result', 'template-wu', 'binary'):
234 param = app[key]
235 if param and (param.startswith('http') or param.startswith('ftp')):
236 #download the specified file
237 cache = os.path.join(self.options['home'].strip(), 'tmp')
238 downloader = zc.buildout.download.Download(self.buildout['buildout'],
239 hash_name=True, cache=cache)
240 path, _ = downloader(param, md5sum=None)
241 mode = 0600
242 if key == 'binary':
243 mode = 0700
244 os.chmod(path, mode)
245 app[key] = path
246
247 def getAppList(self):
248 """Load parameters,
249 check if parameter send is valid to install or update application"""
250 app_list = json.loads(self.options['boinc-app-list'])
251 if not app_list:
252 return None
253 default_template_result = self.options.get('default-template-result', '').strip()
254 default_template_wu = self.options.get('default-template-wu', '').strip()
255 default_extension = self.options.get('default-extension', '').strip()
256 default_platform = self.options.get('default-platform', '').strip()
257 for app in app_list:
258 for version in app_list[app]:
259 current_app = app_list[app][version]
260 #Use default value if empty and Use_default is True
261 #Initialize all values to empty if not define by the user
262 if current_app['use_default']:
263 current_app['template-result'] = current_app.get('template-result',
264 default_template_result).strip()
265 current_app['template-wu'] = current_app.get('template-wu',
266 default_template_wu).strip()
267 current_app['extension'] = current_app.get('extension',
268 default_extension).strip()
269 current_app['platform'] = current_app.get('platform',
270 default_platform).strip()
271 else:
272 current_app['template-result'] = current_app.get('template-result', '').strip()
273 current_app['template-wu'] = current_app.get('template-wu', '').strip()
274 current_app['extension'] = current_app.get('extension', '').strip()
275 current_app['platform'] = current_app.get('platform', '').strip()
276 current_app['input-file'] = current_app.get('input-file', '').strip()
277 current_app['wu-number'] = current_app.get('wu-number', 1)
278 #for new application, check if parameter is complete
279 appdir = os.path.join(self.options['installroot'].strip(), 'apps',
280 app, version)
281 if not os.path.exists(appdir):
282 if not current_app['template-result'] or not current_app['binary'] \
283 or not current_app['input-file'] or not current_app['template-wu'] \
284 or not current_app['platform']:
285 print "BOINC-APP: ERROR - Invalid argements values for % ...operation cancelled" % app
286 app_list[app][version] = None
287 continue
288 #write application to install
289 request_file = os.path.join(self.options['home'].strip(),
290 '.install_' + app + version)
291 toInstall = open(request_file, 'w')
292 toInstall.write('install or update')
293 toInstall.close()
294 return app_list
295
296 def install(self):
297
298 app_list = self.getAppList()
299
300 path_list = []
301 package = self.options['boinc'].strip()
302 #Define environment variable here
303 developegg = self.options['develop-egg'].strip()
304 python_path = os.path.join(package, 'lib/python2.7/site-packages')
305 home = self.options['home'].strip()
306 user = pwd.getpwuid(os.stat(home).st_uid)[0]
307 perl = self.options['perl-binary'].strip()
308 svn = self.options['svn-binary'].strip()
309 for f in os.listdir(developegg):
310 dir = os.path.join(developegg, f)
311 if os.path.isdir(dir):
312 python_path += ":" + dir
313 bin_dir = os.path.join(home, 'bin')
314 environment = dict(
315 PATH=os.pathsep.join([svn, bin_dir, perl, os.environ['PATH']]),
316 PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
317 )
318
319 #generate project.xml and config.xml script updater
320 bash = os.path.join(home, 'bin', 'update_config.sh')
321 sh_script = self.createFile(bash,
322 self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
323 dict(dash=self.options['dash'].strip()))
324 )
325 path_list.append(sh_script)
326 os.chmod(bash , 0700)
327
328 #If useful, download necessary files and update options path
329 start_boinc = os.path.join(home, '.start_boinc')
330 installroot = self.options['installroot'].strip()
331 apps_dir = os.path.join(installroot, 'apps')
332 wrapperdir = self.options['wrapper-dir'].strip()
333 project = self.options['project'].strip()
334 lockfile = os.path.join(self.options['home'].strip(), 'app_install.lock')
335 fd = os.open(lockfile, os.O_RDWR|os.O_CREAT)
336 os.close( fd )
337
338 for appname in app_list:
339 for version in app_list[appname]:
340 if not app_list[appname][version]:
341 continue
342 self.downloadFiles(app_list[appname][version])
343 platform = app_list[appname][version]['platform']
344 application = os.path.join(apps_dir, appname, version, platform)
345 if app_list[appname][version]['binary'] and not platform:
346 print "BOINC-APP: WARNING - Cannot specify binary without giving platform value"
347 app_list[appname][version]['binary'] = '' #Binary will not be updated
348
349 parameter = dict(installroot=installroot,
350 appname=appname, project=project,
351 version=version, platform=platform,
352 application=application, environment=environment,
353 start_boinc=start_boinc,
354 wu_number=app_list[appname][version]['wu-number'],
355 t_result=app_list[appname][version]['template-result'],
356 t_wu=app_list[appname][version]['template-wu'],
357 t_input=app_list[appname][version]['input-file'],
358 binary=app_list[appname][version]['binary'],
359 extension=app_list[appname][version]['extension'],
360 bash=bash, home_dir=home,
361 lockfile=lockfile,
362 )
363 deploy_app = self.createPythonScript(
364 os.path.join(wrapperdir, 'boinc_%s' % appname),
365 '%s.configure.deployApp' % __name__, parameter
366 )
367 path_list.append(deploy_app)
368
369 return path_list
370
371 update = install
372
373 class Client(GenericBaseRecipe):
374 """Deploy a fully fonctionnal boinc client connected to a boinc server instance"""
375
376 def __init__(self, buildout, name, options):
377 #get current uig to create a unique rpc-port for this client
378 stat_info = os.stat(options['home'].strip())
379 options['rpc-port'] = pwd.getpwuid(stat_info.st_uid)[2] + 5000
380
381 return GenericBaseRecipe.__init__(self, buildout, name, options)
382
383 def install(self):
384 path_list = []
385 boincbin = self.options['boinc-bin'].strip()
386 cmdbin = self.options['cmd-bin'].strip()
387 installdir = self.options['install-dir'].strip()
388 url = self.options['server-url'].strip()
389 key = self.options['key'].strip()
390 boinc_wrapper = self.options['client-wrapper'].strip()
391 cmd_wrapper = self.options['cmd-wrapper'].strip()
392 remote_host = os.path.join(installdir, 'remote_hosts.cfg')
393 open(remote_host, 'w').write(self.options['ip'].strip())
394
395 #Generate wrapper for boinc cmd
396 base_cmd = [cmdbin, '--host', str(self.options['rpc-port']),
397 '--passwd', self.options['passwd'].strip()]
398 cc_cmd = ''
399 if self.options['cconfig'].strip() != '':
400 config_dest = os.path.join(installdir, 'cc_config.xml')
401 file = open(config_dest, 'w')
402 file.write(open(self.options['cconfig'].strip(), 'r').read())
403 file.close()
404 cc_cmd = '--read_cc_config'
405 cmd = self.createPythonScript(cmd_wrapper,
406 '%s.configure.runCmd' % __name__,
407 dict(base_cmd=base_cmd, cc_cmd=cc_cmd, installdir=installdir,
408 project_url=url, key=key)
409 )
410 path_list.append(cmd)
411
412 #Generate BOINC client wrapper
413 boinc = self.createPythonScript(boinc_wrapper,
414 'slapos.recipe.librecipe.execute.execute',
415 [boincbin, '--allow_multiple_clients', '--gui_rpc_port',
416 str(self.options['rpc-port']), '--allow_remote_gui_rpc',
417 '--dir', installdir, '--redirectio', '--check_all_logins']
418 )
419 path_list.append(boinc)
420
421 return path_list