Add URL to view and download BOINC result
[slapos.git] / slapos / recipe / boinc / configure.py
1 ##############################################################################
2 #
3 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
4 #
5 # WARNING: This program as such is intended to be used by professional
6 # programmers who take the whole responsibility of assessing all potential
7 # consequences resulting from its eventual inadequacies and bugs
8 # End users who are looking for a ready-to-use solution with commercial
9 # guarantees and support are strongly adviced to contract a Free Software
10 # Service Company
11 #
12 # This program is Free Software; you can redistribute it and/or
13 # modify it under the terms of the GNU General Public License
14 # as published by the Free Software Foundation; either version 3
15 # of the License, or (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
25 #
26 ##############################################################################
27
28 import os
29 import sys
30 import subprocess
31 import time
32 import shutil
33 import re
34 import filecmp
35
36 from lock_file import LockFile
37
38 def checkMysql(args):
39 sys.path += args['environment']['PYTHONPATH'].split(':')
40 import MySQLdb
41 #Sleep until mysql server becomes available
42 while True:
43 try:
44 conn = MySQLdb.connect(host = args['mysql_host'],
45 user = args['mysql_user'],
46 port = int(args['mysql_port']),
47 passwd = args['mysql_password'],
48 db = args['database'])
49 conn.close()
50 print "Successfully connect to MySQL database... "
51 if args.has_key('file_status'):
52 writeFile(args['file_status'], "starting")
53 break
54 except Exception, ex:
55 print "The result is: \n" + ex.message
56 print "Could not connect to MySQL database... sleep for 2 secondes"
57 time.sleep(2)
58
59
60 def checkFile(file, stime):
61 """Loop until 'file' is created (exist)"""
62 while True:
63 print "Search for file %s..." % file
64 if not os.path.exists(file):
65 print "File not found... sleep for %s secondes" % stime
66 time.sleep(stime)
67 else:
68 break
69
70
71 def restart_boinc(args):
72 """Stop (if currently is running state) and start all Boinc service"""
73 if args['drop_install']:
74 checkFile(args['service_status'], 3)
75 else:
76 checkMysql(args)
77 print "Restart Boinc..."
78 env = os.environ
79 env['PATH'] = args['environment']['PATH']
80 env['PYTHONPATH'] = args['environment']['PYTHONPATH']
81 binstart = os.path.join(args['installroot'], 'bin/start')
82 binstop = os.path.join(args['installroot'], 'bin/stop')
83 os.system(binstop)
84 os.system(binstart)
85 writeFile(args['start_boinc'], "started")
86 print "Done."
87
88
89 def check_installRequest(args):
90 print "Cheking if needed to install %s..." % args['appname']
91 install_request_file = os.path.join(args['home_dir'],
92 '.install_' + args['appname'] + args['version'])
93 if not os.path.exists(install_request_file):
94 print "No install or update request for %s version %s..." % (
95 args['appname'], args['version'])
96 return False
97 os.unlink(install_request_file)
98 return True
99
100 def copy_file(source, dest):
101 """"Copy file with source to dest with auto replace
102 return True if file has been copied and dest ha been replaced
103 """
104 result = False
105 if source and os.path.exists(source):
106 if os.path.exists(dest):
107 if filecmp.cmp(dest, source):
108 return False
109 os.unlink(dest)
110 result = True
111 shutil.copy(source, dest)
112 return result
113
114
115 def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
116 process = subprocess.Popen(launch_args, stdout=stdout,
117 stderr=subprocess.STDOUT, env=env,
118 cwd=cwd)
119 result = process.communicate()[0]
120 if process.returncode is None or process.returncode != 0:
121 print "Failed to execute executable.\nThe error was: %s" % result
122 return False
123 return True
124
125 def makeProject(args):
126 """Run BOINC make_project script but once only"""
127 #Wait for DateBase initialization...
128 checkFile(args['make_sig'], 3)
129 print "Cheking if needed to run BOINC make_project..."
130 if os.path.exists(args['request_file']):
131 env = os.environ
132 env['PATH'] = args['env']['PATH']
133 env['PYTHONPATH'] = args['env']['PYTHONPATH']
134 if startProcess(args['launch_args'], env=env):
135 os.unlink(args['request_file'])
136 print "Finished running BOINC make_projet...Ending"
137 else:
138 print "No new request for make_project. Exiting..."
139
140
141 def services(args):
142 """This function configure a new installed boinc project instance"""
143 print "Checking if needed to install or reinstall Boinc-server..."
144 if not args['drop_install']:
145 print "Not need to install Boinc-server...skipped"
146 return
147 #Sleep until file 'boinc_project'.readme exist
148 checkFile(args['readme'], 3)
149
150 topath = os.path.join(args['installroot'], 'html/ops/.htpasswd')
151 print "Generating .htpasswd file... File=%s" % topath
152 passwd = open(args['passwd'], 'r').read()
153 htpwd_args = [args['htpasswd'], '-b', '-c', topath, args['username'], passwd]
154 if not startProcess(htpwd_args):
155 return
156
157 print "execute script xadd..."
158 env = os.environ
159 env['PATH'] = args['environment']['PATH']
160 env['PYTHONPATH'] = args['environment']['PYTHONPATH']
161 if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
162 return
163 print "Update files and directories permissions..."
164 upload = os.path.join(args['installroot'], 'upload')
165 inc = os.path.join(args['installroot'], 'html/inc')
166 languages = os.path.join(args['installroot'], 'html/languages')
167 compiled = os.path.join(args['installroot'], 'html/languages/compiled')
168 user_profile = os.path.join(args['installroot'], 'html/user_profile')
169 forum_file = os.path.join(args['installroot'], 'html/ops/create_forums.php')
170 project_inc = os.path.join(args['installroot'], 'html/project/project.inc')
171 cmd = "chmod 02700 -R %s %s, %s %s %s" % (upload, inc,
172 languages, compiled, user_profile)
173 os.system("chmod g+w -R " + args['installroot'])
174 os.system(cmd)
175 os.system("chmod 700 %s" % os.path.join(args['installroot'], 'keys'))
176 os.system("chmod o+x " + inc)
177 os.system("chmod -R o+r " + inc)
178 os.system("chmod o+x " + languages)
179 os.system("chmod o+x " + compiled)
180 sed_args = [args['sedconfig']]
181 startProcess(sed_args)
182
183 #Execute php create_forum.php...
184 print "Boinc Forum: Execute php create_forum.php..."
185 cwd = os.path.join(args['installroot'], 'html/ops')
186 if not startProcess(["php", forum_file], env, cwd):
187 return
188
189 writeFile(args['service_status'], "started")
190
191 def deployApp(args):
192 """Deploy Boinc App with lock"""
193 print "Asking to enter in execution with lock mode..."
194 with LockFile(args['lockfile'], wait=True):
195 print "acquire the lock file..."
196 deployManagement(args)
197 print "Exit execution with lock..."
198
199 def deployManagement(args):
200 """Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
201 if not check_installRequest(args):
202 return
203 token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
204 newInstall = False
205 if os.path.exists(token):
206 args['previous_wu'] = int(open(token, 'r').read().strip())
207 if args['previous_wu'] < args['wu_number']:
208 print args['appname'] + " Work units will be updated from %s to %s" % (
209 args['previous_wu'], args['wu_number'])
210 else:
211 args['previous_wu'] = 0
212 newInstall = True
213 #Sleep until file .start_boinc exist (File indicate that BOINC has been started)
214 checkFile(args['start_boinc'], 3)
215 env = os.environ
216 env['PATH'] = args['environment']['PATH']
217 env['PYTHONPATH'] = args['environment']['PYTHONPATH']
218
219 print "setup directories..."
220 numversion = args['version'].replace('.', '')
221 args['inputfile'] = os.path.join(args['installroot'], 'download',
222 args['appname'] + numversion + '_input')
223 base_app = os.path.join(args['installroot'], 'apps', args['appname'])
224 base_app_version = os.path.join(base_app, args['version'])
225 args['templates'] = os.path.join(args['installroot'], 'templates')
226 t_result = os.path.join(args['templates'],
227 args['appname'] + numversion + '_result')
228 t_wu = os.path.join(args['templates'],
229 args['appname'] + numversion + '_wu')
230 binary_name = args['appname'] +"_"+ args['version'] +"_"+ \
231 args['platform'] + args['extension']
232 binary = os.path.join(args['application'], binary_name)
233 signBin = False
234 if not os.path.exists(base_app):
235 os.mkdir(base_app)
236 if newInstall:
237 if os.path.exists(base_app_version):
238 shutil.rmtree(base_app_version)
239 os.mkdir(base_app_version)
240 os.mkdir(args['application'])
241 if not os.path.exists(args['templates']):
242 os.mkdir(args['templates'])
243 copy_file(args['t_result'], t_result)
244 copy_file(args['t_wu'], t_wu)
245 signBin = copy_file(args['binary'], binary)
246 if args['t_input']:
247 if os.path.exists(args['inputfile']):
248 os.unlink(args['inputfile'])
249 os.symlink(args['t_input'], args['inputfile'])
250
251 project_xml = os.path.join(args['installroot'], 'project.xml')
252 findapp = re.search("<name>(%s)</name>" % args['appname'],
253 open(project_xml, 'r').read())
254 if not findapp:
255 print "Adding '" + args['appname'] + "' to project.xml..."
256 print "Adding deamon for application to config.xml..."
257 sed_args = [args['bash'], args['appname'], args['installroot']]
258 startProcess(sed_args)
259
260 if signBin:
261 print "Sign the application binary..."
262 sign = os.path.join(args['installroot'], 'bin/sign_executable')
263 privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
264 output = open(binary + '.sig', 'w')
265 p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
266 stderr=subprocess.STDOUT)
267 result = p_sign.communicate()[0]
268 if p_sign.returncode is None or p_sign.returncode != 0:
269 print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
270 return
271 output.close()
272
273 print "execute script xadd..."
274
275 if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
276 return
277 print "Running script bin/update_versions..."
278 updt_version = os.path.join(args['installroot'], 'bin/update_versions')
279 p_version = subprocess.Popen([updt_version], stdout=subprocess.PIPE,
280 stderr=subprocess.STDOUT, stdin=subprocess.PIPE, env=env,
281 cwd=args['installroot'])
282 p_version.stdin.write('y\ny\n')
283 result = p_version.communicate()[0]
284 p_version.stdin.close()
285 if p_version.returncode is None or p_version.returncode != 0:
286 print "Failed to execute bin/update_versions.\nThe error was: %s" % result
287 return
288
289 print "Fill the database... calling bin/create_work..."
290 create_wu(args, env)
291
292 print "Restart Boinc..."
293 binstart = os.path.join(args['installroot'], 'bin/start')
294 binstop = os.path.join(args['installroot'], 'bin/stop')
295 os.system(binstop)
296 os.system(binstart)
297
298 print "Boinc Application deployment is done... writing end signal file..."
299 writeFile(token, str(args['wu_number']))
300
301
302 def create_wu(args, env):
303 """Create or update number of work unit for an existing boinc application"""
304 numversion = args['version'].replace('.', '')
305 t_result = "templates/" + args['appname'] + numversion + '_result'
306 t_wu = "templates/" + args['appname'] + numversion + '_wu'
307 launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
308 '--appname', args['appname'], '--wu_name', '',
309 '--wu_template', t_wu, '--result_template', t_result,
310 '--min_quorum', '1', '--target_nresults', '1',
311 args['appname'] + numversion + '_input']
312 for i in range(args['previous_wu'], args['wu_number']):
313 print "Creating project wroker %s..." % str(i+1)
314 launch_args[4] = args['appname'] + str(i+1) + numversion + '_nodelete'
315 startProcess(launch_args, env, args['installroot'])
316
317
318 def runCmd(args):
319 """Wait for Boinc Client started and run boinc cmd"""
320 client_config = os.path.join(args['installdir'], 'client_state.xml')
321 checkFile(client_config, 5)
322 time.sleep(10)
323 #Scan client state xml to find client ipv4 adress
324 host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
325 open(client_config, 'r').read()).group(1)
326 args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
327 print "Run boinccmd with host at %s " % args['base_cmd'][2]
328 project_args = args['base_cmd'] + ['--project_attach', args['project_url'],
329 args['key']]
330 startProcess(project_args, cwd=args['installdir'])
331 if args['cc_cmd'] != '':
332 #Load or reload cc_config file
333 startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
334
335
336 def writeFile(file, content):
337 f = open(file, 'w')
338 f.write(content)
339 f.close()