Use option instead of going through buildout to get slave instance list
[slapos.git] / slapos / recipe / pbs.py
1 ##############################################################################
2 #
3 # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
4 #
5 # WARNING: This program as such is intended to be used by professional
6 # programmers who take the whole responsibility of assessing all potential
7 # consequences resulting from its eventual inadequacies and bugs
8 # End users who are looking for a ready-to-use solution with commercial
9 # guarantees and support are strongly adviced to contract a Free Software
10 # Service Company
11 #
12 # This program is Free Software; you can redistribute it and/or
13 # modify it under the terms of the GNU General Public License
14 # as published by the Free Software Foundation; either version 3
15 # of the License, or (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # GNU General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, write to the Free Software
24 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
25 #
26 ##############################################################################
27 from json import loads as unjson
28 from hashlib import sha512
29 from urlparse import urlparse
30 import os
31 import subprocess
32 import sys
33 import signal
34 import inspect
35
36 from slapos.recipe.librecipe import GenericSlapRecipe
37 from slapos.recipe.dropbear import KnownHostsFile
38 from slapos.recipe.notifier import Notify
39 from slapos.recipe.notifier import Callback
40 from slapos import slap as slapmodule
41
42
43 def promise(args):
44
45 def failed_ssh(partition, ssh):
46 # Bad python 2 syntax, looking forward python 3 to have print(file=)
47 print >> sys.stderr, "SSH Connection failed"
48 try:
49 ssh.terminate()
50 except:
51 pass
52 partition.bang("SSH Connection failed. rdiff-backup is unusable.")
53
54 def sigterm_handler(signum, frame):
55 # Walk up in the stack to get promise local
56 # variables
57 ssh = None
58 for upper_frame in inspect.getouterframes(frame):
59 # Use promise.func_name insteand of 'promise' in order to be
60 # detected by editor if promise func name change.
61 # Else, it's hard to debug this kind of error.
62 if upper_frame[3] == promise.func_name:
63 try:
64 partition = upper_frame[0].f_locals['partition']
65 ssh = upper_frame[0].f_locals['ssh']
66 except KeyError:
67 raise SystemExit("SIGTERM Send too soon.")
68 break
69 # If ever promise function wasn't found in the stack.
70 if ssh is None:
71 raise SystemExit
72 failed_ssh(partition, ssh)
73
74 signal.signal(signal.SIGTERM, sigterm_handler)
75
76 slap = slapmodule.slap()
77 slap.initializeConnection(args['server_url'],
78 key_file=args.get('key_file'), cert_file=args.get('cert_file'))
79 partition = slap.registerComputerPartition(args['computer_id'],
80 args['partition_id'])
81
82 # Rdiff Backup protocol quit command
83 quitcommand = 'q' + chr(255) + chr(0) * 7
84 ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
85
86 ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
87 stdout=open(os.devnull), stderr=open(os.devnull))
88 ssh.stdin.write(quitcommand)
89 ssh.stdin.flush()
90 ssh.stdin.close()
91 ssh.wait()
92
93 if ssh.poll() is None:
94 return 1
95 if ssh.returncode != 0:
96 failed_ssh(partition, ssh)
97 return ssh.returncode
98
99
100
101 class Recipe(GenericSlapRecipe, Notify, Callback):
102
103 def add_slave(self, entry, known_hosts_file):
104 path_list = []
105
106 url = entry.get('url')
107 if url is None:
108 url = ''
109
110 # We assume that thanks to sha512 there's no collisions
111 url_hash = sha512(url).hexdigest()
112 name_hash = sha512(entry['name']).hexdigest()
113
114 promise_path = os.path.join(self.options['promises-directory'],
115 url_hash)
116 parsed_url = urlparse(url)
117 promise_dict = self.promise_base_dict.copy()
118 promise_dict.update(user=parsed_url.username,
119 host=parsed_url.hostname,
120 port=parsed_url.port)
121 promise = self.createPythonScript(promise_path,
122 __name__ + '.promise',
123 promise_dict)
124 path_list.append(promise)
125
126
127 host = parsed_url.hostname
128 known_hosts_file[host] = entry['server-key']
129
130 remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \
131 {
132 'ssh': self.options['sshclient-binary'],
133 'user': parsed_url.username,
134 'host': parsed_url.hostname,
135 }
136
137 command = [self.options['rdiffbackup-binary']]
138 command.extend(['--remote-schema', remote_schema])
139
140 remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
141 'path': parsed_url.path}
142
143 local_directory = self.createDirectory(self.options['directory'],
144 name_hash)
145
146 if entry['type'] == 'push':
147 command.extend(['--restore-as-of', 'now'])
148 command.append('--force')
149 command.extend([local_directory, remote_directory])
150 else:
151 command.extend([remote_directory, local_directory])
152
153 wrapper_basepath = os.path.join(self.options['wrappers-directory'],
154 url_hash)
155
156 wrapper_path = wrapper_basepath
157 if 'notify' in entry:
158 wrapper_path = '%s_raw' % wrapper_basepath
159
160 wrapper = self.createPythonScript(
161 wrapper_path,
162 'slapos.recipe.librecipe.execute.execute',
163 [str(i) for i in command]
164 )
165 path_list.append(wrapper)
166
167 if 'notify' in entry:
168 feed_url = '%s/get/%s' % (self.options['notifier-url'],
169 entry['notification-id'])
170 wrapper = self.createNotifier(
171 self.options['notifier-binary'],
172 wrapper=wrapper_basepath,
173 executable=wrapper_path,
174 log=os.path.join(self.options['feeds'], entry['notification-id']),
175 title=entry.get('title', 'Untitled'),
176 notification_url=entry['notify'],
177 feed_url=feed_url,
178 )
179 path_list.append(wrapper)
180 #self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
181
182 if 'on-notification' in entry:
183 path_list.append(self.createCallback(str(entry['on-notification']),
184 wrapper))
185 else:
186 cron_entry = os.path.join(self.options['cron-entries'], url_hash)
187 with open(cron_entry, 'w') as cron_entry_file:
188 cron_entry_file.write('%s %s' % (entry['frequency'], wrapper))
189 path_list.append(cron_entry)
190
191 return path_list
192
193 def _install(self):
194 path_list = []
195
196
197 if self.optionIsTrue('client', True):
198 self.logger.info("Client mode")
199
200 slap_connection = self.buildout['slap-connection']
201 self.promise_base_dict = dict(
202 server_url=slap_connection['server-url'],
203 computer_id=slap_connection['computer-id'],
204 cert_file=slap_connection.get('cert-file'),
205 key_file=slap_connection.get('key-file'),
206 partition_id=slap_connection['partition-id'],
207 ssh_client=self.options['sshclient-binary'],
208 )
209
210 slaves = unjson(self.options['slave-instance-list'])
211 known_hosts = KnownHostsFile(self.options['known-hosts'])
212 with known_hosts:
213 for slave in slaves:
214 path_list.extend(self.add_slave(slave, known_hosts))
215
216 else:
217 command = [self.options['rdiffbackup-binary']]
218 self.logger.info("Server mode")
219 command.extend(['--restrict', self.options['path']])
220 command.append('--server')
221
222 wrapper = self.createPythonScript(
223 self.options['wrapper'],
224 'slapos.recipe.librecipe.execute.execute',
225 command)
226 path_list.append(wrapper)
227
228 return path_list