runner/resilient stack: add status history
[slapos.git] / software / slaprunner / instance-runner.cfg
1 [buildout]
2 parts =
3   nginx_conf
4   nginx-launcher
5   certificate-authority
6   ca-nginx
7   ca-shellinabox
8   gunicorn-launcher
9   gunicorn-graceful
10   sshkeys-dropbear-runner
11   dropbear-server-add-authorized-key
12   sshkeys-authority
13   publish-connection-informations
14   slaprunner-promise
15   slaprunner-frontend-promise
16   dropbear-promise
17   runtestsuite
18   shellinabox-promise
19   symlinks
20   shellinabox
21   slapos-cfg
22   slapos-repo-config
23   cron-entry-prepare-software
24   deploy-instance-parameters
25 {% if slapparameter_dict.get('custom-frontend-backend-url') %}
26   custom-frontend-promise
27 {% endif %}
28 ## Monitoring part
29 ###Parts to add for monitoring
30   certificate-authority
31   cron-entry-monitor
32   cron-entry-rss
33   deploy-index
34   deploy-settings-cgi
35   deploy-status-cgi
36   deploy-status-history-cgi
37   setup-static-files
38   certificate-authority
39   zero-parameters
40   public-symlink
41   cgi-httpd-wrapper
42   cgi-httpd-graceful-wrapper
43   monitor-promise
44   monitor-instance-log-access
45 ## Monitor for runner
46   monitor-current-log-access
47
48 extends = ${monitor-template:output}
49
50 eggs-directory = ${buildout:eggs-directory}
51 develop-eggs-directory = ${buildout:develop-eggs-directory}
52 offline = true
53
54 {% if slapparameter_dict.get('custom-frontend-backend-url') -%}
55 # Requests, if defined, a frontend to allow access to a server
56 # located inside of the virtual machine listening to port X
57 # to LAN IPv4.
58 # Internaly, the frontend will be asked to listen on the IPv6
59 # with port X + 10000, to match NAT rules of Qemu.
60 [request-custom-frontend]
61 recipe = slapos.cookbook:requestoptional
62 software-url = {{ slapparameter_dict.get('custom-frontend-software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') }}
63 software-type = {{ slapparameter_dict.get('custom-frontend-software-type', 'RootSoftwareInstance') }}
64 slave = true
65 name = Custom Web Frontend
66
67 server-url = $${slap-connection:server-url}
68 key-file = $${slap-connection:key-file}
69 cert-file = $${slap-connection:cert-file}
70 computer-id = $${slap-connection:computer-id}
71 partition-id = $${slap-connection:partition-id}
72
73 {%- if slapparameter_dict.get('custom-frontend-instance-guid') -%}
74 sla = instance_guid
75 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
76 {% endif -%}
77
78 {% set custom_frontend_backend_type = slapparameter_dict.get('custom-frontend-backend-type')%}
79 {% if custom_frontend_backend_type %}
80 config = url type
81 config-type = {{ custom_frontend_backend_type }}
82 {% else %}
83 config = url
84 {% endif -%}
85 config-url = {{ slapparameter_dict.get('custom-frontend-backend-url') }}
86 return = site_url domain
87
88 [custom-frontend-promise]
89 recipe = slapos.cookbook:check_url_available
90 path = $${directory:promises}/custom_frontend_promise
91 url = https://$${request-custom-frontend:connection-domain}
92 {% if slapparameter_dict.get('custom-frontend-basic-auth') -%}
93 check-secure = 1
94 {% endif -%}
95 dash_path = {{ dash_executable_location }}
96 curl_path = {{ curl_executable_location }}
97
98 [publish-connection-informations]
99 custom-frontend-url = https://$${request-custom-frontend:connection-domain}
100 {% endif %}
101
102 # Create all needed directories
103 [directory]
104 recipe = slapos.cookbook:mkdirectory
105 etc = $${buildout:directory}/etc/
106 var = $${buildout:directory}/var/
107 srv = $${buildout:directory}/srv/
108 bin = $${buildout:directory}/bin/
109 tmp = $${buildout:directory}/tmp/
110
111 sshkeys = $${:srv}/sshkeys
112 services = $${:etc}/service/
113 scripts = $${:etc}/run/
114 ssh = $${:etc}/ssh/
115 log = $${:var}/log/
116 run = $${:var}/run/
117 backup = $${:srv}/backup/
118 promises = $${:etc}/promise/
119 test = $${:etc}/test/
120 nginx-data = $${directory:srv}/nginx
121 ca-dir = $${:srv}/ssl
122 project = $${:srv}/runner/project
123
124 [runnerdirectory]
125 recipe = slapos.cookbook:mkdirectory
126 home = $${directory:srv}/runner/
127 test = $${directory:srv}/test/
128 project = $${:home}/project
129 public = $${:home}/public
130 software-root = $${:home}/software
131 instance-root = $${:home}/instance
132 project-test = $${:test}/project
133 software-test = $${:test}/software
134 instance-test = $${:test}/instance
135 sessions = $${buildout:directory}/.sessions
136
137 #Create password recovery code for slaprunner
138 [recovery-code]
139 recipe = slapos.cookbook:generate.password
140 storage-path = $${directory:etc}/.rcode
141 bytes = 8
142
143 [slaprunner]
144 slaprunner = ${buildout:directory}/bin/slaprunner
145 slapos = ${buildout:directory}/bin/slapos
146 slapproxy = ${buildout:directory}/bin/slapproxy
147 supervisor = ${buildout:directory}/bin/slapgrid-supervisorctl
148 git-binary = ${git:location}/bin/git
149 root_check = false
150 slapos.cfg = $${directory:etc}/slapos.cfg
151 working-directory = $${runnerdirectory:home}
152 project-directory = $${runnerdirectory:project}
153 instance_root = $${runnerdirectory:instance-root}
154 software_root = $${runnerdirectory:software-root}
155 #XXX-Nico hardcoded default port because overridden by this buildout config
156 instance-monitor-url = https://[$${:ipv6}]:9685
157 etc_dir = $${directory:etc}
158 log_dir =  $${directory:log}
159 run_dir = $${directory:run}
160 ssh_client = $${sshkeys-dropbear-runner:wrapper}
161 public_key = $${sshkeys-dropbear-runner:public-key}
162 private_key = $${sshkeys-dropbear-runner:private-key}
163 ipv4 = $${slap-network-information:local-ipv4}
164 ipv6 = $${slap-network-information:global-ipv6}
165 instance_root = $${runnerdirectory:instance-root}
166 proxy_port = 50000
167 runner_port = 50005
168 partition-amount = $${slap-parameter:instance-amount}
169 wrapper = $${directory:services}/slaprunner
170 debug = $${slap-parameter:debug}
171 access-url = https://[$${:ipv6}]:$${:runner_port}
172 supervisord_config = $${directory:etc}/supervisord.conf
173 proxy_database = $${slaprunner:working-directory}/proxy.db
174 console = False
175 verbose = False
176 debug = False
177 auto_deploy = $${slap-parameter:auto-deploy}
178 auto_deploy_instance = $${slap-parameter:auto-deploy-instance}
179 autorun = $${slap-parameter:autorun}
180 knowledge0_file = $${buildout:directory}/$${public:filename}
181
182 [test-runner]
183 <= slaprunner
184 slapos.cfg = $${directory:etc}/slapos-test.cfg
185 working-directory = $${runnerdirectory:test}
186 project-directory = $${runnerdirectory:project-test}
187 software-directory = $${runnerdirectory:software-test}
188 instance-directory = $${runnerdirectory:instance-test}
189 proxy_port = 8602
190 etc_dir = $${directory:test}
191
192 [runtestsuite]
193 recipe = slapos.cookbook:wrapper
194 command-line = ${buildout:directory}/bin/slaprunnertest
195 wrapper-path = $${directory:bin}/runTestSuite
196 environment = RUNNER_CONFIG=$${slapos-test-cfg:rendered}
197
198 # Deploy dropbear (minimalist SSH server)
199 [sshkeys-directory]
200 recipe = slapos.cookbook:mkdirectory
201 requests = $${directory:sshkeys}/requests/
202 keys = $${directory:sshkeys}/keys/
203
204 [sshkeys-authority]
205 recipe = slapos.cookbook:sshkeys_authority
206 request-directory = $${sshkeys-directory:requests}
207 keys-directory = $${sshkeys-directory:keys}
208 wrapper = $${directory:services}/sshkeys_authority
209 keygen-binary = ${dropbear:location}/bin/dropbearkey
210
211 [dropbear-runner-server]
212 recipe = slapos.cookbook:dropbear
213 host = $${slap-network-information:global-ipv6}
214 port = 22222
215 home = $${directory:ssh}
216 wrapper = $${directory:bin}/runner_sshd
217 shell = ${bash:location}/bin/bash
218 rsa-keyfile = $${directory:ssh}/server_key.rsa
219 dropbear-binary = ${dropbear:location}/sbin/dropbear
220
221 [sshkeys-dropbear-runner]
222 <= sshkeys-authority
223 recipe = slapos.cookbook:sshkeys_authority.request
224 name = dropbear
225 type = rsa
226 executable = $${dropbear-runner-server:wrapper}
227 public-key = $${dropbear-runner-server:rsa-keyfile}.pub
228 private-key = $${dropbear-runner-server:rsa-keyfile}
229 wrapper = $${directory:services}/runner_sshd
230
231 [dropbear-server-add-authorized-key]
232 <= dropbear-runner-server
233 recipe = slapos.cookbook:dropbear.add_authorized_key
234 key = $${slap-parameter:user-authorized-key}
235
236 #---------------------------
237 #--
238 #-- Set nginx frontend
239
240 [tempdirectory]
241 recipe = slapos.cookbook:mkdirectory
242 client_body_temp_path = $${directory:tmp}/client_body_temp_path
243 proxy_temp_path = $${directory:tmp}/proxy_temp_path
244 fastcgi_temp_path = $${directory:tmp}/fastcgi_temp_path
245 uwsgi_temp_path = $${directory:tmp}/uwsgi_temp_path
246 scgi_temp_path = $${directory:tmp}/scgi_temp_path
247
248 [nginx-frontend]
249 # Options
250 nb_workers = 2
251 # Network
252 local-ip = $${slap-network-information:local-ipv4}
253 port = 30001
254 global-ip = $${slap-network-information:global-ipv6}
255 global-port = $${slaprunner:runner_port}
256 # Backend
257 runner-ip = $${slaprunner:ipv4}
258 runner-port = $${slaprunner:runner_port}
259 # SSL
260 ssl-certificate = $${ca-nginx:cert-file}
261 ssl-key = $${ca-nginx:key-file}
262 # Log
263 path_pid = $${directory:run}/nginx.pid
264 path_log = $${directory:log}/nginx.log
265 path_access_log = $${directory:log}/nginx.access.log
266 path_error_log = $${directory:log}/nginx.error.log
267 path_tmp = $${directory:tmp}/
268 # Config files
269 path_nginx_conf = $${directory:etc}/nginx.conf
270 # Executables
271 bin_nginx = ${nginx-webdav:location}/sbin/nginx
272 bin_launcher = $${directory:bin}/launcher
273 # Utils
274 path_shell = ${dash:location}/bin/dash
275 # Misc.
276 etc_dir = $${directory:etc}
277 work_dir = $${slaprunner:working-directory}
278
279 [nginx_conf]
280 recipe = slapos.recipe.template:jinja2
281 template = ${template_nginx_conf:location}/${template_nginx_conf:filename}
282 rendered = $${nginx-frontend:path_nginx_conf}
283 context =
284     key shellinabox_port shellinabox:port
285     key socket gunicorn:socket
286     section param_nginx_frontend nginx-frontend
287     section param_tempdir tempdirectory
288
289 [nginx-launcher]
290 recipe = slapos.recipe.template:jinja2
291 template = ${template_launcher:location}/${template_launcher:filename}
292 rendered = $${nginx-frontend:bin_launcher}
293 mode = 700
294 context =
295     section param_nginx_frontend nginx-frontend
296
297 [httpd-parameters]
298 path_pid = $${directory:run}/httpd.pid
299 path_error_log = $${directory:log}/httpd-error.log
300 path_access_log = $${directory:log}/httpd-access.log
301 key_file = $${ca-httpd:key-file}
302 cert_file = $${ca-httpd:cert-file}
303 global_ip = $${slap-network-information:global-ipv6}
304 global_port = $${slaprunner:runner_port}
305 monitor_port = $${monitor-parameters:port}
306 monitor_index = $${deploy-index:rendered}
307 working_directory = $${slaprunner:working-directory}
308 dav_lock = $${directory:var}/DavLock
309 etc_dir = $${directory:etc}
310 var_dir = $${directory:var}
311 project_folder = $${directory:project}
312 runner_home = $${runnerdirectory:home}
313 git_http_backend = ${git:location}/libexec/git-core/git-http-backend
314 cgi_httpd_conf = $${cgi-httpd-configuration-file:output}
315
316 [httpd-conf]
317 recipe = slapos.recipe.template:jinja2
318 template = ${template_httpd_conf:location}/${template_httpd_conf:filename}
319 rendered = $${directory:etc}/httpd.conf
320 context =
321     section parameters httpd-parameters
322
323 [cgi-httpd-wrapper]
324 recipe = slapos.cookbook:wrapper
325 apache-executable = ${apache:location}/bin/httpd
326 wrapper-path = $${ca-httpd:executable}
327 command-line = $${:apache-executable} -f $${httpd-conf:rendered} -DFOREGROUND
328
329 #--------------------
330 #--
331 #-- WSGI
332
333 [gunicorn]
334 bin_gunicorn = $${directory:bin}/gunicorn
335 bin_launcher = $${directory:services}/gunicorn
336 path_shell = ${dash:location}/bin/dash
337 socket = $${directory:tmp}/flaskserver.sock
338 path_pid = $${directory:run}/gunicorn.pid
339
340 [gunicorn-launcher]
341 recipe = slapos.cookbook:wrapper
342 command-line = $${gunicorn:bin_gunicorn} slapos.runner:app -p $${gunicorn:path_pid} -b unix:$${gunicorn:socket} -e RUNNER_CONFIG=$${slaprunner:slapos.cfg} --preload
343 wrapper-path = $${gunicorn:bin_launcher}
344 environment = PATH=$${environ:PATH}:${git:location}/bin/
345   RUNNER_CONFIG=$${slaprunner:slapos.cfg}
346
347 [gunicorn-graceful]
348 recipe = slapos.cookbook:wrapper
349 command-line = $${directory:bin}/killpidfromfile $${gunicorn:path_pid} SIGHUP
350 wrapper-path = $${directory:scripts}/gunicorn-graceful
351
352 #--------------------
353 #--
354 #-- ssl certificates
355
356 [certificate-authority]
357 recipe = slapos.cookbook:certificate_authority
358 openssl-binary = ${openssl:location}/bin/openssl
359 ca-dir = $${directory:ca-dir}
360 requests-directory = $${cadirectory:requests}
361 wrapper = $${directory:services}/certificate_authority
362 ca-private = $${cadirectory:private}
363 ca-certs = $${cadirectory:certs}
364 ca-newcerts = $${cadirectory:newcerts}
365 ca-crl = $${cadirectory:crl}
366
367 [cadirectory]
368 recipe = slapos.cookbook:mkdirectory
369 requests = $${directory:ca-dir}/requests/
370 private = $${directory:ca-dir}/private/
371 certs = $${directory:ca-dir}/certs/
372 newcerts = $${directory:ca-dir}/newcerts/
373 crl = $${directory:ca-dir}/crl/
374
375 [ca-nginx]
376 <= certificate-authority
377 recipe = slapos.cookbook:certificate_authority.request
378 key-file = $${cadirectory:certs}/nginx_frontend.key
379 cert-file = $${cadirectory:certs}/nginx_frontend.crt
380 executable = $${nginx-launcher:rendered}
381 wrapper = $${directory:services}/nginx-frontend
382 # Put domain name
383 name = example.com
384
385 [ca-shellinabox]
386 <= certificate-authority
387 recipe = slapos.cookbook:certificate_authority.request
388 executable = $${shellinabox:wrapper}
389 wrapper = $${directory:services}/shellinaboxd
390 key-file = $${cadirectory:certs}/shellinabox.key
391 cert-file = $${cadirectory:certs}/shellinabox.crt
392 #--------------------
393 #--
394 #-- Request frontend
395
396 [request-frontend]
397 <= slap-connection
398 recipe = slapos.cookbook:requestoptional
399 name = SlapRunner Frontend
400 # XXX We have hardcoded SR URL here.
401 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
402 slave = true
403 config = url domain
404 config-url = $${slaprunner:access-url}
405 config-domain = $${slap-parameter:frontend-domain}
406 return = site_url domain
407
408 [monitor-frontend]
409 <= slap-connection
410 recipe = slapos.cookbook:requestoptional
411 name = Monitor Frontend
412 # XXX We have hardcoded SR URL here.
413 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
414 slave = true
415 config = url domain
416 config-url = https://[$${cgi-httpd-configuration-file:listening-ip}]:$${monitor-parameters:port}
417 config-domain = $${slap-parameter:frontend-domain}
418 return = site_url domain
419
420 #--------------------------------------
421 #--
422 #-- Send informations to SlapOS Master
423
424 [publish-connection-informations]
425 recipe = slapos.cookbook:publish
426 1_info = On your first run, Use "access_url" to setup you account. Then you can use both "url" or "access_url". Or "backend_url" if you want to use ipv6. Set up your account in the webrunner in order to use webdav, and being able to clone your git repositories from the runner.
427 2_info = In order to set up your account, get the recovery-code from the monitoring interface. Before read the notification on monitor_info.
428 backend_url = $${slaprunner:access-url}
429 access_url = $${:url}/login
430 url =  https://$${request-frontend:connection-domain}
431 ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
432 monitor_url = https://$${monitor-frontend:connection-domain}
433 webdav_url = $${:monitor_url}/share/
434 public_url =  $${:monitor_url}/public/
435 git_public_url =  https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:monitor_port}/git-public/
436 git_private_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:monitor_port}/git/
437
438 #---------------------------
439 #--
440 #-- Deploy promises scripts
441
442 [slaprunner-promise]
443 recipe = slapos.cookbook:check_port_listening
444 path = $${directory:promises}/slaprunner
445 hostname = $${slaprunner:ipv6}
446 port = $${slaprunner:runner_port}
447
448 [slaprunner-frontend-promise]
449 recipe = slapos.cookbook:check_url_available
450 path = $${directory:promises}/slaprunner_frontend
451 url = https://$${request-frontend:connection-domain}/login
452 dash_path = ${dash:location}/bin/dash
453 curl_path = ${curl:location}/bin/curl
454
455 [dropbear-promise]
456 recipe = slapos.cookbook:check_port_listening
457 path = $${directory:promises}/dropbear
458 hostname = $${dropbear-runner-server:host}
459 port = $${dropbear-runner-server:port}
460
461 [shellinabox-promise]
462 recipe = slapos.cookbook:check_port_listening
463 path = $${directory:promises}/shellinabox
464 hostname = $${shellinabox:ipv6}
465 port = $${shellinabox:port}
466
467 [symlinks]
468 recipe = cns.recipe.symlink
469 symlink_target = $${directory:bin}
470 symlink_base = ${buildout:directory}/bin
471
472 [slap-parameter]
473 # Default value if no ssh key is specified
474 user-authorized-key =
475 # Default value of instances number in slaprunner
476 instance-amount = 10
477 debug = false
478 frontend-domain =
479 slapos-repository = http://git.erp5.org/repos/slapos.git
480 slapos-software =
481 slapos-reference = master
482 auto-deploy = false
483 auto-deploy-instance = true
484 autorun = false
485 monitor-port = 9684
486
487 [monitor-parameters]
488 port = $${slap-parameter:monitor-port}
489
490 [slapos-cfg]
491 recipe = slapos.recipe.template:jinja2
492 template = ${slapos-cfg-template:location}/${slapos-cfg-template:filename}
493 rendered = $${slaprunner:slapos.cfg}
494 mode = 700
495 context =
496   section slaprunner slaprunner
497
498 [slapos-test-cfg]
499 recipe = slapos.recipe.template:jinja2
500 template = ${slapos-cfg-template:location}/${slapos-cfg-template:filename}
501 rendered = $${test-runner:slapos.cfg}
502 mode = 700
503 context =
504   section slaprunner test-runner
505
506 [shellinabox]
507 recipe = slapos.cookbook:shellinabox
508 ipv6 = $${slap-network-information:global-ipv6}
509 port = 8080
510 shell = $${shell:wrapper}
511 wrapper = $${directory:bin}/shellinaboxd
512 shellinabox-binary = ${shellinabox:location}/bin/shellinaboxd
513 password = $${zero-parameters:shell-password}
514 directory = $${runnerdirectory:home}
515 login-shell = $${directory:bin}/login
516 certificate-directory = $${cadirectory:certs}
517 cert-file = $${ca-shellinabox:cert-file}
518 key-file = $${ca-shellinabox:key-file}
519
520 [shellinabox-code]
521 recipe = slapos.cookbook:generate.password
522 storage-path = $${directory:etc}/.scode
523 bytes = 8
524
525 [shell]
526 recipe = slapos.cookbook:shell
527 wrapper = $${directory:bin}/sh
528 shell = ${bash:location}/bin/bash
529 home = $${runnerdirectory:home}
530 path = $${environ:PATH}:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin
531 ps1 = "\\w> "
532
533 [environ]
534 recipe = collective.recipe.environment
535
536 [slapos-repo]
537 recipe = slapos.recipe.build:gitclone
538 repository = $${slap-parameter:slapos-repository}
539 git-executable = ${git:location}/bin/git
540 develop = true
541 location = $${directory:project}/slapos
542
543 [slapos-repo-config]
544 recipe = plone.recipe.command
545 stop-on-error = true
546 command = cd $${slapos-repo:location} && ${git:location}/bin/git checkout $${slap-parameter:slapos-reference} && SR=$${slap-parameter:slapos-software} && if [ -n "$SR" ] && [ ! -f "$${directory:etc}/.project" ]; then echo workspace/slapos/$${slap-parameter:slapos-software}/ > $${directory:etc}/.project; fi
547 update-command = true
548
549 [prepare-software]
550 recipe = slapos.cookbook:wrapper
551 command-line = ${curl:location}/bin/curl -g https://[$${slaprunner:ipv6}]:$${slaprunner:runner_port}/isSRReady --max-time 1 --insecure
552 wrapper-path = $${directory:scripts}/prepareSoftware
553
554 [cron-entry-prepare-software]
555 <= cron
556 recipe = slapos.cookbook:cron.d
557 name = prepare-software
558 frequency = */2 * * * *
559 command = $${prepare-software:wrapper-path}
560
561 [instance-parameters]
562 recipe = slapos.recipe.template:jinja2
563 extensions = jinja2.ext.do
564 template = ${parameters-template:location}/${parameters-template:filename}
565 rendered = $${directory:etc}/.parameter.xml.default
566 mode = 0644
567 context =
568   key slapparameter_dict slap-configuration:configuration
569
570 [deploy-instance-parameters]
571 recipe = plone.recipe.command
572 stop-on-error = true
573 parameter-xml = $${directory:etc}/.parameter.xml
574 command = if [ ! -f $${:parameter-xml} ]; then cp $${instance-parameters:rendered} $${:parameter-xml}; fi
575
576 [slap-configuration]
577 recipe = slapos.cookbook:slapconfiguration.serialised
578 computer = $${slap-connection:computer-id}
579 partition = $${slap-connection:partition-id}
580 url = $${slap-connection:server-url}
581 key = $${slap-connection:key-file}
582 cert = $${slap-connection:cert-file}
583
584 [public]
585 shell-password = $${shellinabox-code:passwd}
586 recovery-code = $${recovery-code:passwd}
587
588 [zero-parameters]
589
590
591 [monitor-current-log-access]
592 < = monitor-directory-access
593 source = $${directory:log}