Commit a755772d authored by Philippe Pepiot's avatar Philippe Pepiot Committed by Denis Laxalde
Browse files

working formula on centos 6 (#1)

* working formula on centos 6

- Add dirty supervisor installation & config for centos 6
- Split states (install, config, supervisor)

* Test and dev with docker and testinfra

Also add a travis config
parent 39ca8c0e
.cache
test/*_salted.Dockerfile
sudo: required
services:
- docker
language: python
python:
- "2.7"
env:
matrix:
- IMAGE=centos6
install:
- pip install testinfra
script:
- ./run-tests.py test $IMAGE -v
saem formula
================
saemref formula
===============
0.0.0
- Work in progress
......@@ -2,14 +2,52 @@
saemref-formula
===============
A saltstack formula handling installation of `saem_ref`_.
Available states
================
.. contents::
:local:
``saemref``
-----------
``saemref.install``
-------------------
Installs the saemref package from logilab repositories and create an instance.
``saemref.config``
------------------
Manage instance configuration files.
``saemref.supervisor``
----------------------
Install and manage `supervisor`_ configuration for the saemref installation.
Testing
=======
The script `run-test.py` can help to develop and test the formula using
`docker`_ and `testinfra`_.
The command ``./run-test.py dev centos6`` will build image from
``test/centos6.Dockerfile`` and span a shell in a new container with mounted
volumes from the host (so you can develop formula on the host and test it in
the container with ``salt-call``).
The command ``./run-test.py dev centos6 --salt`` is the same as above but will
provision the container (eg. calling ``salt-call state.highstate``)
The command ``./run-test.py test centos6`` built a provisioned image
(highstate) and run testinfra tests on it.
Note that you can add any parameters that testinfra (pytest) accept, for
instance ``./run-test.py test centos6 -k idempotence --pdb``.
``saemref.db-create``
---------------------
.. _saem_ref: https://www.cubicweb.org/project/cubicweb-saem_ref
.. _supervisor: http://supervisord.org
.. _docker: https://www.docker.com/
.. _testinfra: https://testinfra.readthedocs.org/
saemref:
instance:
name: saemref
user: saemref
db:
host: ""
port: ""
name: saemref
user: saemref
pass: saemref
admin:
login: admin
pass: admin
lookup:
instance:
name: saemref
user: saemref
db:
host: ""
port: ""
name: saemref
user: saemref
pass: saemref
admin:
login: admin
pass: admin
#!/usr/bin/env python
import os
import argparse
import subprocess
BASEDIR = os.path.abspath(os.path.dirname(__file__))
_formula = "saemref"
_images = ["centos6"]
def get_tag(image, salt=False):
tag = "{0}-formula:{1}".format(_formula, image)
if salt:
return tag + "_salted"
else:
return tag
def image_exists(image):
# docker image accept tags only in version >= 1.8...
# so workaround with docker history
rc = subprocess.call("docker history -q {0} > /dev/null".format(image), shell=True)
if rc == 0:
return True
elif rc == 1:
return False
else:
raise RuntimeError("Cannot test if image exists")
def _build(image, salt=False):
dockerfile = "test/{0}.Dockerfile".format(image)
tag = get_tag(image, salt)
if salt:
dockerfile_content = open(dockerfile, "rb").read()
dockerfile_content += (
b"\n"
b"ADD test/minion.conf /etc/salt/minion.d/minion.conf\n"
b"ADD test/salt /srv/salt\n"
b"ADD test/pillar /srv/pillar\n"
b"ADD {0} /srv/formula/{0}\n"
b"RUN salt-call -l debug state.highstate\n"
).format(_formula)
dockerfile = os.path.join("test", "{0}_salted.Dockerfile".format(image))
with open(dockerfile, "wb") as fd:
fd.write(dockerfile_content)
subprocess.check_call([
"docker", "build", "-t", tag, "-f", dockerfile, ".",
])
def build(args, remain):
_build(args.image, args.salt)
def test(args, remain):
tag = get_tag(args.image, True)
_build(args.image, True)
import pytest
pytest.main(["--docker-image", tag] + remain)
def dev(args, remain):
tag = get_tag(args.image, args.salt)
if not image_exists(tag):
_build(args.image, args.salt)
subprocess.call([
"docker", "run", "--rm", "-it",
"-v", "{0}/test/minion.conf:/etc/salt/minion.d/minion.conf".format(BASEDIR),
"-v", "{0}/test/salt:/srv/salt".format(BASEDIR),
"-v", "{0}/test/pillar:/srv/pillar".format(BASEDIR),
"-v", "{0}/{1}:/srv/formula/{1}".format(BASEDIR, _formula),
tag, "/bin/bash",
])
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Test {0} formula in docker".format(_formula))
subparsers = parser.add_subparsers(help="sub-command help")
parser_build = subparsers.add_parser("build", help="build images")
parser_build.add_argument("image", choices=_images)
parser_build.add_argument("--salt", action="store_true")
parser_build.set_defaults(func=build)
parser_dev = subparsers.add_parser("dev", help="drop a shell in dev container")
parser_dev.add_argument("image", choices=_images)
parser_dev.add_argument("--salt", action="store_true")
parser_dev.set_defaults(func=dev)
parser_test = subparsers.add_parser("test", help="provision a container and run tests on it")
parser_test.add_argument("image", choices=_images)
parser_test.set_defaults(func=test)
args, remain = parser.parse_known_args()
args.func(args, remain)
{% from "saemref/map.jinja" import saemref with context %}
include:
- saemref.install
cubicweb-config:
file.managed:
- name: /home/{{ saemref.instance.user }}/etc/cubicweb.d/{{ saemref.instance.name }}/sources
- source: salt://saemref/files/sources.j2
- template: jinja
{% if saemref.instance.base_url %}
/home/{{ saemref.instance.user }}/etc/cubicweb.d/saemref/all-in-one.conf:
file.replace:
- pattern: "^#? *base-url *=(.*)$"
- repl: base-url={{ saemref.instance.base_url }}
- append_if_not_found: true
{% endif %}
# -*- coding: utf-8 -*-
# vim: ft=yaml
saemref:
instance:
name: saemref
......
{% from "saemref/map.jinja" import saemref with context -%}
[program:saemref]
command=cubicweb-ctl start -D {{ saemref.instance.name }}
user={{ saemref.instance.user }}
redirect_stderr=true
stdout_file=/tmp/{{ saemref.instance.name }}.log
environment=CW_MODE=user
numprocs=1
autostart=true
autorestart=true
startsecs=10
{% from "saemref/map.jinja" import saemref, supervisor_confdir with context %}
{% set user = saemref.instance.user %}
{% set bindir = ['/home', user, '.local', 'bin']|join('/') %}
{% set confdir = ['/home', user, 'etc']|join('/') %}
#!/bin/bash
#
# supervisord Startup script for the Supervisor process control system
#
# Author: Mike McGrath <mmcgrath@redhat.com> (based off yumupdatesd)
# Jason Koppe <jkoppe@indeed.com> adjusted to read sysconfig,
# use supervisord tools to start/stop, conditionally wait
# for child processes to shutdown, and startup later
# Erwan Queffelec <erwan.queffelec@gmail.com>
# make script LSB-compliant
#
# chkconfig: 345 83 04
# description: Supervisor is a client/server system that allows \
# its users to monitor and control a number of processes on \
# UNIX-like operating systems.
# processname: supervisord
# config: {{ confdir }}/supervisord.conf
# config: /etc/sysconfig/supervisord
# pidfile: /tmp/supervisord.pid
#
### BEGIN INIT INFO
# Provides: supervisord
# Required-Start: $all
# Required-Stop: $all
# Short-Description: start and stop Supervisor process control system
# Description: Supervisor is a client/server system that allows
# its users to monitor and control a number of processes on
# UNIX-like operating systems.
### END INIT INFO
# Source function library
. /etc/rc.d/init.d/functions
# Source system settings
if [ -f /etc/sysconfig/supervisord ]; then
. /etc/sysconfig/supervisord
fi
# Path to the supervisorctl script, server binary,
# and short-form for messages.
prog=supervisord
pidfile=${PIDFILE-/tmp/supervisord.pid}
lockfile=${LOCKFILE-/var/lock/subsys/supervisord}
STOP_TIMEOUT=${STOP_TIMEOUT-60}
RETVAL=0
supervisorctl() {
su -l {{ user }} -c "{{ bindir }}/supervisorctl -c {{ confdir }}/supervisord.conf $*"
}
start() {
echo -n $"Starting $prog: "
su -l {{ user }} -c "{{ bindir }}/supervisord -c {{ confdir }}/supervisord.conf"
RETVAL=$?
echo
if [ $RETVAL -eq 0 ]; then
touch ${lockfile}
supervisorctl status
fi
return $RETVAL
}
stop() {
echo -n $"Stopping $prog: "
killproc -p ${pidfile} -d ${STOP_TIMEOUT} $prog
RETVAL=$?
echo
[ $RETVAL -eq 0 ] && rm -rf ${lockfile} ${pidfile}
}
reload() {
echo -n $"Reloading $prog: "
LSB=1 killproc -p $pidfile $prog -HUP
RETVAL=$?
echo
if [ $RETVAL -eq 7 ]; then
failure $"$prog reload"
else
supervisorctl status
fi
}
restart() {
stop
start
}
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status -p ${pidfile} $prog
RETVAL=$?
[ $RETVAL -eq 0 ] && supervisorctl status
;;
restart)
restart
;;
condrestart|try-restart)
if status -p ${pidfile} $prog >&/dev/null; then
stop
start
fi
;;
force-reload|reload)
reload
;;
*)
echo $"Usage: $prog {start|stop|restart|condrestart|try-restart|force-reload|reload}"
RETVAL=2
esac
exit $RETVAL
{% from "saemref/map.jinja" import supervisor_confdir with context -%}
[unix_http_server]
file=/tmp/supervisor.sock ; (the path to the socket file)
[supervisord]
logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10 ; (num of main logfile rotation backups;default 10)
loglevel=info ; (log level;default info; others: debug,warn,trace)
pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
nodaemon=false ; (start in foreground if true;default false)
minfds=1024 ; (min. avail startup file descriptors;default 1024)
minprocs=200 ; (min. avail process descriptors;default 200)
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket
[include]
files = {{ supervisor_confdir }}/*.conf
# -*- coding: utf-8 -*-
# vim: ft=jinja et
{% from "saemref/map.jinja" import saemref with context %}
include:
- epel
logilab_extranet:
pkgrepo.managed:
- humanname: Logilab extranet BRDX $releasever $basearch
- baseurl: https://extranet.logilab.fr/static/BRDX/rpms/epel-$releasever-$basearch
- gpgcheck: 0
cubicweb-saem-ref:
pkg.installed:
- require:
- pkgrepo: logilab_extranet
create-saemref-user:
user.present:
- name: {{ saemref.instance.user }}
cubicweb-create:
cmd.run:
- name: cubicweb-ctl create -Sa saem_ref {{ saemref.instance.name }}
- creates: /home/{{ saemref.instance.user }}/etc/cubicweb.d/{{ saemref.instance.name }}
- user: {{ saemref.instance.user }}
- env:
CW_MODE: user
- require:
- pkg: cubicweb-saem-ref
- user: {{ saemref.instance.user }}
cubicweb-config:
file.managed:
- name: /home/{{ saemref.instance.user }}/etc/cubicweb.d/{{ saemref.instance.name }}/sources
- source: salt://saemref/templates/sources.j2
- template: jinja
- require:
- cmd: cubicweb-create
python-pip:
pkg.installed
upgrade-pip:
pip.installed:
- name: pip
- upgrade: true
upgrade-setuptools:
pip.installed:
- name: setuptools
- upgrade: true
supervisor:
pip.installed
/etc/init.d/supervisor:
file.managed:
- source: salt://saemref/files/supervisor.init
/etc/supervisord.conf:
file.managed:
- source: salt://saemref/files/supervisord.conf
- saemref.install
- saemref.config
# -*- coding: utf-8 -*-
{% from "saemref/map.jinja" import saemref with context %}
{% if grains['os_family'] == 'Debian' %}
{% elif grains['os_family'] == 'RedHat' %}
include:
- epel
logilab_extranet:
pkgrepo.managed:
- humanname: Logilab extranet BRDX $releasever $basearch
- baseurl: https://extranet.logilab.fr/static/BRDX/rpms/epel-$releasever-$basearch
- gpgcheck: 0
{% endif %}
cubicweb-saem-ref:
pkg.installed:
- require:
- pkgrepo: logilab_extranet
create-saemref-user:
user.present:
- name: {{ saemref.instance.user }}
cubicweb-create:
cmd.run:
- name: cubicweb-ctl create --no-db-create -a saem_ref {{ saemref.instance.name }}
- creates: /home/{{ saemref.instance.user }}/etc/cubicweb.d/{{ saemref.instance.name }}
- user: {{ saemref.instance.user }}
- env:
CW_MODE: user
- require:
- pkg: cubicweb-saem-ref
- user: {{ saemref.instance.user }}
# -*- coding: utf-8 -*-
# vim: ft=jinja
{% import_yaml 'saemref/defaults.yaml' as default_settings %}
{% set os_family_map = salt['grains.filter_by']({
......@@ -8,7 +5,7 @@
'RedHat': {},
},
grain="os_family",
merge=salt['pillar.get']('template:lookup'))
merge=salt['pillar.get']('saemref:lookup'))
%}
{% do default_settings.saemref.update(os_family_map) %}
......@@ -19,3 +16,9 @@
merge=True,
)
%}
{% if grains['os_family'] == 'Debian' %}
{% set supervisor_confdir = '/etc/supervisor/conf.d' %}
{% elif grains['os_family'] == 'RedHat' %}
{% set supervisor_confdir = ['/home', saemref.instance.user, 'etc', 'supervisor.d']|join('/') %}
{% endif %}
{% from "saemref/map.jinja" import saemref, supervisor_confdir with context %}
include:
- saemref.install
{% if grains['os_family'] == 'Debian' %}
supervisor:
pkg:
- installed
supervisor_confdir:
file.directory:
- name: {{ supervisor_confdir }}
- require:
- pkg: supervisor
{% elif grains['os_family'] == 'RedHat' %}
python-pip:
pkg:
- installed
supervisor:
pip.installed:
- user: {{ saemref.instance.user }}
- install_options:
- "--user"
- require:
- user: {{ saemref.instance.user }}
- pkg: python-pip
{% for fname in ('supervisorctl', 'supervisord') %}
/home/{{ saemref.instance.user }}/bin/{{ fname }}:
file.symlink:
- target: /home/{{ saemref.instance.user }}/.local/bin/{{ fname }}
- user: {{ saemref.instance.user }}
- makedirs: true
- require:
- pip: supervisor
{% endfor %}
supervisor_confdir:
file.directory:
- name: {{ supervisor_confdir }}
- user: {{ saemref.instance.user }}
- require:
- user: {{ saemref.instance.user }}
/home/{{ saemref.instance.user }}/etc/supervisord.conf:
file.managed:
- source: salt://saemref/files/supervisord.conf
- template: jinja
- user: {{ saemref.instance.user }}
/etc/init.d/supervisor:
file.managed:
- source: salt://saemref/files/supervisor.init
- template: jinja
- mode: 755
{% endif %}
{{ supervisor_confdir }}/saemref.conf:
file.managed:
- source: salt://saemref/files/saemref-supervisor.conf
- template: jinja
{% if grains['os_family'] != 'Debian' %}
- user: {{ saemref.instance.user }}
{% endif %}
- require:
- file: supervisor_confdir
supervisor-service-running:
service.running:
- name: supervisor
- enable: true
#!/bin/sh
### BEGIN INIT INFO
# Provides: saemref
# Required-Start: $all
# Required-Stop:
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
### END INIT INFO
# chkconfig header
# chkconfig: 345 97 04
# description: CubicWeb saem_ref
#
# processname: /usr/bin/cubicweb-ctl
DEBIAN_VERSION=/etc/debian_version
# Source function library.
if [ -f $DEBIAN_VERSION ]; then
break
else
. /etc/rc.d/init.d/functions
fi
# Default values (can be overridden below)
CUBICWEBCTL=/usr/bin/cubicweb-ctl
if [ -f /etc/default/saemref ]; then
. /etc/default/saemref
fi
SERVICE=saemref
PROCESS=saemref