Commit f6c096b0 authored by rgaudin's avatar rgaudin
Browse files

bundle ansiblecube inside pibox-installer

ansiblecube eventually grew into a direct dependency of pibox-installer.
there is no reason to keep it as a separate project.

Design around a three phases installation:

1. image creation: --tags master,rename,setup
	prepares the system and its dependencies
	create an exfat partition to hold contents
	install softwares for all projects but no content.
	expects content data to be placed into /data (exfat partition)

2. image configuration: --tags resize,rename,reconfigure
	resize the data partition to use the image's new size
	change domain names according to config file
	configure each software according to the image config file

3. content finalization: --tags move-content,seal
	move content to their final location
	run software configurations based on the content
	any image finalization
parent 9a8a170f
# AnsibleCube
This is a **very distant** fork of [ideascube/ansiblecube](https://github.com/ideascube/ansiblecube/) dedicated to [pibox-installer](https://framagit.org/ideascube/pibox-installer)'s use case.
## Goal
It aims at setting-up a complete RaspberryPi box to use as a Content-HotSpot.
This is achieved by using a three steps scenario:
1. A base image is created, running this playbook with default (all-content) values using `--tags master,rename,setup`
2. Content is configured by re-running the playbook with a configuration file and `--tags resize,rename,reconfigure`.
3. After the large content files has been put inside the data partition, the playbook is ran again `--tags move-content,seal`.
## Tags
* `master`: installs all non-content-specific softwares (system, `ideascube`, `kiwix-serve`).
* `resize`: resize the `/` and `/data` partitions (image should have been already resized qemu-wise).
* `rename`: reconfigure all software according to `project_name`.
* `setup`: install and prepare all softwares without enabling them.
* `reconfigure`: sets all configuration and software according to configuration.
* `move-content`: move expected content from `/data/xxx` to proper locations, according to configuration.
* `seal`: enables requested software and mark system as ready.
**Notes**:
* `resize` requires a re-run of `setup` and `reconfigure` as most content and softwares lives in `/data`.
## Features
* WiFi Hot Spot (`hostapd`)
* DNS Masquerade (`dnsmasq`)
* Captive Portal
* Web & Application servers (`nginx`, `uswgi`)
* Content & Box Manager (`ideascube`)
* ZIM files (all of [kiwix](https://kiwix.org)'s library) with indexes.
* Khan-Accademy Videos (`ka-lite`)
* Aflatoun
* Wikifundi
* EduPi
## Usage
This playbook is not meant to be run standalone on any system.
Please check [make-base-image](https://framagit.org/ideascube/pibox-installer/tree/master/make-vexpress-boot) and more generally [pibox-installer](https://framagit.org/ideascube/pibox-installer) to see it in action.
## Tests
The repo is plugged in [Travis CI](https://travis-ci.org/ideascube/ansiblecube).
### Testing locally
__Setting-up test envirronment__
``` bash
sudo apt-get install libssl-dev
virtualenv ~/.virtualenv/pytest-ansible
. ~/.virtualenv/pytest-ansible/bin/activate
cd ~/dev/ansiblecube
pip install -r tests/requirements-dev.txt
```
__Running them__
``` bash
py.test
```
__Pre-push git hook__
``` sh
#!/bin/sh
${HOME}/.virtualenv/pytest-ansible/bin/py.test
```
[defaults]
inventory=hosts
gather_timeout = 240
---
- name: clean apt cache
command: apt clean -y
---
- name: disable service {{ service }}
service:
name: "{{ service }}"
enabled: no
state: stopped
---
- name: enable service {{ service }}
service:
name: "{{ service }}"
enabled: yes
state: restarted
---
- name: enable nginx virtualhost {{ name }}
file:
src: /etc/nginx/sites-available/{{ name }}
dest: /etc/nginx/sites-enabled/{{ name }}
state: link
force: yes
notify: restart nginx
---
# Network
hotspot_ip: 192.168.2.1
hotspot_interface: wlan0
lan_interface: eth0
lan_interface2: eth1
client_interface: wlan1
max_wifi_client: 15
cert_domain: ideascube.lan
hostname: ideasbox
tld: "lan"
fqdn: "{{ hostname }}.{{ tld }}"
hotspot_name: ideasbox
project_name: ideasbox
ideascube_id: pibox
# System
disk: /dev/mmcblk0
root_partition: "{{ disk }}p2"
data_partition: "{{ disk }}p3"
data_path: /data
disk_size: 4
root_partition_size: 3
username: ideascube
password: $6$YcJS1HjRAv/$1bUosoPalzHEvfpx71UhnG6vCeckTGbA5U1i6UpA8X21Jrv7fYRPyNrq/YWuiBp3xQWd5kRRhrKKPHcbTkkfQ/
mirror: https://download.kiwix.org
timezone: "Europe/Paris"
language: "en"
language_name: "English"
# versions
qemu_kernel_version: "4.10.0"
kernel_version: "4.14.34"
kiwix_version: "0.6.0"
kalite_version: "0.17.4"
ideascube_version: "0.37.2"
edupi_version: "1.4.5"
python3_version: "3.5"
# Apps path
zim_path: /var/ideascube/kiwix
venvs_root: /home/{{ username }}/venvs
ideascube_handlers: ['kiwix', 'nginx', 'mediacenter']
aflatoun_root: "{{ data_path }}/aflatoun"
aflatoun_env: "{{ venvs_root }}/aflatoun"
edupi_root: "/home/{{ username }}/edupi"
edupi_data_root: "{{ data_path }}/edupi"
edupi_env: "{{ venvs_root }}/edupi"
kalite_root: "{{ data_path }}/kalite"
kalite_env: "{{ venvs_root }}/kalite"
wikifundi_root: "{{ data_path }}/wikifundi"
wikifundi_parsoid: "{{ wikifundi_root }}/parsoid"
# Domain names
kiwix_server_name: kiwix.{{ hostname }}
kalite_server_name: khanacademy.{{ hostname }}
aflatoun_server_name: aflatoun.{{ hostname }}
edupi_server_name: edupi.{{ hostname }}
wikifundi_server_name: wikifundi.{{ hostname }}
sites_server_name: sites.{{ hostname }}
kiwix_fqdn: kiwix.{{ fqdn }}
kalite_fqdn: khanacademy.{{ fqdn }}
aflatoun_fqdn: aflatoun.{{ fqdn }}
edupi_fqdn: edupi.{{ fqdn }}
wikifundi_fqdn: wikifundi.{{ fqdn }}
sites_fqdn: sites.{{ fqdn }}
# content & features
captive_portal: true
has_custom_branding: false
custom_branding_path: /tmp
kalite_languages: []
wikifundi_languages: []
aflatoun_languages: []
packages: []
edupi: false
catalogs: []
# temporary path for contents in data partition
aflatoun_content_path: "{{ data_path }}/aflatoun_content"
aflatoun_langpack_prefix: "{{ data_path }}/aflatoun_pack_"
kalite_langpack_prefix: "{{ data_path }}/kalite_pack_"
kalite_videos_prefix: "{{ data_path }}/kalite_videos_"
packages_cache: "{{ data_path }}/packages_cache"
wikifundi_langpack_prefix: "{{ data_path }}/wikifundi_langpack_"
wikifundi_parsoid_path: "{{ data_path }}/wikifundi_parsoid"
---
- name: restart systemd-logind
service:
name: systemd-logind
state: restarted
- name: update timezone
command: dpkg-reconfigure --frontend noninteractive tzdata
- name: update grub
command: update-grub
- name: reload systemd
systemd:
daemon-reload: yes
- name: restart hostapd
service:
name: hostapd
state: restarted
- name: restart nginx
service:
name: nginx
state: restarted
- name: restart uwsgi
service:
name: uwsgi
state: restarted
- name: restart kiwix
service:
name: kiwix-server
state: restarted
- name: restart network manager
service:
name: network-manager
state: restarted
localhost ansible_connection=local
---
- hosts: localhost
handlers:
- import_tasks: handlers.yml
pre_tasks:
- name: Set the LED steady before upgrading !!!
shell: echo default-on >/sys/class/leds/a20-olinuxino-lime2:green:usr/trigger
when: ansible_architecture == 'armv7l' and ansible_lsb.id == 'Debian'
ignore_errors: yes
tags: ['always']
roles:
- role: variables
# master, setup (timezone, apt-update)
- role: system
# master, resize
- role: resize
- role: nginx
tags: master
- role: uwsgi
tags: master
# master, rename, setup, reconfigure
- role: ideascube
# master, rename
- role: kiwix
# master, rename
- role: captive_portal
when: ansible_local.config.captive_portal|bool
# content setup (install, dependencies)
- role: wikifundi_setup
when: ansible_local.config.wikifundi_languages | length
- role: aflatoun_setup
when: ansible_local.config.aflatoun_languages | length
- role: edupi
when: ansible_local.config.edupi|bool
- role: kalite_setup
when: ansible_local.config.kalite_languages | length
# move content
- role: packages
when: ansible_local.config.packages | length
tags: move-content
- role: kalite_content
when: ansible_local.config.kalite_languages | length
- role: aflatoun_content
when: ansible_local.config.aflatoun_languages | length
- role: wikifundi_content
when: ansible_local.config.wikifundi_languages | length
- role: services
tags: seal
post_tasks:
- name: Heartbeat mode on KoomBook LED, update is over !!!
shell: echo heartbeat >/sys/class/leds/a20-olinuxino-lime2:green:usr/trigger
when: ansible_architecture == 'armv7l' and ansible_lsb.id == 'Debian'
ignore_errors: yes
tags: ['always']
---
- name: mark role {{ role }} complete
ini_file:
dest: /etc/ansible/facts.d/progress.fact
section: main
option: "{{ role }}"
value: "y"
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu
''' compute and print new partition boundaries for / dans /data
Used by providing an image's fdisk output and requested size for partitions
fdisk -l <disk> | partition_boundaries.py <root_size> <disk_size>
Note: since root partition will be resized online (no unmount),
we reuse its start (found in fdisk) and only expand it (no shrink)
prints out 4 numbers to be fed into fdisk for recreation
- root partition start
- root partition end
- data partition start
- data partition end
output format: <root_start> <root_end> <data_start> <data_end>
'''
from __future__ import (unicode_literals, absolute_import,
division, print_function)
import re
import sys
try:
text_type = unicode # Python 2
except NameError:
text_type = str # Python 3
ONE_GB = int(1e9)
def main(root_size=7, disk_size=8):
# sanitize input
if disk_size == '-':
disk_size = None
elif not isinstance(disk_size, int):
disk_size = int(disk_size)
if not isinstance(root_size, int):
root_size = int(root_size)
try:
data = get_partitions_boundaries(
lines=sys.stdin.read().splitlines(),
root_size=root_size, disk_size=disk_size)
print(" ".join([text_type(x) for x in data]))
except Exception as exp:
print("Error: {}".format(repr(exp)))
sys.exit(1)
def get_partitions_boundaries(lines, root_size, disk_size=None):
sector_size = 512
round_bound = 128
end_margin = 4194304 # 4MiB
def roundup(sector):
return rounddown(sector) + round_bound \
if sector % round_bound != 0 else sector
def rounddown(sector):
return sector - (sector % round_bound) \
if sector % round_bound != 0 else sector
# parse all lines
number_of_sector_match = []
second_partition_match = []
target_reg = r'[0-9a-zA-Z\.\-\_]+\.img' \
if '.img' in "\n".join(lines) else r'\/dev\/[0-9a-z]+'
for line in lines:
number_of_sector_match += re.findall(
r"^Disk {}:.*, (\d+) sectors$".format(target_reg), line)
second_partition_match += re.findall(
r"^{}\d +(\d+) +(\d+) +\d+ +\S+ +\d+ +Linux$".format(target_reg),
line)
# ensure we retrieved nb of sectors correctly
if len(number_of_sector_match) != 1:
raise ValueError("cannot find the number of sector of disk")
number_of_sector = int(number_of_sector_match[0])
# ensure we retrieved the start of the root partition correctly
if len(second_partition_match) != 1:
raise ValueError(
"cannot find start and/or end of root partition of disk")
second_partition_start = int(second_partition_match[0][0])
second_partition_end = int(second_partition_match[0][1])
# whether disk is already full
is_full = second_partition_end + 1 == number_of_sector
if is_full:
pass # whether root part was already expanded
size_up_to_root_b = root_size * ONE_GB
nb_clusters_endofroot = size_up_to_root_b // sector_size
# align partitions (otherwise exfat-fuse gets often corrupt)
root_start = second_partition_start
root_end = roundup(nb_clusters_endofroot)
data_start = root_end + 1
# data_end = number_of_sector (using full avail space)
# end second partition on a predicatble cluster
# full_size - root_size (root_end) - margin
data_bytes = disk_size * ONE_GB - root_size * ONE_GB - end_margin
data_clusters = data_bytes // sector_size
data_end = data_start + data_clusters
return root_start, root_end, data_start, data_end
if __name__ == '__main__':
main(*sys.argv[1:])
---
- name: Make sure aflatoun content directory does not exist
file:
dest: "{{ aflatoun_root }}/content"
state: absent
tags: move-content
# downloaded content was placed in a different location on data part
- name: Move downloaded content to proper location
command: mv {{ aflatoun_content_path }} {{ aflatoun_root }}/content
tags: move-content
- name: re-launch setup for it annotates the content items
become: yes
become_user: "{{ username }}"
environment:
KALITE_PYTHON: "{{ aflatoun_env }}/bin/python"
KALITE_HOME: "{{ aflatoun_root }}"
command: "{{ aflatoun_env }}/bin/kalite manage setup --noinput --no-assessment-items"
tags: move-content
- name: Install Aflatoun language pack
become: yes
become_user: "{{ username }}"
environment:
KALITE_PYTHON: "{{ aflatoun_env }}/bin/python"
KALITE_HOME: "{{ aflatoun_root }}"
command: "{{ aflatoun_env }}/bin/kalite manage retrievecontentpack local {{ item }} {{ aflatoun_langpack_prefix }}{{ item }}.zip"
with_items: "{{ ansible_local.config.aflatoun_languages }}"
tags: move-content
---
- import_tasks: clean_apt.yml
tags: setup
- name: Create working directory
file:
path: "{{ aflatoun_root }}"
state: directory
tags: ['setup', 'reconfigure']
- name: Download Aflatoun
get_url:
url: "{{ mirror }}/other/aflatoun/ka-lite-static-0.16.7b3.tar.gz"
dest: /tmp/ka-lite-static-0.16.7b3.tar.gz
tags: setup
- name: Create aflatoun environment
become: yes
become_user: "{{ username }}"
command: virtualenv -p python2.7 {{ aflatoun_env }}
tags: setup
- name: downgrade setuptools for KA-lite 0.16 (aflatoun)
pip:
name: setuptools
version: 28.8.0
virtualenv: "{{ aflatoun_env }}"
tags: setup
- name: Install Aflatoun
pip:
name: /tmp/ka-lite-static-0.16.7b3.tar.gz
virtualenv: "{{ aflatoun_env }}"
tags: setup
- name: Initialize aflatoun and Create an admin user
become: yes
become_user: "{{ username }}"
shell: KALITE_PYTHON={{ aflatoun_env }}/bin/python KALITE_HOME={{ aflatoun_root }} {{ aflatoun_env }}/bin/kalite manage setup --username={{ admin_account }} --password={{ admin_password }} --noinput --no-assessment-items
when: admin_account is defined
tags: ['setup', 'reconfigure']
- name: Initialize Aflatoun without creating an admin user
become: yes
become_user: "{{ username }}"
shell: KALITE_PYTHON={{ aflatoun_env }}/bin/python KALITE_HOME={{ aflatoun_root }} {{ aflatoun_env }}/bin/kalite manage setup --noinput --no-assessment-items
when: admin_account is not defined
tags: ['setup', 'reconfigure']
- name: Copy nginx vhost
template:
src: aflatoun.vhost.j2
dest: /etc/nginx/sites-available/aflatoun
notify: restart nginx
tags: ['setup', 'rename']
- name: Setup startup file
template:
src: aflatoun.service.j2
dest: /etc/systemd/system/aflatoun.service
notify: reload systemd
tags: setup
[Unit]
Description=Aflatoun server
[Service]
User=ideascube
Group=ideascube
Environment="KALITE_HOME={{ aflatoun_root }}"
Environment="KALITE_PYTHON={{ aflatoun_env }}/bin/python"
ExecStart={{ aflatoun_env }}/bin/kalite start --port=8009 --foreground
ExecStop={{ aflatoun_env }}/bin/kalite stop
Restart=always
TimeoutStopSec=60
PIDFile=/var/run/aflatoun.pid
KillSignal=SIGINT
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
# NGINX configuration for KA Lite
#
# Upstream Aflatoun server uses port 7007
# Nginx proxy for Aflatoun uses port 8008
#
# If you want the website to be accessible at a different port, change
# PROXY_PORT = nnnn setting in /var/ka-lite/.aflatoun/settings.py
# and change the below accordingly.
server {
listen 80;
server_name {{ aflatoun_fqdn }} {{ aflatoun_server_name }};
# Default value, overwritten in nginx.d
set $aflatoun_home {{ aflatoun_root }}/;
# include /etc/ka-lite/nginx.d/*.conf;
location /static {
alias $aflatoun_home/httpsrv/static/;
}
location /media {
alias $aflatoun_home/httpsrv/media/;
}
location /content {
alias $aflatoun_home/content/;
}
location /content/khan {
alias $aflatoun_home/content/assessment/khan/;
}
location /favicon.ico {
empty_gif;
}
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://localhost:8009;
}
location /ideascube-static/ { alias /var/ideascube/static/; expires 1y; }
location /502.html { internal; root /var/www; }
error_page 502 /502.html;
}
[uwsgi]
autoload = false
uid = ideascube
gid = ideascube
socket = :3031
wsgi-file = /var/www/captiveportal/cap.py