Commit d78d2efd authored by thiolliere's avatar thiolliere
Browse files

refactor restructure

the main restructuration is the creation of logger
a class that cli and gui would implement differently
and so both will be able to use run_installation.
parent 4b68ccc2
from . import systemd
import json
# machine must provide write_file and exec_cmd functions
def run(machine, name, timezone, wifi_pwd, kalite, zim_install):
machine.write_file("/etc/systemd/system.conf", systemd.system_conf)
machine.write_file("/etc/systemd/user.conf", systemd.user_conf)
machine.exec_cmd("sudo apt-get update")
machine.exec_cmd("sudo apt-get install -y python-pip git python-dev libffi-dev libssl-dev gnutls-bin")
machine.exec_cmd("sudo pip install ansible==2.1.2 markupsafe")
machine.exec_cmd("sudo pip install cryptography --upgrade")
ansiblecube_url = "https://github.com/thiolliere/ansiblecube.git"
ansiblecube_path = "/var/lib/ansible/local"
machine.exec_cmd("sudo mkdir --mode 0755 -p %s" % ansiblecube_path)
machine.exec_cmd("sudo git clone {url} {path}".format(url=ansiblecube_url, path=ansiblecube_path))
machine.exec_cmd("sudo mkdir --mode 0755 -p /etc/ansible")
machine.exec_cmd("sudo cp %s/hosts /etc/ansible/hosts" % ansiblecube_path)
hostname = name.replace("_", "-")
machine.exec_cmd("sudo hostname %s" % hostname)
package_management = [{"name": x, "status": "present"} for x in zim_install]
device_list = {hostname: {
"kalite": {
"activated": kalite != None,
"version": "0.16.9",
"language": kalite or [],
},
"idc_import": {
"activated": False,
"content_name": [],
},
"package_management": package_management,
"portal": {
"activated": True,
}
}}
facts_path = "/etc/ansible/facts.d"
machine.exec_cmd("sudo mkdir --mode 0755 -p %s" % facts_path)
machine.write_file(facts_path, json.dumps(device_list, indent=4))
extra_vars = "ideascube_project_name=%s" % name
extra_vars += " timezone=%s" % timezone
if wifi_pwd:
extra_vars += " wpa_pass=%s" % wifi_pwd
extra_vars += " git_branch=oneUpdateFile"
extra_vars += " own_config_file=True"
extra_vars += " managed_by_bsf=False"
ansible_pull_cmd = "sudo /usr/local/bin/ansible-pull"
ansible_pull_cmd += " --checkout oneUpdateFile"
ansible_pull_cmd += " --directory /var/lib/ansible/local"
ansible_pull_cmd += " --inventory hosts"
ansible_pull_cmd += " --url https://github.com/thiolliere/ansiblecube.git"
ansible_pull_cmd += " --tags master,custom"
ansible_pull_cmd += " --extra-vars \"%s\"" % extra_vars
ansible_pull_cmd += " main.yml"
machine.exec_cmd(ansible_pull_cmd)
import os
import urllib.request
import yaml
from . import pretty_print
catalog_url_path = "http://catalog.ideascube.org/"
......
......@@ -3,57 +3,62 @@ import urllib.request
import posixpath
import shutil
from zipfile import ZipFile
from . import pretty_print
def _download_and_extract(url, content):
pretty_print.step("download " + url)
(zip_filename, _) = urllib.request.urlretrieve(url, reporthook=pretty_print.ReportHook().reporthook)
with ZipFile(zip_filename) as zipFile:
for (zip_path, path) in content:
pretty_print.step("extract " + zip_path)
extraction = zipFile.extract(zip_path)
shutil.move(extraction, path)
os.remove(zip_filename)
class vexpress_boot:
boot_dir = "pibox-installer-vexpress-boot"
kernel_name = "zImage"
kernel_path = os.path.join(boot_dir, kernel_name)
kernel_zip_path = posixpath.join(boot_dir, kernel_name)
dtb_name = "vexpress-v2p-ca9.dtb"
dtb_path = os.path.join(boot_dir, dtb_name)
dtb_zip_path = posixpath.join(boot_dir, dtb_name)
url = "http://download.kiwix.org/dev/" + boot_dir + ".zip"
def get():
pretty_print.step("get vexpress boot")
os.makedirs(vexpress_boot.boot_dir, exist_ok=True)
if os.path.isfile(vexpress_boot.kernel_path) and os.path.isfile(vexpress_boot.dtb_path):
pretty_print.std("nothing to do")
return
content = [(vexpress_boot.kernel_zip_path, vexpress_boot.kernel_path),
(vexpress_boot.dtb_zip_path, vexpress_boot.dtb_path)]
print(vexpress_boot.url)
_download_and_extract(vexpress_boot.url, content)
class raspbian:
version = "2017-03-02"
image_path = version + "-raspbian-jessie-lite.img"
image_zip_path = image_path
zip_filename = version + "-raspbian-jessie-lite.zip"
url_dir_version = "2017-03-03"
url = "https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-{}/{}".format(url_dir_version, zip_filename)
def get():
pretty_print.step("get raspbian-lite image")
if os.path.isfile(raspbian.image_path):
pretty_print.std("nothing to do")
return
_download_and_extract(raspbian.url, [(raspbian.image_zip_path, raspbian.image_path)])
class Downloader:
_logger = None
def __init__(self, logger):
self._logger = logger
def _download_and_extract(self, url, content):
self._logger.step("download " + url)
(zip_filename, _) = urllib.request.urlretrieve(url, reporthook=self._logger.ReportHook().reporthook)
with ZipFile(zip_filename) as zipFile:
for (zip_path, path) in content:
self._logger.step("extract " + zip_path)
extraction = zipFile.extract(zip_path)
shutil.move(extraction, path)
os.remove(zip_filename)
def download_vexpress_boot(self):
boot_dir = "pibox-installer-vexpress-boot"
kernel_name = "zImage"
kernel_path = os.path.join(boot_dir, kernel_name)
kernel_zip_path = posixpath.join(boot_dir, kernel_name)
dtb_name = "vexpress-v2p-ca9.dtb"
dtb_path = os.path.join(boot_dir, dtb_name)
dtb_zip_path = posixpath.join(boot_dir, dtb_name)
url = "http://download.kiwix.org/dev/" + boot_dir + ".zip"
self._logger.step("get vexpress boot")
os.makedirs(boot_dir, exist_ok=True)
if os.path.isfile(kernel_path) and os.path.isfile(dtb_path):
self._logger.std("nothing to do")
else:
content = [(kernel_zip_path, kernel_path),
(dtb_zip_path, dtb_path)]
self._download_and_extract(url, content)
return kernel_path, dtb_path
def download_raspbian(self):
version = "2017-03-02"
image_path = version + "-raspbian-jessie-lite.img"
image_zip_path = image_path
zip_filename = version + "-raspbian-jessie-lite.zip"
url_dir_version = "2017-03-03"
url = "https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-{}/{}".format(url_dir_version, zip_filename)
self._logger.step("get raspbian-lite image")
if os.path.isfile(image_path):
self._logger.std("nothing to do")
else:
self._download_and_extract(url, [(image_zip_path, image_path)])
return image_path
import sys
def step(step):
print("\033[00;34m--> " + step + "\033[00m")
def err(err):
print("\033[00;31m" + err + "\033[00m")
def std(std):
print(std)
def newline():
print()
class ReportHook():
_current_size = 0
width = 60
_last_line = None
def reporthook (self, chunk, chunk_size, total_size):
if chunk != 0:
self._current_size += chunk_size
avail_dots = self.width-2
if total_size == -1:
line = "unknown size"
elif self._current_size >= total_size:
line = "[" + "."*avail_dots + "] 100%\n"
else:
ratio = min(float(self._current_size) / total_size, 1.)
shaded_dots = min(int( ratio * avail_dots), avail_dots)
percent = min(int(ratio*100), 100)
line = "[" + "."*shaded_dots + " "*(avail_dots-shaded_dots) + "] " + str(percent) + "%\r"
if line != self._last_line:
self._last_line = line
sys.stdout.write(line)
......@@ -4,13 +4,9 @@ import collections
import sys
import socket
import paramiko
import select
import re
import json
import random
import threading
from select import select
from . import pretty_print
timeout = 60*3
......@@ -40,47 +36,24 @@ def get_free_port():
return port
def wait_signal(reader_fd, writer_fd, signal, timeout):
timeout_event = threading.Event()
def raise_timeout():
timeout_event.set()
os.write(writer_fd, b"piboxinstaller timeout")
ring_buf = collections.deque(maxlen=len(signal))
while True:
timer = threading.Timer(timeout, raise_timeout)
timer.start()
buf = os.read(reader_fd, 1024)
try:
sys.stdout.write(buf.decode("utf-8"))
except:
pass
if timeout_event.is_set():
raise QemuWaitSignalTimeoutError("signal %s" % signal)
timer.cancel()
ring_buf.extend(buf)
if list(ring_buf) == list(signal):
return
class Emulator:
_image = None
_kernel = None
_dtb = None
_logger = None
# login=pi
# password=raspberry
# prompt end by ":~$ "
# sudo doesn't require password
def __init__(self, kernel, dtb, image):
def __init__(self, kernel, dtb, image, logger):
self._kernel = kernel
self._dtb = dtb
self._image = image
self._logger = logger
def run(self):
return _RunningInstance(self)
return _RunningInstance(self, self._logger)
def get_image_size(self):
pipe_reader, pipe_writer = os.pipe()
......@@ -98,7 +71,7 @@ class Emulator:
subprocess.check_call([qemu_img_exe, "resize", "-f", "raw", self._image, "{}G".format(size)])
def copy_image(self, device_name):
pretty_print.step("copy image to sd card")
self._logger.step("copy image to sd card")
image = os.open(self._image, os.O_RDONLY)
device = os.open(device_name, os.O_WRONLY)
......@@ -113,7 +86,7 @@ class Emulator:
new_percentage = (100 * current_size) / total_size
if new_percentage != current_percentage:
current_percentage = new_percentage
pretty_print.std(str(current_percentage) + "%")
self._logger.std(str(current_percentage) + "%\r")
buf = os.read(image, 4096)
if buf == b"":
......@@ -121,19 +94,19 @@ class Emulator:
os.write(device, buf)
os.close(image)
pretty_print.step("sync")
self._logger.step("sync")
os.fsync(device)
os.close(device)
pretty_print.step("done")
class _RunningInstance:
_emulation = None
_qemu = None
_client = None
_logger = None
def __init__(self, emulation):
def __init__(self, emulation, logger):
self._emulation = emulation
self._logger = logger
def __enter__(self):
try:
......@@ -151,13 +124,41 @@ class _RunningInstance:
self._qemu.kill()
raise
def _wait_signal(self, reader_fd, writer_fd, signal, timeout):
timeout_event = threading.Event()
def raise_timeout():
timeout_event.set()
os.write(writer_fd, b"piboxinstaller timeout")
ring_buf = collections.deque(maxlen=len(signal))
while True:
timer = threading.Timer(timeout, raise_timeout)
timer.start()
buf = os.read(reader_fd, 1024)
try:
decoded_buf = buf.decode("utf-8")
except:
pass
else:
self._logger.raw_std(decoded_buf)
if timeout_event.is_set():
raise QemuWaitSignalTimeoutError("signal %s" % signal)
timer.cancel()
ring_buf.extend(buf)
if list(ring_buf) == list(signal):
return
def _boot(self):
ssh_port = get_free_port()
stdout_reader, stdout_writer = os.pipe()
stdin_reader, stdin_writer = os.pipe()
pretty_print.step("launch qemu with ssh on port {}".format(ssh_port))
self._logger.step("launch qemu with ssh on port {}".format(ssh_port))
self._qemu = subprocess.Popen([
qemu_system_arm_exe,
......@@ -173,18 +174,18 @@ class _RunningInstance:
"-no-reboot",
], stdin=stdin_reader, stdout=stdout_writer, stderr=subprocess.STDOUT)
wait_signal(stdout_reader, stdout_writer, b"login: ", timeout)
self._wait_signal(stdout_reader, stdout_writer, b"login: ", timeout)
os.write(stdin_writer, b"pi\n")
wait_signal(stdout_reader, stdout_writer, b"Password: ", timeout)
self._wait_signal(stdout_reader, stdout_writer, b"Password: ", timeout)
os.write(stdin_writer, b"raspberry\n")
wait_signal(stdout_reader, stdout_writer, b":~$ ", timeout)
self._wait_signal(stdout_reader, stdout_writer, b":~$ ", timeout)
# TODO: This is a ugly hack. But writing all at once doesn't work
os.write(stdin_writer, b"sudo systemctl")
wait_signal(stdout_reader, stdout_writer, b"sudo systemctl", timeout)
self._wait_signal(stdout_reader, stdout_writer, b"sudo systemctl", timeout)
os.write(stdin_writer, b" start ssh;")
wait_signal(stdout_reader, stdout_writer, b" start ssh;", timeout)
self._wait_signal(stdout_reader, stdout_writer, b" start ssh;", timeout)
os.write(stdin_writer, b" exit\n")
wait_signal(stdout_reader, stdout_writer, b"login: ", timeout)
self._wait_signal(stdout_reader, stdout_writer, b"login: ", timeout)
self._client = paramiko.SSHClient()
self._client.set_missing_host_key_policy(paramiko.MissingHostKeyPolicy())
......@@ -199,7 +200,7 @@ class _RunningInstance:
if return_stdout:
stdout_buffer = ""
pretty_print.std(command)
self._logger.std(command)
_, stdout, stderr = self._client.exec_command(command)
while True:
line = stdout.readline()
......@@ -207,16 +208,16 @@ class _RunningInstance:
break
if return_stdout:
stdout_buffer += line
pretty_print.std(line.replace("\n", ""))
self._logger.std(line.replace("\n", ""))
for line in stderr.readlines():
pretty_print.err("STDERR: " + line.replace("\n", ""))
self._logger.err("STDERR: " + line.replace("\n", ""))
if return_stdout:
return stdout_buffer
def resize_fs(self):
pretty_print.step("resize partition")
self._logger.step("resize partition")
stdout = self.exec_cmd("sudo LANG=C fdisk -l /dev/mmcblk0", return_stdout=True)
lines = stdout.splitlines()
......@@ -237,7 +238,7 @@ class _RunningInstance:
second_partition_end = int(second_partition_match[0][1])
if second_partition_end + 1 == number_of_sector:
pretty_print.std("nothing to do")
self._logger.std("nothing to do")
else:
# d delete partition
......@@ -261,7 +262,7 @@ END_OF_CMD""" % second_partition_start
self.exec_cmd(fdiskCmd)
self._reboot()
pretty_print.step("resize filesystem")
self._logger.step("resize filesystem")
self.exec_cmd("sudo resize2fs /dev/mmcblk0p2")
def write_file(self, path, content):
......@@ -272,7 +273,7 @@ END_OF_CMD""" % second_partition_start
self.exec_cmd("sudo mv {} '{}'".format(tmp, path))
def _reboot(self):
pretty_print.step("reboot qemu")
self._logger.std("reboot qemu")
self._shutdown()
self._boot()
#!/usr/bin/python3
import os
import argparse
import sys
import yaml
from backend.downloads import vexpress_boot, raspbian
from backend import catalog
from backend import pretty_print
from backend import qemu
from run_installation import run_installation
from logger import Logger
from set_path import set_path
if getattr(sys, "frozen", False):
if os.name == "nt":
os.environ["PATH"] += ";" + sys._MEIPASS
else:
os.environ["PATH"] += ":" + sys._MEIPASS
set_path()
parser = argparse.ArgumentParser(description="ideascube/kiwix installer for raspberrypi.")
parser.add_argument("-n", "--name", help="name of the box (mybox)", default="mybox")
......@@ -27,34 +21,18 @@ parser.add_argument("-s", "--sd", help="sd card device to put the image onto")
args = parser.parse_args()
os.makedirs("build", exist_ok=True)
os.chdir("build")
if args.catalog:
for catalog in catalog.get_catalogs():
print(yaml.dump(catalog, default_flow_style=False, default_style=''))
exit(0)
vexpress_boot.get()
raspbian.get()
emulator = qemu.Emulator(vexpress_boot.kernel_path, vexpress_boot.dtb_path, raspbian.image_path)
if args.resize:
emulator.resize_image(args.resize)
with emulator.run() as emulation:
emulation.resize_fs()
ansiblecube.run(
machine=emulation,
name=name,
timezone=timezone,
wifi_pwd=wifi_pwd,
kalite=kalite,
zim_install=zim_install)
pretty_print.step("image done")
if args.sd:
emulator.copy_image(args.sd)
run_installation(
name=args.name,
timezone=args.timezone,
wifi_pwd=args.wifi_pwd,
kalite=args.kalite,
zim_install=args.zim_install,
size=args.resize,
logger=Logger,
directory="build",
sd_card=args.sd)
......@@ -2,21 +2,17 @@ import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gdk
from backend import catalog
from run_installation import run_installation
from logger import Logger
from set_path import set_path
import pytz
import re
import os
import sys
from backend.downloads import vexpress_boot, raspbian
from backend import catalog
from backend import pretty_print
from backend import qemu
from backend import ansiblecube
set_path()
if getattr(sys, "frozen", False):
if os.name == "nt":
os.environ["PATH"] += ";" + sys._MEIPASS
else:
os.environ["PATH"] += ":" + sys._MEIPASS
DATA_DIR = sys._MEIPASS
else:
DATA_DIR = ""
......@@ -130,7 +126,15 @@ class ConfigurationWindow:
if all_valid:
self.component.window.destroy()
Gtk.main_quit()
run_installation(name=project_name, timezone=timezone, wifi_pwd=wifi_pwd, kalite=None, zim_install=zim_install, size=size)
run_installation(
name=project_name,
timezone=timezone,
wifi_pwd=wifi_pwd,
kalite=None,
zim_install=zim_install,
size=size,
logger=Logger,
directory="build")
class ZimChooserWindow:
def __init__(self, parent, zim_list_store):
......@@ -153,14 +157,14 @@ class ZimChooserWindow:
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Name", renderer_text, text=1)
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Description", renderer_text, text=3)
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Language", renderer_text, text=5)
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Size", renderer_text, text=4)
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Type", renderer_text, text=6)
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Description", renderer_text, text=3)
self.component.zim_tree_view.append_column(column_text)
self.component.zim_language_list_store.append(["all"])
renderer = Gtk.CellRendererText()
......@@ -170,6 +174,7 @@ class ZimChooserWindow:
self.component.zim_window.set_transient_for(parent)
self.component.zim_window.set_modal(True)
self.component.zim_window.set_default_size(1280, 800)
self.component.zim_window.show()
self.component.zim_window_done_button.connect("clicked", self.done_button_clicked)
......@@ -180,31 +185,6 @@ class ZimChooserWindow:
def renderer_radio_toggled(self, widget, path):
self.component.zim_list_store[path][8] = not self.component.zim_list_store[path][8]
def run_installation(name, timezone, wifi_pwd, kalite, zim_install, size):
os.makedirs("build", exist_ok=True)
os.chdir("build")
vexpress_boot.get()
raspbian.get()
emulator = qemu.Emulator(vexpress_boot.kernel_path, vexpress_boot.dtb_path, raspbian.image_path)