Commit 9f4f9639 authored by rgaudin's avatar rgaudin
Browse files

Add support for EduPi resources

	In ansible through edupi_has_resources variable (bool)
	In backend via a fake-content shortcut
	In process stage, by extracting the supplied file into the mount point
	In move-content, by calling import-from-folder

	supports both a URL and a File path.
	supports CLI via --edupi-resources
	supports GUI via 2 inputs (URL or FileChooser). URL superseeds local file.
	supports config saving/loading
parent b61b7177
......@@ -34,7 +34,7 @@ kernel_version: "4.14.34"
kiwix_version: "0.6.0"
kalite_version: "0.17.4"
ideascube_version: "0.37.2"
edupi_version: "1.4.5"
edupi_version: "1.4.7"
python3_version: "3.5"
mediawiki_major_version: 1.31
mediawiki_version: 1.31.0
......@@ -90,3 +90,4 @@ kalite_langpack_prefix: "{{ data_path }}/kalite_pack_"
kalite_videos_prefix: "{{ data_path }}/kalite_videos_"
packages_cache: "{{ data_path }}/packages_cache"
wikifundi_langpack_prefix: "{{ data_path }}/wikifundi_langpack_"
edupi_resources_path: "{{ data_path }}/edupi_resources"
......@@ -43,13 +43,16 @@
- role: aflatoun_setup
when: ansible_local.config.aflatoun_languages | length
- role: edupi
- role: edupi_setup
when: ansible_local.config.edupi|bool
- role: kalite_setup
when: ansible_local.config.kalite_languages | length
# move content
- role: edupi_content
when: ansible_local.config.edupi_has_resources|bool
- role: packages
when: ansible_local.config.packages | length
......
---
- name: Install the user-provided resources
become: yes
become_user: "{{ username }}"
command: "{{ edupi_env }}/bin/python3 {{ edupi_root }}/manage.py import-from-folder {{ edupi_resources_path }}"
tags: move-content
......@@ -18,7 +18,7 @@ def test_jinja2_file(jinja2_file):
def test_json_file(json_file):
try:
with open(json_file, mode='rb') as f:
with open(json_file, mode='r') as f:
json.load(f, encoding='utf-8')
except Exception as e:
......
......@@ -88,6 +88,7 @@ def run_for_image(machine, root_partition_size, disk_size):
name="default", timezone="UTC",
language="en", language_name="English", wifi_pwd="",
edupi=True,
edupi_resources=None,
wikifundi_languages=['en', 'fr'],
aflatoun_languages=['en', 'fr'],
kalite_languages=['en', 'fr', 'es'],
......@@ -100,7 +101,8 @@ def run_for_image(machine, root_partition_size, disk_size):
def build_extra_vars(name, timezone, language, language_name, wifi_pwd,
edupi, wikifundi_languages,
edupi, edupi_resources,
wikifundi_languages,
aflatoun_languages, kalite_languages, packages,
admin_account,
root_partition_size, disk_size):
......@@ -117,6 +119,7 @@ def build_extra_vars(name, timezone, language, language_name, wifi_pwd,
'wikifundi_languages': wikifundi_languages,
'aflatoun_languages': aflatoun_languages,
'edupi': edupi,
'edupi_has_resources': bool(edupi_resources),
'packages': packages,
'captive_portal': True,
'custom_branding_path': '/tmp',
......
......@@ -9,6 +9,8 @@ import math
import shutil
import itertools
import requests
from data import content_file, mirror
from util import get_temp_folder, get_checksum, ONE_GiB, ONE_GB
from backend.catalog import YAML_CATALOGS
......@@ -28,7 +30,53 @@ def get_content(key):
return CONTENTS.get(key)
def isremote(path_or_url):
return path_or_url.startswith('http')
def isarchive(fpath):
path, ext = os.path.splitext(fpath)
return ext in ('.zip', '.tar', '.tar.bz2', '.tar.gz', '.tar.xz')
def get_alien_content(path_or_url):
return get_remote_content(path_or_url) \
if isremote(path_or_url) else get_local_content(path_or_url)
def get_local_content(fpath):
''' content-like dict for a user-provided local file
WARN: file should be copied into cache manually '''
fname = os.path.basename(fpath)
fsize = os.path.getsize(fpath)
assert fsize > 0
return {
"url": "file://{fpath}".format(fpath=fpath),
"name": fname,
"checksum": None,
"copied_on_destination": False,
"archive_size": fsize,
"expanded_size": fsize * 1.2 if isarchive(fpath) else fsize
}
def get_remote_content(url):
fname = os.path.basename(url)
fsize = int(requests.head(url).headers['Content-Length'])
assert fsize > 0
return {
"url": url,
"name": fname,
"checksum": None,
"copied_on_destination": False,
"archive_size": fsize,
"expanded_size": fsize * 1.2 if isarchive(url) else fsize
}
def get_collection(edupi=False,
edupi_resources=None,
packages=[],
kalite_languages=[],
wikifundi_languages=[],
......@@ -57,7 +105,8 @@ def get_collection(edupi=False,
if edupi:
collection.append(('EduPi',
get_edupi_contents, run_edupi_actions,
{'enable': edupi}))
{'enable': edupi,
'resources_path': edupi_resources}))
if len(packages):
collection.append(('Packages',
......@@ -91,9 +140,9 @@ def get_all_contents_for(collection):
])
def get_edupi_contents(enable=False):
''' edupi: has no large downloads '''
return []
def get_edupi_contents(enable=False, resources_path=None):
''' edupi: has no large downloads. might have user-specified one '''
return [get_alien_content(resources_path)] if resources_path else []
def get_kalite_contents(languages=[]):
......@@ -109,7 +158,7 @@ def get_kalite_contents(languages=[]):
def get_wikifundi_contents(languages=[]):
''' wikifundi: small size parsoid + large language pack for each lang '''
return [get_content('wikifundi_parsoid')] + [
return [
get_content('wikifundi_langpack_{lang}'.format(lang=lang))
for lang in languages]
......@@ -181,9 +230,17 @@ def copy(content, cache_folder, final_path, logger):
shutil.copy(archive_fpath, final_path)
def run_edupi_actions(cache_folder, mount_point, logger, enable=False):
def run_edupi_actions(cache_folder, mount_point, logger,
enable=False, resources_path=None):
''' no action for EduPi ; everything within ansiblecube '''
return
if not enable or not resources_path:
return
extract_and_move(content=get_alien_content(resources_path),
cache_folder=cache_folder,
root_path=mount_point,
final_path=os.path.join(mount_point, 'edupi_resources'),
logger=logger)
def run_kalite_actions(cache_folder, mount_point, logger, languages=[]):
......
......@@ -8,7 +8,7 @@ import tempfile
import data
from backend.catalog import YAML_CATALOGS
from backend.content import (get_collection, get_required_building_space,
get_required_image_size, get_content)
get_required_image_size, get_content, isremote)
from run_installation import run_installation
from util import CancelEvent
from util import get_free_space_in_dir
......@@ -99,6 +99,12 @@ def set_config(config, args):
vl = "yes" if config["content"][key] in ("yes", True) else "no"
setif(key, vl)
# edupi resources
if config["content"].get("edupi_resources") is not None:
rsc = config["content"]["edupi_resources"]
setif('edupi_resources',
rsc if isremote(rsc) else os.path.abspath(rsc))
try:
assert len(YAML_CATALOGS)
......@@ -143,6 +149,8 @@ parser.add_argument("--aflatoun", help="install aflatoun",
parser.add_argument("--wikifundi", help="install wikifundi",
choices=["fr", "en"], nargs="+")
parser.add_argument("--edupi", help="install edupi", choices=["yes", "no"])
parser.add_argument("--edupi-resources",
help="Zipped folder of resources to init EduPi with")
parser.add_argument("--zim-install", help="install zim",
choices=zim_choices, nargs="+")
parser.add_argument("--size", help="resize image ({})"
......@@ -214,6 +222,7 @@ for name in keys:
# check disk space
collection = get_collection(edupi=args.edupi == "yes",
edupi_resources=args.edupi_resources,
packages=args.zim_install,
kalite_languages=args.kalite,
wikifundi_languages=args.wikifundi,
......@@ -268,6 +277,7 @@ try:
wikifundi=args.wikifundi,
aflatoun=args.aflatoun == "yes",
edupi=args.edupi == "yes",
edupi_resources=args.edupi_resources,
zim_install=args.zim_install,
size=args.size,
logger=logger,
......
......@@ -3,7 +3,7 @@ import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, Gdk, GLib, GdkPixbuf, GObject
from backend.catalog import YAML_CATALOGS
from backend.content import get_expanded_size, get_collection, get_required_image_size, get_content
from backend.content import get_expanded_size, get_collection, get_required_image_size, get_content, isremote
from run_installation import run_installation
import pytz
import tzlocal
......@@ -216,6 +216,11 @@ class Application:
self.component.css_chooser.add_filter(
self.component.css_filter)
self.component.edupi_resources_filter.set_name("ZIP File") # opt
self.component.edupi_resources_filter.add_pattern("*.zip")
self.component.css_chooser.add_filter(
self.component.edupi_resources_filter)
# menu bar
self.component.menu_quit.connect("activate", lambda widget: self.component.window.close())
self.component.menu_about.connect("activate", self.activate_menu_about)
......@@ -233,6 +238,9 @@ class Application:
# admin account
self.component.admin_account_switch.connect("notify::active", lambda switch, state: self.component.admin_account_revealer.set_reveal_child(switch.get_active()))
# edupi resources
self.component.edupi_switch.connect("notify::active", lambda switch, state: self.component.edupi_resources_revealer.set_reveal_child(switch.get_active()))
# ideascube language
for code, language in data.ideascube_languages:
self.component.language_tree_store.append([code, language])
......@@ -396,7 +404,9 @@ class Application:
# edupi
self.component.edupi_switch.connect("notify::active", lambda switch, state: self.update_free_space())
self.component.edupi_label.set_label("{} ({})".format(self.component.edupi_label.get_label(), human_readable_size(ONE_MiB)))
self.component.edupi_label.set_label("{} ({})".format(self.component.edupi_label.get_label(), human_readable_size(10 * ONE_MiB)))
self.component.edupi_resources_url_entry.connect("changed", lambda _: self.update_free_space())
self.component.edupi_resources_chooser.connect("file-set", lambda _: self.update_free_space())
# language tree view
renderer_text = Gtk.CellRendererText()
......@@ -645,6 +655,16 @@ class Application:
getattr(self.component, '{}_switch'.format(key)) \
.set_active(config["content"][key])
# edupi resources
if config["content"].get("edupi_resources") is not None:
rsc = config["content"].get("edupi_resources")
if isremote(rsc):
self.component.edupi_resources_url_entry \
.set_text(str(rsc))
else:
self.component.edupi_resources_chooser \
.set_filename(str(rsc))
if "zims" in config["content"] \
and isinstance(config["content"]["zims"], list):
......@@ -682,6 +702,10 @@ class Application:
except Exception:
timezone = None
edupi_resources = self.get_edupi_resources()
if not isremote(edupi_resources):
edupi_resources = relpathto(self.get_edupi_resources())
zim_install = []
for zim in self.component.zim_list_store:
if zim[8]:
......@@ -727,13 +751,14 @@ class Application:
}),
("build_dir",
relpathto(self.component.build_path_chooser.get_filename())),
("size", human_readable_size(size, False)),
("size", None if size is None else human_readable_size(size, False)),
("content", {
"zims": zim_install, # content-ids list
"kalite": kalite_active_langs, # languages list
"wikifundi": wikifundi_active_langs, # languages list
"aflatoun": self.component.aflatoun_switch.get_active(),
"edupi": self.component.edupi_switch.get_active(),
"edupi_resources": edupi_resources,
}),
("branding", branding),
])
......@@ -918,6 +943,11 @@ class Application:
def zim_choose_content_button_clicked(self, button):
self.component.zim_window.show()
def get_edupi_resources(self):
local_rsc = self.component.edupi_resources_chooser.get_filename()
remote_rsc = self.component.edupi_resources_url_entry.get_text()
return (remote_rsc if remote_rsc else local_rsc) or None
def get_free_space(self):
zim_list = []
for zim in self.component.zim_list_store:
......@@ -936,9 +966,11 @@ class Application:
aflatoun = self.component.aflatoun_switch.get_active()
edupi = self.component.edupi_switch.get_active()
edupi_resources = self.get_edupi_resources()
collection = get_collection(
edupi=edupi,
edupi_resources=edupi_resources,
packages=zim_list,
kalite_languages=kalite,
wikifundi_languages=wikifundi,
......
from backend import ansiblecube
from backend import qemu
from backend.content import get_collection, get_content, get_all_contents_for
from backend.content import (get_collection, get_content,
get_all_contents_for, isremote)
from backend.download import download_content, unzip_file
from backend.mount import mount_data_partition, unmount_data_partition, test_mount_procedure
from backend.util import subprocess_pretty_check_call, subprocess_pretty_call
......@@ -11,13 +12,15 @@ from datetime import datetime
import os
import sys
import re
import shutil
def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite, aflatoun, wikifundi, edupi, zim_install, size, logger, cancel_event, sd_card, favicon, logo, css, done_callback=None, build_dir=".", qemu_ram="2G"):
def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite, aflatoun, wikifundi, edupi, edupi_resources, zim_install, size, logger, cancel_event, sd_card, favicon, logo, css, done_callback=None, build_dir=".", qemu_ram="2G"):
logger.start(bool(sd_card))
logger.stage('init')
cache_folder = get_cache(build_dir)
try:
logger.step("Check System Requirements")
......@@ -96,10 +99,15 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
wikifundi_languages = [] if wikifundi is None else wikifundi
aflatoun_languages = ['fr', 'en'] if aflatoun else []
if edupi_resources and not isremote(edupi_resources):
logger.step("Copying EduPi resources into cache")
shutil.copy(edupi_resources, cache_folder)
# collection contains both downloads and processing callbacks
# for all requested contents
collection = get_collection(
edupi=edupi,
edupi_resources=edupi_resources,
packages=packages,
kalite_languages=kalite_languages,
wikifundi_languages=wikifundi_languages,
......@@ -157,6 +165,7 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
'language_name': dict(data.ideascube_languages)[language],
'edupi': edupi,
'edupi_resources': edupi_resources,
'wikifundi_languages': wikifundi_languages,
'aflatoun_languages': aflatoun_languages,
'kalite_languages': kalite_languages,
......@@ -198,7 +207,6 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
expanded_total_size = sum([c['expanded_size'] for c in downloads])
processed = 0
cache_folder = get_cache(build_dir)
for category, content_dl_cb, \
content_run_cb, cb_kwargs in collection:
......
......@@ -20,6 +20,11 @@
<mime-type>image/png</mime-type>
</mime-types>
</object>
<object class="GtkFileFilter" id="edupi_resources_filter">
<mime-types>
<mime-type>application/zip</mime-type>
</mime-types>
</object>
<object class="GtkFileFilter" id="favicon_filter">
<mime-types>
<mime-type>image/png</mime-type>
......@@ -1199,6 +1204,66 @@ In File mode the image is created at build path, you can choose the size
<property name="position">16</property>
</packing>
</child>
<child>
<object class="GtkRevealer" id="edupi_resources_revealer">
<property name="visible">True</property>
<property name="can_focus">False</property>
<child>
<object class="GtkBox" id="no_id_77">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="spacing">5</property>
<child>
<object class="GtkLabel" id="edupi_resources_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="tooltip_text" translatable="yes">Preload EduPi with content from a ZIP file</property>
<property name="label" translatable="yes">EduPi Initial Data :</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkEntry" id="edupi_resources_url_entry">
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="text" translatable="yes"></property>
<property name="tooltip_text" translatable="yes">URL to a ZIP file containing EduPi resources (overrides local ZIP file)</property>
</object>
<packing>
<property name="expand">True</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
<child>
<object class="GtkFileChooserButton" id="edupi_resources_chooser">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="create_folders">False</property>
<property name="filter">edupi_resources_filter</property>
<property name="title" translatable="yes"/>
<property name="tooltip_text" translatable="yes">ZIP file containing EduPi resources</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">False</property>
<property name="pack_type">end</property>
<property name="position">2</property>
</packing>
</child>
</object>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">17</property>
</packing>
</child>
<child>
<object class="GtkBox" id="no_id_40">
<property name="visible">True</property>
......@@ -1275,7 +1340,7 @@ In File mode the image is created at build path, you can choose the size
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">17</property>
<property name="position">18</property>
</packing>
</child>
<child>
......@@ -1340,7 +1405,7 @@ In File mode the image is created at build path, you can choose the size
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">18</property>
<property name="position">19</property>
</packing>
</child>
<child>
......@@ -1376,7 +1441,7 @@ In File mode the image is created at build path, you can choose the size
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">19</property>
<property name="position">20</property>
</packing>
</child>
</object>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment