Commit 4bd7df19 authored by rgaudin's avatar rgaudin
Browse files

Issue #246: moved catalogs download out of imports

- YAML_CATALOGS replaced by get_catalogs() (cached)
- UI starts without catalogs
- catalogs downloaded asynchronously on startup
- catalogs checked or attempted to download on ZIM Select button click: raise error if failed
parent f65828ba
......@@ -7,14 +7,14 @@ import shutil
from util import human_readable_size
from backend.content import CONTENTS
from backend.content import get_content
from backend.catalog import YAML_CATALOGS
from backend.catalog import get_catalogs
from util import get_cache, get_folder_size, get_free_space_in_dir, get_checksum
def package_is_latest_version(fpath, fname):
def package_is_latest_version(fpath, fname, logger):
""" whether a package (ZIM or ZIP) is in the current catalog """
for catalog in YAML_CATALOGS:
for catalog in get_catalogs(logger):
for (package_id, package) in catalog["all"].items():
package.update({"ext": "zip" if package["type"] != "zim" else "zim"})
package.update({"langid": package.get("langid") or package_id})
......@@ -26,7 +26,7 @@ def package_is_latest_version(fpath, fname):
return False
def is_latest_version(fpath, fname):
def is_latest_version(fpath, fname, logger):
""" whether the filename is a usable content """
if fname.startswith("package_"):
......@@ -53,7 +53,7 @@ def get_cache_file_details(logger, cache_folder, fname):
alien = True
if isdir:
alien = True # our cache contains only files
latest = is_latest_version(fpath, fname)
latest = is_latest_version(fpath, fname, logger)
return {
"fname": fname,
......
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu
import requests
import tempfile
import yaml
from backend.download import download_file
CATALOGS = [
{
"name": "Kiwix",
......@@ -17,11 +20,32 @@ CATALOGS = [
},
]
YAML_CATALOGS = []
try:
for catalog in CATALOGS:
YAML_CATALOGS.append(
yaml.load(requests.get(catalog.get("url")).content.decode("utf-8"))
)
except Exception:
pass
YAML_CATALOGS = None
def fetch_catalogs(logger):
""" build a dict of loaded (yaml) catalogs from CATALOGS """
catalogs = []
logger.std("downloading catalogs...")
try:
for catalog in CATALOGS:
fpath = tempfile.NamedTemporaryFile(suffix=".yml")
dlf = download_file(catalog.get("url"), fpath.name, logger)
fpath.seek(0) # reset as we'll read it right away
if dlf.successful:
catalogs.append(yaml.load(fpath.read()))
fpath.close()
else:
raise ValueError("Unable to download {}".format(catalog.get("url")))
except Exception as exp:
logger.err("Exception while downloading/parsing catalogs: {}".format(exp))
return None
return catalogs if len(catalogs) else None
def get_catalogs(logger):
""" cached-shortcut to YAML_CATALOGS """
global YAML_CATALOGS
if YAML_CATALOGS is None:
YAML_CATALOGS = fetch_catalogs(logger)
return YAML_CATALOGS
......@@ -10,9 +10,9 @@ import itertools
import requests
from data import content_file, mirror
from backend.catalog import YAML_CATALOGS
from backend.catalog import get_catalogs
from backend.download import get_content_cache, unarchive
from util import get_temp_folder, get_checksum, ONE_GiB, ONE_MiB
from util import get_temp_folder, get_checksum, ONE_GiB, ONE_MiB, CLILogger
# prepare CONTENTS from JSON file
with open(content_file, "r") as fp:
......@@ -195,7 +195,7 @@ def get_aflatoun_contents(languages=[]):
def get_package_content(package_id):
""" content-like dict for packages (zim file or static site) """
for catalog in YAML_CATALOGS:
for catalog in get_catalogs(CLILogger()):
try:
package = catalog["all"][package_id]
package.update({"ext": "zip" if package["type"] != "zim" else "zim"})
......
......@@ -14,7 +14,7 @@ import argparse
from backend.content import CONTENTS
from util import CLILogger, get_cache
from backend.catalog import YAML_CATALOGS
from backend.catalog import get_catalogs
from backend.cache import list_cache_files, clean_cache, reset_cache
......@@ -22,7 +22,7 @@ def init(logger):
""" verify that the CATALOGS and CONTENTS list are populated (not empty) """
logger.step("initializing...", end="")
nb_contents = len(CONTENTS)
nb_packages = sum([len(c["all"]) for c in YAML_CATALOGS])
nb_packages = sum([len(c["all"]) for c in get_catalogs(logger)])
nums = "({}+{})".format(nb_contents, nb_packages)
if not nb_contents or not nb_packages:
logger.err("FAILED. Check your internet connection. {}".format(nums))
......
......@@ -23,7 +23,7 @@ from util import check_user_inputs
from util import CLILogger, b64decode
from util import get_free_space_in_dir
from util import get_adjusted_image_size
from backend.catalog import YAML_CATALOGS
from backend.catalog import get_catalogs
from run_installation import run_installation
from util import human_readable_size, get_cache
......@@ -127,15 +127,12 @@ def set_config(config, args):
setif("edupi_resources", rsc if isremote(rsc) else os.path.abspath(rsc))
try:
assert len(YAML_CATALOGS)
except Exception as exception:
print(exception, file=sys.stderr)
if get_catalogs(logger) is None:
print("Catalog downloads failed, you may check your internet connection")
sys.exit(2)
zim_choices = []
for catalog in YAML_CATALOGS:
for catalog in get_catalogs(logger):
for (key, value) in catalog["all"].items():
zim_choices.append(key)
......@@ -218,7 +215,7 @@ for key, value in defaults.items():
setattr(args, key, value)
if args.catalog:
for catalog in YAML_CATALOGS:
for catalog in get_catalogs(logger):
print(yaml.dump(catalog, default_flow_style=False, default_style=""))
sys.exit(0)
......
......@@ -36,7 +36,7 @@ from version import get_version_str
from util import b64encode, b64decode
from util import get_free_space_in_dir
from util import get_adjusted_image_size
from backend.catalog import YAML_CATALOGS
from backend.catalog import get_catalogs
from util import CancelEvent, ProgressHelper
from run_installation import run_installation
from backend.mount import open_explorer_for_imdisk
......@@ -231,8 +231,8 @@ def validate_label(label, condition):
class Application:
def __init__(self, catalog):
self.catalog = catalog
def __init__(self):
self.catalogs = None
builder = Gtk.Builder()
builder.add_from_file(data.ui_glade)
......@@ -410,9 +410,96 @@ class Application:
)
self.component.zim_list_store.set_sort_column_id(1, Gtk.SortType.ASCENDING)
def get_project_size(name, lang):
langs = ["fr", "en"] if name == "aflatoun" else [lang]
return get_expanded_size(
get_collection(**{"{}_languages".format(name): langs})
)
# kalite
for lang, button in self.iter_kalite_check_button():
button.set_label(
"{} ({})".format(
button.get_label(),
human_readable_size(get_project_size("kalite", lang)),
)
)
button.connect("toggled", lambda button: self.update_free_space())
# wikifundi
for lang, button in self.iter_wikifundi_check_button():
button.set_label(
"{} ({})".format(
button.get_label(),
human_readable_size(get_project_size("wikifundi", lang)),
)
)
button.connect("toggled", lambda button: self.update_free_space())
# aflatoun
self.component.aflatoun_switch.connect(
"notify::active", lambda switch, state: self.update_free_space()
)
self.component.aflatoun_label.set_label(
"{} ({})".format(
self.component.aflatoun_label.get_label(),
human_readable_size(get_project_size("aflatoun", lang)),
)
)
# edupi
self.component.edupi_switch.connect(
"notify::active", lambda switch, state: self.update_free_space()
)
self.component.edupi_label.set_label(
"{} ({})".format(
self.component.edupi_label.get_label(),
human_readable_size(10 * ONE_MiB),
)
)
self.component.edupi_resources_url_entry.connect(
"changed", lambda _: self.update_free_space()
)
self.component.edupi_resources_chooser.connect(
"file-set", lambda _: self.update_free_space()
)
self.refresh_disk_list()
self.reset_config() # will calculate free space
self.component.window.show()
self.catalogs_thread = threading.Thread(target=self.download_catalogs)
self.catalogs_thread.start()
def download_catalogs(self):
print("downloading catalogs...")
self.catalogs = get_catalogs(CLILogger())
print("catalogs downloaded")
if self.catalogs is not None:
self.build_zim_store()
return self.catalogs is not None
def ensure_catalogs(self):
if self.catalogs_thread.is_alive():
print("waiting for the download thread to complete")
self.catalogs_thread.join()
if self.catalogs is None:
if not self.download_catalogs():
self.display_error_message(
title="Catalogs Download Failed",
message="Could not download the Content Catalogs. Please check your Internet connection and/or Proxy Settings (File menu).",
parent=self.component.window,
)
return False
return True
def build_zim_store(self):
print("building zim store")
all_languages = set()
for one_catalog in catalog:
for one_catalog in self.catalogs:
for (key, value) in one_catalog["all"].items():
name = value["name"]
url = value["url"]
......@@ -495,60 +582,6 @@ class Application:
column_text = Gtk.TreeViewColumn("Description", renderer_text, text=3)
self.component.choosen_zim_tree_view.append_column(column_text)
def get_project_size(name, lang):
langs = ["fr", "en"] if name == "aflatoun" else [lang]
return get_expanded_size(
get_collection(**{"{}_languages".format(name): langs})
)
# kalite
for lang, button in self.iter_kalite_check_button():
button.set_label(
"{} ({})".format(
button.get_label(),
human_readable_size(get_project_size("kalite", lang)),
)
)
button.connect("toggled", lambda button: self.update_free_space())
# wikifundi
for lang, button in self.iter_wikifundi_check_button():
button.set_label(
"{} ({})".format(
button.get_label(),
human_readable_size(get_project_size("wikifundi", lang)),
)
)
button.connect("toggled", lambda button: self.update_free_space())
# aflatoun
self.component.aflatoun_switch.connect(
"notify::active", lambda switch, state: self.update_free_space()
)
self.component.aflatoun_label.set_label(
"{} ({})".format(
self.component.aflatoun_label.get_label(),
human_readable_size(get_project_size("aflatoun", lang)),
)
)
# edupi
self.component.edupi_switch.connect(
"notify::active", lambda switch, state: self.update_free_space()
)
self.component.edupi_label.set_label(
"{} ({})".format(
self.component.edupi_label.get_label(),
human_readable_size(10 * ONE_MiB),
)
)
self.component.edupi_resources_url_entry.connect(
"changed", lambda _: self.update_free_space()
)
self.component.edupi_resources_chooser.connect(
"file-set", lambda _: self.update_free_space()
)
# language tree view
renderer_text = Gtk.CellRendererText()
column_text = Gtk.TreeViewColumn("Language", renderer_text, text=0)
......@@ -565,11 +598,12 @@ class Application:
"changed", self.zim_language_selection_changed
)
self.refresh_disk_list()
self.reset_config() # will calculate free space
# apply chosen zim filter
choosen_zim_filter = self.component.zim_list_store.filter_new()
choosen_zim_filter.set_visible_func(self.choosen_zim_filter_func)
self.component.choosen_zim_tree_view.set_model(choosen_zim_filter)
self.component.window.show()
self.update_free_space()
def reset_config(self):
""" restore UI to its initial (non-configured) state """
......@@ -1497,6 +1531,8 @@ class Application:
self.component.space_error_window.show()
all_valid = False
all_valid = all_valid and self.ensure_catalogs()
if all_valid:
def target():
......@@ -1554,7 +1590,8 @@ class Application:
self.component.sd_card_combobox.set_active(id)
def zim_choose_content_button_clicked(self, button):
self.component.zim_window.show()
if self.ensure_catalogs():
self.component.zim_window.show()
def get_edupi_resources(self):
local_rsc = self.component.edupi_resources_chooser.get_filename()
......@@ -1676,19 +1713,6 @@ class Application:
return model[iter][8]
try:
assert len(YAML_CATALOGS)
except Exception as exception:
dialog = ShortDialog(
None,
(Gtk.STOCK_OK, Gtk.ResponseType.OK),
"Catalog downloads failed, you may check your internet connection",
)
dialog.run()
print(exception, file=sys.stderr)
dialog.destroy()
sys.exit(1)
Application(YAML_CATALOGS)
Application()
run()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment