catalog.py 2.24 KB
Newer Older
1
2
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu
3

rgaudin's avatar
rgaudin committed
4
import os
5
import random
rgaudin's avatar
rgaudin committed
6
import shutil
7
8
import tempfile

thiolliere's avatar
thiolliere committed
9
import yaml
thiolliere's avatar
thiolliere committed
10

11
12
from backend.download import download_file

rgaudin's avatar
rgaudin committed
13
CATALOGS = [
14
15
16
    {
        "name": "Kiwix",
        "description": "Kiwix ZIM Content",
17
        "url": "http://mirror.download.kiwix.org/library/ideascube.yml",
18
    }
19
20
]

21
22
23
24
25
26
27
28
YAML_CATALOGS = None


def fetch_catalogs(logger):
    """ build a dict of loaded (yaml) catalogs from CATALOGS """
    catalogs = []
    logger.std("downloading catalogs...")
    try:
rgaudin's avatar
rgaudin committed
29
30
        tmp_dir = tempfile.mkdtemp()

31
        for catalog in CATALOGS:
rgaudin's avatar
rgaudin committed
32
33
            tmp_fpath = os.path.join(tmp_dir, "{}.catalog".format(catalog["name"]))
            dlf = download_file(catalog.get("url"), tmp_fpath, logger)
34
            if dlf.successful:
rgaudin's avatar
rgaudin committed
35
                with open(tmp_fpath, "r") as fp:
36
                    catalogs.append(yaml.load(fp.read()))
rgaudin's avatar
rgaudin committed
37
                os.unlink(tmp_fpath)
38
39
            else:
                raise ValueError("Unable to download {}".format(catalog.get("url")))
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

            # ensure the content is readable (prevent incorrect encoding)
            entry = catalogs[-1]["all"][random.choice(list(catalogs[-1]["all"].keys()))]
            for key in (
                "name",
                "description",
                "version",
                "language",
                "id",
                "url",
                "sha256sum",
                "type",
                "langid",
            ):
                if not entry.get(key) or not isinstance(entry[key], str):
                    logger.err("Catalog format is not valid")
                    catalogs.pop()  # remove catalog from list
                    break
rgaudin's avatar
rgaudin committed
58
59

        shutil.rmtree(tmp_dir, ignore_errors=True)
60
61
62
63
64
65
66
67
68
69
70
71
    except Exception as exp:
        logger.err("Exception while downloading/parsing catalogs: {}".format(exp))
        return None
    return catalogs if len(catalogs) else None


def get_catalogs(logger):
    """ cached-shortcut to YAML_CATALOGS """
    global YAML_CATALOGS
    if YAML_CATALOGS is None:
        YAML_CATALOGS = fetch_catalogs(logger)
    return YAML_CATALOGS
rgaudin's avatar
rgaudin committed
72
73
74
75
76
77


def get_package(logger, package_id):
    for catalog in get_catalogs(logger):
        if package_id in catalog["all"].keys():
            return catalog["all"][package_id]