catalog.py 2.01 KB
Newer Older
1
2
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu
3

rgaudin's avatar
rgaudin committed
4
import os
5
import random
6
7
import tempfile

thiolliere's avatar
thiolliere committed
8
import yaml
thiolliere's avatar
thiolliere committed
9

10
11
from backend.download import download_file

rgaudin's avatar
rgaudin committed
12
CATALOGS = [
13
14
15
16
    {
        "name": "Kiwix",
        "description": "Kiwix ZIM Content",
        "url": "http://download.kiwix.org/library/ideascube.yml",
17
    }
18
19
]

20
21
22
23
24
25
26
27
28
YAML_CATALOGS = None


def fetch_catalogs(logger):
    """ build a dict of loaded (yaml) catalogs from CATALOGS """
    catalogs = []
    logger.std("downloading catalogs...")
    try:
        for catalog in CATALOGS:
rgaudin's avatar
rgaudin committed
29
30
31
            tmpfile = tempfile.NamedTemporaryFile(suffix=".yml", delete=False)
            dlf = download_file(catalog.get("url"), tmpfile.name, logger)
            tmpfile.seek(0)  # reset as we'll read it right away
32
            if dlf.successful:
rgaudin's avatar
rgaudin committed
33
34
35
                catalogs.append(yaml.load(tmpfile.read()))
                tmpfile.close()
                os.unlink(tmpfile.name)
36
37
            else:
                raise ValueError("Unable to download {}".format(catalog.get("url")))
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

            # ensure the content is readable (prevent incorrect encoding)
            entry = catalogs[-1]["all"][random.choice(list(catalogs[-1]["all"].keys()))]
            for key in (
                "name",
                "description",
                "version",
                "language",
                "id",
                "url",
                "sha256sum",
                "type",
                "langid",
            ):
                if not entry.get(key) or not isinstance(entry[key], str):
                    logger.err("Catalog format is not valid")
                    catalogs.pop()  # remove catalog from list
                    break
56
57
58
59
60
61
62
63
64
65
66
67
    except Exception as exp:
        logger.err("Exception while downloading/parsing catalogs: {}".format(exp))
        return None
    return catalogs if len(catalogs) else None


def get_catalogs(logger):
    """ cached-shortcut to YAML_CATALOGS """
    global YAML_CATALOGS
    if YAML_CATALOGS is None:
        YAML_CATALOGS = fetch_catalogs(logger)
    return YAML_CATALOGS