__init__.py 6.57 KB
Newer Older
Louis's avatar
Louis committed
1
2
#!/usr/bin/env python3

3
import colorsys
Louis's avatar
Louis committed
4
import datetime
5
import functools
6
7
import itertools
import jinja2
8
9
10
11
import json
import operator
import os
import pkg_resources
12
import random
13
import requests
Louis's avatar
Louis committed
14
import slumber
Louis's avatar
Louis committed
15
import sys
Louis's avatar
Louis committed
16
import urllib
17
import xmlrpc.client as xmlrpclib
18
import yaml
Louis's avatar
Louis committed
19

20
21
from pypimonitor.column import load_column_plugins

22
23
VERSION = "0.2.0-dev"

24
_client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
25

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
class Counter:
    def __init__(self):
        self.value = 0

    def count(self):
        self.value += 1
        return self.value

def as_id(string):
    return "".join(list(filter(str.isalnum, string)))

def random_color(seed, saturation, value):
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
    return ("#%02x%02x%02x" % tuple([int(value * 255) for value in rgb]))

def iter_user_packages(user):
    for _, package in _client.user_packages(user):
        try:
            yield package
        except urllib.error.HTTPError as error:
            sys.stderr.write("Ignoring user '{}': {}.".format(
                user,
                str(error),
                ))

class Renderer:

Louis's avatar
Louis committed
56
    def __init__(self, config, form=None):
57
        # Processing config
58
        self.config = config
59
60
61
        if "columns" not in self.config:
            self.config['columns'] = list(set(itertools.chain(*[
                    package.keys()
62
                    for package
63
64
                    in self.config['packages'].values()
                ])))
65
66
67
68
69
70
71
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
            for column in self.config['columns']:
                if self.config['packages'][package].get(column, None) is None:
                    self.config['packages'][package][column] = {}
                self.config['packages'][package][column]['column'] = self.config['packages'][package][column].get('column', column)
72
73

        # Processing packages
Louis's avatar
Louis committed
74
        self.packages = {}
75
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
76
77
            try:
                self.packages[name] = Package(name)
78
79
            except Exception as error:
                sys.stderr.write("Ignoring package '{}': {}.\n".format(name, error))
Louis's avatar
Louis committed
80
                continue
81

Louis's avatar
Louis committed
82
83
84
85
86
87
        # Processing form
        self.form = form
        if self.form is not None:
            for key in ['pkg', 'user', 'col']:
                self.form[key] = ",".join(set(itertools.chain(*[item.split(",") for item in self.form.get(key, [])])))

88
89
90
91
92
93
94
        # Loading plugins
        self.columns = {
            plugin.keyword: plugin(self)
            for plugin
            in load_column_plugins()
            }

95
    @classmethod
Louis's avatar
Louis committed
96
    def from_yaml(cls, filename, form=None):
97
        with open(filename, 'r') as stream:
Louis's avatar
Louis committed
98
            return cls(yaml.safe_load(stream.read()), form=form)
99
100

    @classmethod
Louis's avatar
Louis committed
101
    def from_args(cls, packages=None, columns=None, users=None, form=None):
102
103
104
105
106
107
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
108
109
        if users is not None:
            for user in users:
110
111
                packages.extend(iter_user_packages(user))
        packages = list(set(packages))
112
        config['packages'] = {pkg: {} for pkg in set(packages)}
113

114
115
116
117
118
119
        # Processing columns
        if columns is None:
            columns = []
        columns = list(itertools.chain(*[item.split(",") for item in columns]))
        if columns:
            config['columns'] = columns
120

Louis's avatar
Louis committed
121
        return cls(config, form=form)
122
123
124
125

    def __iter__(self):
        yield from self.packages

Louis's avatar
Louis committed
126
127
128
129
130
131
    def values(self):
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    @staticmethod
    def _get_environment():
        TEMPLATEDIR = os.path.join(
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

        env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATEDIR))
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
        env = self._get_environment()
        template = env.get_template("index.html")
Louis's avatar
Louis committed
149
        return template.render(
150
151
            config=self.config,
            columns=self.columns,
152
            packages=self.packages,
Louis's avatar
Louis committed
153
            form=self.form,
Louis's avatar
Louis committed
154
            )
155
156


157
158
class Package:

Louis's avatar
Louis committed
159
    docs_api = {
160
        'readthedocs': slumber.API(base_url="http://readthedocs.io/api/v1/"),
Louis's avatar
Louis committed
161
162
        }

163
    def __init__(self, name):
Louis's avatar
Louis committed
164
        sys.stderr.write("Retrieving information about pypi package '{}'...\n".format(name))
165
166
167
168
        try:
            self.rawdata = json.loads(requests.get("http://pypi.python.org/pypi/{}/json".format(name)).text)
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
169
        self.name = name
170
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
171
        self.releases = {}
172
173
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
174
                continue
175
176
177
178
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
179
            self.releases[date] = {}
180
181
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
182
183
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
184
185
186
187
188
189
190
191
192
193
194
195
196

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    def iter_dates(self):
        yield from self.releases.keys()

    @property
    def total_downloads(self):
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']
197

Louis's avatar
Louis committed
198
199
200
201
202
203
204
205
206
    def get_documentation(self):
        # Readthedocs
        if self.docs_api['readthedocs'].project.get(slug=self.name.lower())['objects']:
            return (
                'readthedocs', 
                None,
                )

        # Pypi docs
207
        if self.info['docs_url'] is not None:
Louis's avatar
Louis committed
208
            return ("pypi", None)
Louis's avatar
Louis committed
209
210

        # No documentation
Louis's avatar
Louis committed
211
        return (None, None)