__init__.py 8.18 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# pypimonitor — An HTML dashboard to monitor your python packages
# Copyright (C) 2016 Louis Paternault
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

Louis's avatar
Louis committed
17
18
"""Gather data about packages, and render them as an HTML page."""

19
import colorsys
Louis's avatar
Louis committed
20
import datetime
21
import functools
22
import itertools
23
import json
24
import logging
25
26
import operator
import os
27
import random
Louis's avatar
Louis committed
28
import sys
Louis's avatar
Louis committed
29
import urllib
30
import xmlrpc.client as xmlrpclib
31
import yaml
Louis's avatar
Louis committed
32

Louis's avatar
Louis committed
33
34
35
36
import jinja2
import pkg_resources
import requests

Louis's avatar
Louis committed
37
from pypimonitor.cell import load_cell_plugins
38

39
VERSION = "0.2.0-dev"
40
LOGGER = logging.getLogger(__name__)
41

Louis's avatar
Louis committed
42
CLIENT = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
43

44
class Counter:
Louis's avatar
Louis committed
45
46
47
    """Each call to :meth:`Counter.count` increments a counter, and return its value."""
    # pylint: disable=too-few-public-methods

48
49
50
51
    def __init__(self):
        self.value = 0

    def count(self):
Louis's avatar
Louis committed
52
        """Increments counter, and return counter value."""
53
54
55
56
        self.value += 1
        return self.value

def random_color(seed, saturation, value):
Louis's avatar
Louis committed
57
    """Return a pseudo-random color."""
58
59
60
61
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
Louis's avatar
Louis committed
62
    return "#%02x%02x%02x" % tuple([int(value * 255) for value in rgb])
63
64

def iter_user_packages(user):
Louis's avatar
Louis committed
65
66
    """Iterate over packages owned by a given user."""
    for _, package in CLIENT.user_packages(user):
67
        yield package
68
69

class Renderer:
Louis's avatar
Louis committed
70
    """Get information about packages, and produce an HTML dashboard to monitor them."""
71

Louis's avatar
Louis committed
72
    def __init__(self, config, form=None):
73
74
        self.errors = set()

75
        # Processing config
76
        self.config = config
Louis's avatar
Louis committed
77
78
        if "cells" not in self.config:
            self.config['cells'] = list(set(itertools.chain(*[
Louis's avatar
Louis committed
79
80
81
                package.keys()
                for package
                in self.config['packages'].values()
82
                ])))
83
84
85
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
Louis's avatar
Louis committed
86
87
88
            for cell in self.config['cells']:
                if self.config['packages'][package].get(cell, None) is None:
                    self.config['packages'][package][cell] = {}
89
90
91
92
                # Set default values
                for key in self.config.get('default', {}):
                    if key not in self.config['packages'][package]:
                        self.config['packages'][package][key] = self.config['default'][key]
Louis's avatar
Louis committed
93
94
                if 'cell' not in self.config['packages'][package][cell]:
                    self.config['packages'][package][cell]['cell'] = cell
95
96

        # Processing packages
Louis's avatar
Louis committed
97
        self.packages = {}
98
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
99
100
            try:
                self.packages[name] = Package(name)
Louis's avatar
Louis committed
101
            except Exception as error: # pylint: disable=broad-except
102
                self.log_error("Ignoring package '{}': {}".format(name, error))
Louis's avatar
Louis committed
103
                continue
104

Louis's avatar
Louis committed
105
106
107
108
        # Processing form
        self.form = form
        if self.form is not None:
            for key in ['pkg', 'user', 'col']:
Louis's avatar
Louis committed
109
110
111
                self.form[key] = ",".join(set(
                    itertools.chain(*[item.split(",") for item in self.form.get(key, [])])
                ))
Louis's avatar
Louis committed
112

113
        # Loading plugins
Louis's avatar
Louis committed
114
        self.cells = {
115
116
            plugin.keyword: plugin(self)
            for plugin
Louis's avatar
Louis committed
117
            in load_cell_plugins()
118
119
            }

120
    @classmethod
Louis's avatar
Louis committed
121
    def from_yaml(cls, filename, form=None):
Louis's avatar
Louis committed
122
        """Factory to return a :class:`Renderer` from a yaml file."""
123
        with open(filename, 'r') as stream:
Louis's avatar
Louis committed
124
            return cls(yaml.safe_load(stream.read()), form=form)
125
126

    @classmethod
Louis's avatar
Louis committed
127
    def from_args(cls, packages=None, cells=None, users=None, form=None):
Louis's avatar
Louis committed
128
        """Factory to return a :class:`Renderer` from list of arguments."""
129
130
131
132
133
134
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
135
136
        if users is not None:
            for user in users:
137
138
139
140
141
142
143
                try:
                    packages.extend(iter_user_packages(user))
                except urllib.error.HTTPError as error:
                    LOGGER.error("Ignoring user '{}': {}".format(
                        user,
                        str(error),
                        ))
144
        packages = list(set(packages))
145
        config['packages'] = {pkg: {} for pkg in set(packages)}
146

Louis's avatar
Louis committed
147
148
149
150
151
152
        # Processing cells
        if cells is None:
            cells = []
        cells = list(itertools.chain(*[item.split(",") for item in cells]))
        if cells:
            config['cells'] = cells
153

Louis's avatar
Louis committed
154
        return cls(config, form=form)
155
156

    def __iter__(self):
Louis's avatar
Louis committed
157
        """Return iterator to `self.packages`."""
158
159
        yield from self.packages

Louis's avatar
Louis committed
160
    def values(self):
Louis's avatar
Louis committed
161
        """Yield `self.packages` values."""
Louis's avatar
Louis committed
162
163
164
165
166
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

167
168
    @staticmethod
    def _get_environment():
Louis's avatar
Louis committed
169
        templatedir = os.path.join(
170
171
172
173
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

Louis's avatar
Louis committed
174
        env = jinja2.Environment(loader=jinja2.FileSystemLoader(templatedir))
175
176
177
178
179
180
181
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
Louis's avatar
Louis committed
182
183
        """Return the HTML code."""
        return self._get_environment().get_template("index.html").render( # pylint: disable=no-member
184
            config=self.config,
Louis's avatar
Louis committed
185
            cells=self.cells,
186
            packages=self.packages,
Louis's avatar
Louis committed
187
            form=self.form,
188
            errors=self.errors,
189
            version=VERSION,
Louis's avatar
Louis committed
190
            )
191

192
    def log_error(self, message):
Louis's avatar
Louis committed
193
        """Log error, both to HTML page and console."""
194
195
196
        LOGGER.error(message)
        self.errors.add(message)

197

198
class Package:
Louis's avatar
Louis committed
199
200
    """Gather information about a single package."""
    # pylint: disable=too-few-public-methods
Louis's avatar
Louis committed
201

202
    def __init__(self, name):
203
        LOGGER.info("Retrieving information about pypi package '{}'...\n".format(name))
204
        try:
Louis's avatar
Louis committed
205
206
207
208
209
            self.rawdata = json.loads(
                requests.get(
                    "http://pypi.python.org/pypi/{}/json".format(name)
                    ).text
                )
210
211
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
212
        self.name = name
213
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
214
        self.releases = {}
215
216
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
217
                continue
218
219
220
221
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
222
            self.releases[date] = {}
223
224
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
225
226
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
227
228
229
230
231
232
233
234

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    @property
    def total_downloads(self):
Louis's avatar
Louis committed
235
        """Return this package's total downloads (all versions combined)."""
Louis's avatar
Louis committed
236
237
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']