__init__.py 8.17 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# pypimonitor — An HTML dashboard to monitor your python packages
# Copyright (C) 2016 Louis Paternault
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

Louis's avatar
Louis committed
17
18
"""Gather data about packages, and render them as an HTML page."""

19
import colorsys
Louis's avatar
Louis committed
20
import datetime
21
import functools
22
import itertools
23
import json
24
import logging
25
26
import operator
import os
27
import random
Louis's avatar
Louis committed
28
import sys
Louis's avatar
Louis committed
29
import urllib
30
import xmlrpc.client as xmlrpclib
31
import yaml
Louis's avatar
Louis committed
32

Louis's avatar
Louis committed
33
34
35
36
37
import jinja2
import pkg_resources
import requests
import slumber

Louis's avatar
Louis committed
38
from pypimonitor.cell import load_cell_plugins
39

40
VERSION = "0.2.0-dev"
41
LOGGER = logging.getLogger(__name__)
42

Louis's avatar
Louis committed
43
CLIENT = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
44

45
class Counter:
Louis's avatar
Louis committed
46
47
48
    """Each call to :meth:`Counter.count` increments a counter, and return its value."""
    # pylint: disable=too-few-public-methods

49
50
51
52
    def __init__(self):
        self.value = 0

    def count(self):
Louis's avatar
Louis committed
53
        """Increments counter, and return counter value."""
54
55
56
57
        self.value += 1
        return self.value

def random_color(seed, saturation, value):
Louis's avatar
Louis committed
58
    """Return a pseudo-random color."""
59
60
61
62
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
Louis's avatar
Louis committed
63
    return "#%02x%02x%02x" % tuple([int(value * 255) for value in rgb])
64
65

def iter_user_packages(user):
Louis's avatar
Louis committed
66
67
    """Iterate over packages owned by a given user."""
    for _, package in CLIENT.user_packages(user):
68
        yield package
69
70

class Renderer:
Louis's avatar
Louis committed
71
    """Get information about packages, and produce an HTML dashboard to monitor them."""
72

Louis's avatar
Louis committed
73
    def __init__(self, config, form=None):
74
75
        self.errors = set()

76
        # Processing config
77
        self.config = config
Louis's avatar
Louis committed
78
79
        if "cells" not in self.config:
            self.config['cells'] = list(set(itertools.chain(*[
Louis's avatar
Louis committed
80
81
82
                package.keys()
                for package
                in self.config['packages'].values()
83
                ])))
84
85
86
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
Louis's avatar
Louis committed
87
88
89
            for cell in self.config['cells']:
                if self.config['packages'][package].get(cell, None) is None:
                    self.config['packages'][package][cell] = {}
90
91
92
93
                # Set default values
                for key in self.config.get('default', {}):
                    if key not in self.config['packages'][package]:
                        self.config['packages'][package][key] = self.config['default'][key]
Louis's avatar
Louis committed
94
95
                if 'cell' not in self.config['packages'][package][cell]:
                    self.config['packages'][package][cell]['cell'] = cell
96
97

        # Processing packages
Louis's avatar
Louis committed
98
        self.packages = {}
99
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
100
101
            try:
                self.packages[name] = Package(name)
Louis's avatar
Louis committed
102
            except Exception as error: # pylint: disable=broad-except
103
                self.log_error("Ignoring package '{}': {}".format(name, error))
Louis's avatar
Louis committed
104
                continue
105

Louis's avatar
Louis committed
106
107
108
109
        # Processing form
        self.form = form
        if self.form is not None:
            for key in ['pkg', 'user', 'col']:
Louis's avatar
Louis committed
110
111
112
                self.form[key] = ",".join(set(
                    itertools.chain(*[item.split(",") for item in self.form.get(key, [])])
                ))
Louis's avatar
Louis committed
113

114
        # Loading plugins
Louis's avatar
Louis committed
115
        self.cells = {
116
117
            plugin.keyword: plugin(self)
            for plugin
Louis's avatar
Louis committed
118
            in load_cell_plugins()
119
120
            }

121
    @classmethod
Louis's avatar
Louis committed
122
    def from_yaml(cls, filename, form=None):
Louis's avatar
Louis committed
123
        """Factory to return a :class:`Renderer` from a yaml file."""
124
        with open(filename, 'r') as stream:
Louis's avatar
Louis committed
125
            return cls(yaml.safe_load(stream.read()), form=form)
126
127

    @classmethod
Louis's avatar
Louis committed
128
    def from_args(cls, packages=None, cells=None, users=None, form=None):
Louis's avatar
Louis committed
129
        """Factory to return a :class:`Renderer` from list of arguments."""
130
131
132
133
134
135
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
136
137
        if users is not None:
            for user in users:
138
139
140
141
142
143
144
                try:
                    packages.extend(iter_user_packages(user))
                except urllib.error.HTTPError as error:
                    LOGGER.error("Ignoring user '{}': {}".format(
                        user,
                        str(error),
                        ))
145
        packages = list(set(packages))
146
        config['packages'] = {pkg: {} for pkg in set(packages)}
147

Louis's avatar
Louis committed
148
149
150
151
152
153
        # Processing cells
        if cells is None:
            cells = []
        cells = list(itertools.chain(*[item.split(",") for item in cells]))
        if cells:
            config['cells'] = cells
154

Louis's avatar
Louis committed
155
        return cls(config, form=form)
156
157

    def __iter__(self):
Louis's avatar
Louis committed
158
        """Return iterator to `self.packages`."""
159
160
        yield from self.packages

Louis's avatar
Louis committed
161
    def values(self):
Louis's avatar
Louis committed
162
        """Yield `self.packages` values."""
Louis's avatar
Louis committed
163
164
165
166
167
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

168
169
    @staticmethod
    def _get_environment():
Louis's avatar
Louis committed
170
        templatedir = os.path.join(
171
172
173
174
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

Louis's avatar
Louis committed
175
        env = jinja2.Environment(loader=jinja2.FileSystemLoader(templatedir))
176
177
178
179
180
181
182
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
Louis's avatar
Louis committed
183
184
        """Return the HTML code."""
        return self._get_environment().get_template("index.html").render( # pylint: disable=no-member
185
            config=self.config,
Louis's avatar
Louis committed
186
            cells=self.cells,
187
            packages=self.packages,
Louis's avatar
Louis committed
188
            form=self.form,
189
            errors=self.errors,
Louis's avatar
Louis committed
190
            )
191

192
    def log_error(self, message):
Louis's avatar
Louis committed
193
        """Log error, both to HTML page and console."""
194
195
196
        LOGGER.error(message)
        self.errors.add(message)

197

198
class Package:
Louis's avatar
Louis committed
199
200
    """Gather information about a single package."""
    # pylint: disable=too-few-public-methods
Louis's avatar
Louis committed
201

202
    def __init__(self, name):
203
        LOGGER.info("Retrieving information about pypi package '{}'...\n".format(name))
204
        try:
Louis's avatar
Louis committed
205
206
207
208
209
            self.rawdata = json.loads(
                requests.get(
                    "http://pypi.python.org/pypi/{}/json".format(name)
                    ).text
                )
210
211
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
212
        self.name = name
213
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
214
        self.releases = {}
215
216
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
217
                continue
218
219
220
221
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
222
            self.releases[date] = {}
223
224
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
225
226
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
227
228
229
230
231
232
233
234

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    @property
    def total_downloads(self):
Louis's avatar
Louis committed
235
        """Return this package's total downloads (all versions combined)."""
Louis's avatar
Louis committed
236
237
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']