__init__.py 8.26 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# pypimonitor 鈥 An HTML dashboard to monitor your python packages
# Copyright (C) 2016 Louis Paternault
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

Louis's avatar
Louis committed
17
18
"""Gather data about packages, and render them as an HTML page."""

19
import colorsys
Louis's avatar
Louis committed
20
import datetime
21
import functools
22
import itertools
23
import json
24
import logging
25
26
import operator
import os
27
import random
Louis's avatar
Louis committed
28
import sys
Louis's avatar
Louis committed
29
import urllib
30
import xmlrpc.client as xmlrpclib
31
import yaml
Louis's avatar
Louis committed
32

Louis's avatar
Louis committed
33
34
35
36
import jinja2
import pkg_resources
import requests

Louis's avatar
Louis committed
37
from pypimonitor.cell import load_cell_plugins
38
from pypimonitor.cell.base import Error
39

Louis's avatar
v0.2.0    
Louis committed
40
VERSION = "0.2.0"
41
LOGGER = logging.getLogger(__name__)
42

Louis's avatar
Louis committed
43
CLIENT = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
44

45
class Counter:
Louis's avatar
Louis committed
46
47
48
    """Each call to :meth:`Counter.count` increments a counter, and return its value."""
    # pylint: disable=too-few-public-methods

49
50
51
52
    def __init__(self):
        self.value = 0

    def count(self):
Louis's avatar
Louis committed
53
        """Increments counter, and return counter value."""
54
55
56
57
        self.value += 1
        return self.value

def random_color(seed, saturation, value):
Louis's avatar
Louis committed
58
    """Return a pseudo-random color."""
59
60
61
62
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
Louis's avatar
Louis committed
63
    return "#%02x%02x%02x" % tuple([int(value * 255) for value in rgb])
64
65

def iter_user_packages(user):
Louis's avatar
Louis committed
66
67
    """Iterate over packages owned by a given user."""
    for _, package in CLIENT.user_packages(user):
68
        yield package
69
70

class Renderer:
Louis's avatar
Louis committed
71
    """Get information about packages, and produce an HTML dashboard to monitor them."""
72

Louis's avatar
Louis committed
73
    def __init__(self, config, form=None):
74
75
        self.errors = set()

76
        # Processing config
77
        self.config = config
Louis's avatar
Louis committed
78
79
        if "cells" not in self.config:
            self.config['cells'] = list(set(itertools.chain(*[
Louis's avatar
Louis committed
80
81
82
                package.keys()
                for package
                in self.config['packages'].values()
83
                ])))
84
85
86
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
Louis's avatar
Louis committed
87
88
89
            for cell in self.config['cells']:
                if self.config['packages'][package].get(cell, None) is None:
                    self.config['packages'][package][cell] = {}
90
91
92
93
                # Set default values
                for key in self.config.get('default', {}):
                    if key not in self.config['packages'][package]:
                        self.config['packages'][package][key] = self.config['default'][key]
Louis's avatar
Louis committed
94
95
                if 'cell' not in self.config['packages'][package][cell]:
                    self.config['packages'][package][cell]['cell'] = cell
96
97

        # Processing packages
Louis's avatar
Louis committed
98
        self.packages = {}
99
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
100
101
            try:
                self.packages[name] = Package(name)
Louis's avatar
Louis committed
102
            except Exception as error: # pylint: disable=broad-except
103
                self.log_error("Ignoring package '{}': {}".format(name, error))
Louis's avatar
Louis committed
104
                continue
105

Louis's avatar
Louis committed
106
107
108
109
        # Processing form
        self.form = form
        if self.form is not None:
            for key in ['pkg', 'user', 'col']:
Louis's avatar
Louis committed
110
111
112
                self.form[key] = ",".join(set(
                    itertools.chain(*[item.split(",") for item in self.form.get(key, [])])
                ))
Louis's avatar
Louis committed
113

114
        # Loading plugins
Louis's avatar
Louis committed
115
        self.cells = {
116
117
            plugin.keyword: plugin(self)
            for plugin
Louis's avatar
Louis committed
118
            in load_cell_plugins()
119
            }
120
        self.cells['error'] = Error(self)
121

122
    @classmethod
Louis's avatar
Louis committed
123
    def from_yaml(cls, filename, form=None):
Louis's avatar
Louis committed
124
        """Factory to return a :class:`Renderer` from a yaml file."""
125
        with open(filename, 'r') as stream:
Louis's avatar
Louis committed
126
            return cls(yaml.safe_load(stream.read()), form=form)
127
128

    @classmethod
Louis's avatar
Louis committed
129
    def from_args(cls, packages=None, cells=None, users=None, form=None):
Louis's avatar
Louis committed
130
        """Factory to return a :class:`Renderer` from list of arguments."""
131
132
133
134
135
136
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
137
138
        if users is not None:
            for user in users:
139
140
141
142
143
144
145
                try:
                    packages.extend(iter_user_packages(user))
                except urllib.error.HTTPError as error:
                    LOGGER.error("Ignoring user '{}': {}".format(
                        user,
                        str(error),
                        ))
146
        packages = list(set(packages))
147
        config['packages'] = {pkg: {} for pkg in set(packages)}
148

Louis's avatar
Louis committed
149
150
151
152
153
154
        # Processing cells
        if cells is None:
            cells = []
        cells = list(itertools.chain(*[item.split(",") for item in cells]))
        if cells:
            config['cells'] = cells
155

Louis's avatar
Louis committed
156
        return cls(config, form=form)
157
158

    def __iter__(self):
Louis's avatar
Louis committed
159
        """Return iterator to `self.packages`."""
160
161
        yield from self.packages

Louis's avatar
Louis committed
162
    def values(self):
Louis's avatar
Louis committed
163
        """Yield `self.packages` values."""
Louis's avatar
Louis committed
164
165
166
167
168
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

169
170
    @staticmethod
    def _get_environment():
Louis's avatar
Louis committed
171
        templatedir = os.path.join(
172
173
174
175
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

Louis's avatar
Louis committed
176
        env = jinja2.Environment(loader=jinja2.FileSystemLoader(templatedir))
177
178
179
180
181
182
183
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
Louis's avatar
Louis committed
184
185
        """Return the HTML code."""
        return self._get_environment().get_template("index.html").render( # pylint: disable=no-member
186
            config=self.config,
Louis's avatar
Louis committed
187
            cells=self.cells,
188
            packages=self.packages,
Louis's avatar
Louis committed
189
            form=self.form,
190
            errors=self.errors,
191
            version=VERSION,
Louis's avatar
Louis committed
192
            )
193

194
    def log_error(self, message):
Louis's avatar
Louis committed
195
        """Log error, both to HTML page and console."""
196
197
198
        LOGGER.error(message)
        self.errors.add(message)

199

200
class Package:
Louis's avatar
Louis committed
201
202
    """Gather information about a single package."""
    # pylint: disable=too-few-public-methods
Louis's avatar
Louis committed
203

204
    def __init__(self, name):
205
        LOGGER.info("Retrieving information about pypi package '{}'...\n".format(name))
206
        try:
Louis's avatar
Louis committed
207
208
209
210
211
            self.rawdata = json.loads(
                requests.get(
                    "http://pypi.python.org/pypi/{}/json".format(name)
                    ).text
                )
212
213
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
214
        self.name = name
215
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
216
        self.releases = {}
217
218
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
219
                continue
220
221
222
223
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
224
            self.releases[date] = {}
225
226
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
227
228
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
229
230
231
232
233
234
235
236

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    @property
    def total_downloads(self):
Louis's avatar
Louis committed
237
        """Return this package's total downloads (all versions combined)."""
Louis's avatar
Louis committed
238
239
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']