__init__.py 7.47 KB
Newer Older
Louis's avatar
Louis committed
1
2
"""Gather data about packages, and render them as an HTML page."""

3
import colorsys
Louis's avatar
Louis committed
4
import datetime
5
import functools
6
import itertools
7
import json
8
import logging
9
10
import operator
import os
11
import random
Louis's avatar
Louis committed
12
import sys
Louis's avatar
Louis committed
13
import urllib
14
import xmlrpc.client as xmlrpclib
15
import yaml
Louis's avatar
Louis committed
16

Louis's avatar
Louis committed
17
18
19
20
21
import jinja2
import pkg_resources
import requests
import slumber

22
23
from pypimonitor.column import load_column_plugins

24
VERSION = "0.2.0-dev"
25
LOGGER = logging.getLogger(__name__)
26

Louis's avatar
Louis committed
27
CLIENT = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
28

29
class Counter:
Louis's avatar
Louis committed
30
31
32
    """Each call to :meth:`Counter.count` increments a counter, and return its value."""
    # pylint: disable=too-few-public-methods

33
34
35
36
    def __init__(self):
        self.value = 0

    def count(self):
Louis's avatar
Louis committed
37
        """Increments counter, and return counter value."""
38
39
40
41
        self.value += 1
        return self.value

def random_color(seed, saturation, value):
Louis's avatar
Louis committed
42
    """Return a pseudo-random color."""
43
44
45
46
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
Louis's avatar
Louis committed
47
    return "#%02x%02x%02x" % tuple([int(value * 255) for value in rgb])
48
49

def iter_user_packages(user):
Louis's avatar
Louis committed
50
51
    """Iterate over packages owned by a given user."""
    for _, package in CLIENT.user_packages(user):
52
        yield package
53
54

class Renderer:
Louis's avatar
Louis committed
55
    """Get information about packages, and produce an HTML dashboard to monitor them."""
56

Louis's avatar
Louis committed
57
    def __init__(self, config, form=None):
58
59
        self.errors = set()

60
        # Processing config
61
        self.config = config
62
63
        if "columns" not in self.config:
            self.config['columns'] = list(set(itertools.chain(*[
Louis's avatar
Louis committed
64
65
66
                package.keys()
                for package
                in self.config['packages'].values()
67
                ])))
68
69
70
71
72
73
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
            for column in self.config['columns']:
                if self.config['packages'][package].get(column, None) is None:
                    self.config['packages'][package][column] = {}
74
75
76
77
                # Set default values
                for key in self.config.get('default', {}):
                    if key not in self.config['packages'][package]:
                        self.config['packages'][package][key] = self.config['default'][key]
Louis's avatar
Louis committed
78
79
                if 'column' not in self.config['packages'][package][column]:
                    self.config['packages'][package][column]['column'] = column
80
81

        # Processing packages
Louis's avatar
Louis committed
82
        self.packages = {}
83
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
84
85
            try:
                self.packages[name] = Package(name)
Louis's avatar
Louis committed
86
            except Exception as error: # pylint: disable=broad-except
87
                self.log_error("Ignoring package '{}': {}".format(name, error))
Louis's avatar
Louis committed
88
                continue
89

Louis's avatar
Louis committed
90
91
92
93
        # Processing form
        self.form = form
        if self.form is not None:
            for key in ['pkg', 'user', 'col']:
Louis's avatar
Louis committed
94
95
96
                self.form[key] = ",".join(set(
                    itertools.chain(*[item.split(",") for item in self.form.get(key, [])])
                ))
Louis's avatar
Louis committed
97

98
99
100
101
102
103
104
        # Loading plugins
        self.columns = {
            plugin.keyword: plugin(self)
            for plugin
            in load_column_plugins()
            }

105
    @classmethod
Louis's avatar
Louis committed
106
    def from_yaml(cls, filename, form=None):
Louis's avatar
Louis committed
107
        """Factory to return a :class:`Renderer` from a yaml file."""
108
        with open(filename, 'r') as stream:
Louis's avatar
Louis committed
109
            return cls(yaml.safe_load(stream.read()), form=form)
110
111

    @classmethod
Louis's avatar
Louis committed
112
    def from_args(cls, packages=None, columns=None, users=None, form=None):
Louis's avatar
Louis committed
113
        """Factory to return a :class:`Renderer` from list of arguments."""
114
115
116
117
118
119
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
120
121
        if users is not None:
            for user in users:
122
123
124
125
126
127
128
                try:
                    packages.extend(iter_user_packages(user))
                except urllib.error.HTTPError as error:
                    LOGGER.error("Ignoring user '{}': {}".format(
                        user,
                        str(error),
                        ))
129
        packages = list(set(packages))
130
        config['packages'] = {pkg: {} for pkg in set(packages)}
131

132
133
134
135
136
137
        # Processing columns
        if columns is None:
            columns = []
        columns = list(itertools.chain(*[item.split(",") for item in columns]))
        if columns:
            config['columns'] = columns
138

Louis's avatar
Louis committed
139
        return cls(config, form=form)
140
141

    def __iter__(self):
Louis's avatar
Louis committed
142
        """Return iterator to `self.packages`."""
143
144
        yield from self.packages

Louis's avatar
Louis committed
145
    def values(self):
Louis's avatar
Louis committed
146
        """Yield `self.packages` values."""
Louis's avatar
Louis committed
147
148
149
150
151
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

152
153
    @staticmethod
    def _get_environment():
Louis's avatar
Louis committed
154
        templatedir = os.path.join(
155
156
157
158
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

Louis's avatar
Louis committed
159
        env = jinja2.Environment(loader=jinja2.FileSystemLoader(templatedir))
160
161
162
163
164
165
166
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
Louis's avatar
Louis committed
167
168
        """Return the HTML code."""
        return self._get_environment().get_template("index.html").render( # pylint: disable=no-member
169
170
            config=self.config,
            columns=self.columns,
171
            packages=self.packages,
Louis's avatar
Louis committed
172
            form=self.form,
173
            errors=self.errors,
Louis's avatar
Louis committed
174
            )
175

176
    def log_error(self, message):
Louis's avatar
Louis committed
177
        """Log error, both to HTML page and console."""
178
179
180
        LOGGER.error(message)
        self.errors.add(message)

181

182
class Package:
Louis's avatar
Louis committed
183
184
    """Gather information about a single package."""
    # pylint: disable=too-few-public-methods
Louis's avatar
Louis committed
185

186
    def __init__(self, name):
187
        LOGGER.info("Retrieving information about pypi package '{}'...\n".format(name))
188
        try:
Louis's avatar
Louis committed
189
190
191
192
193
            self.rawdata = json.loads(
                requests.get(
                    "http://pypi.python.org/pypi/{}/json".format(name)
                    ).text
                )
194
195
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
196
        self.name = name
197
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
198
        self.releases = {}
199
200
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
201
                continue
202
203
204
205
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
206
            self.releases[date] = {}
207
208
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
209
210
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
211
212
213
214
215
216
217
218

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    @property
    def total_downloads(self):
Louis's avatar
Louis committed
219
        """Return this package's total downloads (all versions combined)."""
Louis's avatar
Louis committed
220
221
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']