__init__.py 6.26 KB
Newer Older
Louis's avatar
Louis committed
1
2
3
#!/usr/bin/env python3

import collections
4
import colorsys
Louis's avatar
Louis committed
5
import datetime
6
import functools
7
8
import itertools
import jinja2
9
10
11
12
import json
import operator
import os
import pkg_resources
13
import random
14
import requests
Louis's avatar
Louis committed
15
import slumber
Louis's avatar
Louis committed
16
import sys
Louis's avatar
Louis committed
17
import urllib
18
import xmlrpc.client as xmlrpclib
19
import yaml
Louis's avatar
Louis committed
20

21
22
from pypimonitor.column import load_column_plugins

23
24
VERSION = "0.2.0-dev"

25
_client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class Counter:
    def __init__(self):
        self.value = 0

    def count(self):
        self.value += 1
        return self.value

def as_id(string):
    return "".join(list(filter(str.isalnum, string)))

def random_color(seed, saturation, value):
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
    return ("#%02x%02x%02x" % tuple([int(value * 255) for value in rgb]))

def iter_user_packages(user):
    for _, package in _client.user_packages(user):
        try:
            yield package
        except urllib.error.HTTPError as error:
            sys.stderr.write("Ignoring user '{}': {}.".format(
                user,
                str(error),
                ))

class Renderer:

    def __init__(self, config):
58
        # Processing config
59
        self.config = config
60
61
62
        if "columns" not in self.config:
            self.config['columns'] = list(set(itertools.chain(*[
                    package.keys()
63
                    for package
64
65
                    in self.config['packages'].values()
                ])))
66
67
68
69
70
71
72
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
            for column in self.config['columns']:
                if self.config['packages'][package].get(column, None) is None:
                    self.config['packages'][package][column] = {}
                self.config['packages'][package][column]['column'] = self.config['packages'][package][column].get('column', column)
73
74

        # Processing packages
Louis's avatar
Louis committed
75
        self.packages = {}
76
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
77
78
            try:
                self.packages[name] = Package(name)
79
80
            except Exception as error:
                sys.stderr.write("Ignoring package '{}': {}.\n".format(name, error))
Louis's avatar
Louis committed
81
                continue
82

83
84
85
86
87
88
89
        # Loading plugins
        self.columns = {
            plugin.keyword: plugin(self)
            for plugin
            in load_column_plugins()
            }

90
91
92
93
94
95
    @classmethod
    def from_yaml(cls, filename):
        with open(filename, 'r') as stream:
            return cls(yaml.safe_load(stream.read()))

    @classmethod
96
    def from_args(cls, packages=None, columns=None, users=None):
97
98
99
100
101
102
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
103
104
        if users is not None:
            for user in users:
105
106
                packages.extend(iter_user_packages(user))
        packages = list(set(packages))
107
        config['packages'] = {pkg: {} for pkg in set(packages)}
108

109
110
111
112
113
114
        # Processing columns
        if columns is None:
            columns = []
        columns = list(itertools.chain(*[item.split(",") for item in columns]))
        if columns:
            config['columns'] = columns
115
116

        return cls(config)
117
118
119
120

    def __iter__(self):
        yield from self.packages

Louis's avatar
Louis committed
121
122
123
124
125
126
    def values(self):
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
    @staticmethod
    def _get_environment():
        TEMPLATEDIR = os.path.join(
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

        env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATEDIR))
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
        env = self._get_environment()
        template = env.get_template("index.html")
Louis's avatar
Louis committed
144
        return template.render(
145
146
            config=self.config,
            columns=self.columns,
147
            packages=self.packages,
Louis's avatar
Louis committed
148
            )
149
150


151
152
class Package:

Louis's avatar
Louis committed
153
    docs_api = {
154
        'readthedocs': slumber.API(base_url="http://readthedocs.io/api/v1/"),
Louis's avatar
Louis committed
155
156
        }

157
    def __init__(self, name):
Louis's avatar
Louis committed
158
        sys.stderr.write("Retrieving information about pypi package '{}'...\n".format(name))
159
160
161
162
        try:
            self.rawdata = json.loads(requests.get("http://pypi.python.org/pypi/{}/json".format(name)).text)
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
163
        self.name = name
164
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
165
        self.releases = {}
166
167
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
168
                continue
169
170
171
172
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
173
            self.releases[date] = {}
174
175
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
176
177
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
178
179
180
181
182
183
184
185
186
187
188
189
190

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    def iter_dates(self):
        yield from self.releases.keys()

    @property
    def total_downloads(self):
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']
191

Louis's avatar
Louis committed
192
193
194
195
196
197
198
199
200
    def get_documentation(self):
        # Readthedocs
        if self.docs_api['readthedocs'].project.get(slug=self.name.lower())['objects']:
            return (
                'readthedocs', 
                None,
                )

        # Pypi docs
201
        if self.info['docs_url'] is not None:
Louis's avatar
Louis committed
202
            return ("pypi", None)
Louis's avatar
Louis committed
203
204

        # No documentation
Louis's avatar
Louis committed
205
        return (None, None)