__init__.py 7.03 KB
Newer Older
1
import colorsys
Louis's avatar
Louis committed
2
import datetime
3
import functools
4
5
import itertools
import jinja2
6
import json
7
import logging
8
9
10
import operator
import os
import pkg_resources
11
import random
12
import requests
Louis's avatar
Louis committed
13
import slumber
Louis's avatar
Louis committed
14
import sys
Louis's avatar
Louis committed
15
import urllib
16
import xmlrpc.client as xmlrpclib
17
import yaml
Louis's avatar
Louis committed
18

19
20
from pypimonitor.column import load_column_plugins

21
VERSION = "0.2.0-dev"
22
LOGGER = logging.getLogger(__name__)
23

24
_client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
25

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
class Counter:
    def __init__(self):
        self.value = 0

    def count(self):
        self.value += 1
        return self.value

def as_id(string):
    return "".join(list(filter(str.isalnum, string)))

def random_color(seed, saturation, value):
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
    return ("#%02x%02x%02x" % tuple([int(value * 255) for value in rgb]))

def iter_user_packages(user):
    for _, package in _client.user_packages(user):
46
        yield package
47
48
49

class Renderer:

Louis's avatar
Louis committed
50
    def __init__(self, config, form=None):
51
52
        self.errors = set()

53
        # Processing config
54
        self.config = config
55
56
57
        if "columns" not in self.config:
            self.config['columns'] = list(set(itertools.chain(*[
                    package.keys()
58
                    for package
59
60
                    in self.config['packages'].values()
                ])))
61
62
63
64
65
66
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
            for column in self.config['columns']:
                if self.config['packages'][package].get(column, None) is None:
                    self.config['packages'][package][column] = {}
67
68
69
70
                # Set default values
                for key in self.config.get('default', {}):
                    if key not in self.config['packages'][package]:
                        self.config['packages'][package][key] = self.config['default'][key]
71
                self.config['packages'][package][column]['column'] = self.config['packages'][package][column].get('column', column)
72
73

        # Processing packages
Louis's avatar
Louis committed
74
        self.packages = {}
75
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
76
77
            try:
                self.packages[name] = Package(name)
78
            except Exception as error:
79
                self.log_error("Ignoring package '{}': {}".format(name, error))
Louis's avatar
Louis committed
80
                continue
81

Louis's avatar
Louis committed
82
83
84
85
86
87
        # Processing form
        self.form = form
        if self.form is not None:
            for key in ['pkg', 'user', 'col']:
                self.form[key] = ",".join(set(itertools.chain(*[item.split(",") for item in self.form.get(key, [])])))

88
89
90
91
92
93
94
        # Loading plugins
        self.columns = {
            plugin.keyword: plugin(self)
            for plugin
            in load_column_plugins()
            }

95
    @classmethod
Louis's avatar
Louis committed
96
    def from_yaml(cls, filename, form=None):
97
        with open(filename, 'r') as stream:
Louis's avatar
Louis committed
98
            return cls(yaml.safe_load(stream.read()), form=form)
99
100

    @classmethod
Louis's avatar
Louis committed
101
    def from_args(cls, packages=None, columns=None, users=None, form=None):
102
103
104
105
106
107
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
108
109
        if users is not None:
            for user in users:
110
111
112
113
114
115
116
                try:
                    packages.extend(iter_user_packages(user))
                except urllib.error.HTTPError as error:
                    LOGGER.error("Ignoring user '{}': {}".format(
                        user,
                        str(error),
                        ))
117
        packages = list(set(packages))
118
        config['packages'] = {pkg: {} for pkg in set(packages)}
119

120
121
122
123
124
125
        # Processing columns
        if columns is None:
            columns = []
        columns = list(itertools.chain(*[item.split(",") for item in columns]))
        if columns:
            config['columns'] = columns
126

Louis's avatar
Louis committed
127
        return cls(config, form=form)
128
129
130
131

    def __iter__(self):
        yield from self.packages

Louis's avatar
Louis committed
132
133
134
135
136
137
    def values(self):
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    @staticmethod
    def _get_environment():
        TEMPLATEDIR = os.path.join(
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

        env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATEDIR))
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
        env = self._get_environment()
        template = env.get_template("index.html")
Louis's avatar
Louis committed
155
        return template.render(
156
157
            config=self.config,
            columns=self.columns,
158
            packages=self.packages,
Louis's avatar
Louis committed
159
            form=self.form,
160
            errors=self.errors,
Louis's avatar
Louis committed
161
            )
162

163
164
165
166
    def log_error(self, message):
        LOGGER.error(message)
        self.errors.add(message)

167

168
169
class Package:

Louis's avatar
Louis committed
170
    docs_api = {
171
        'readthedocs': slumber.API(base_url="http://readthedocs.io/api/v1/"),
Louis's avatar
Louis committed
172
173
        }

174
    def __init__(self, name):
175
        LOGGER.info("Retrieving information about pypi package '{}'...\n".format(name))
176
177
178
179
        try:
            self.rawdata = json.loads(requests.get("http://pypi.python.org/pypi/{}/json".format(name)).text)
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
180
        self.name = name
181
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
182
        self.releases = {}
183
184
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
185
                continue
186
187
188
189
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
190
            self.releases[date] = {}
191
192
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
193
194
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
195
196
197
198
199
200
201
202
203
204
205
206
207

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    def iter_dates(self):
        yield from self.releases.keys()

    @property
    def total_downloads(self):
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']
208

Louis's avatar
Louis committed
209
210
211
212
213
214
215
216
217
    def get_documentation(self):
        # Readthedocs
        if self.docs_api['readthedocs'].project.get(slug=self.name.lower())['objects']:
            return (
                'readthedocs', 
                None,
                )

        # Pypi docs
218
        if self.info['docs_url'] is not None:
Louis's avatar
Louis committed
219
            return ("pypi", None)
Louis's avatar
Louis committed
220
221

        # No documentation
Louis's avatar
Louis committed
222
        return (None, None)