__init__.py 6.79 KB
Newer Older
1
import colorsys
Louis's avatar
Louis committed
2
import datetime
3
import functools
4
5
import itertools
import jinja2
6
7
8
9
import json
import operator
import os
import pkg_resources
10
import random
11
import requests
Louis's avatar
Louis committed
12
import slumber
Louis's avatar
Louis committed
13
import sys
Louis's avatar
Louis committed
14
import urllib
15
import xmlrpc.client as xmlrpclib
16
import yaml
Louis's avatar
Louis committed
17

18
19
from pypimonitor.column import load_column_plugins

20
21
VERSION = "0.2.0-dev"

22
_client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
23

24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
class Counter:
    def __init__(self):
        self.value = 0

    def count(self):
        self.value += 1
        return self.value

def as_id(string):
    return "".join(list(filter(str.isalnum, string)))

def random_color(seed, saturation, value):
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
    return ("#%02x%02x%02x" % tuple([int(value * 255) for value in rgb]))

def iter_user_packages(user):
    for _, package in _client.user_packages(user):
        try:
            yield package
        except urllib.error.HTTPError as error:
            sys.stderr.write("Ignoring user '{}': {}.".format(
                user,
                str(error),
                ))

class Renderer:

Louis's avatar
Louis committed
54
    def __init__(self, config, form=None):
55
        # Processing config
56
        self.config = config
57
58
59
        if "columns" not in self.config:
            self.config['columns'] = list(set(itertools.chain(*[
                    package.keys()
60
                    for package
61
62
                    in self.config['packages'].values()
                ])))
63
64
65
66
67
68
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
            for column in self.config['columns']:
                if self.config['packages'][package].get(column, None) is None:
                    self.config['packages'][package][column] = {}
69
70
71
72
                # Set default values
                for key in self.config.get('default', {}):
                    if key not in self.config['packages'][package]:
                        self.config['packages'][package][key] = self.config['default'][key]
73
                self.config['packages'][package][column]['column'] = self.config['packages'][package][column].get('column', column)
74
75

        # Processing packages
Louis's avatar
Louis committed
76
        self.packages = {}
77
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
78
79
            try:
                self.packages[name] = Package(name)
80
81
            except Exception as error:
                sys.stderr.write("Ignoring package '{}': {}.\n".format(name, error))
Louis's avatar
Louis committed
82
                continue
83

Louis's avatar
Louis committed
84
85
86
87
88
89
        # Processing form
        self.form = form
        if self.form is not None:
            for key in ['pkg', 'user', 'col']:
                self.form[key] = ",".join(set(itertools.chain(*[item.split(",") for item in self.form.get(key, [])])))

90
91
92
93
94
95
96
        # Loading plugins
        self.columns = {
            plugin.keyword: plugin(self)
            for plugin
            in load_column_plugins()
            }

97
    @classmethod
Louis's avatar
Louis committed
98
    def from_yaml(cls, filename, form=None):
99
        with open(filename, 'r') as stream:
Louis's avatar
Louis committed
100
            return cls(yaml.safe_load(stream.read()), form=form)
101
102

    @classmethod
Louis's avatar
Louis committed
103
    def from_args(cls, packages=None, columns=None, users=None, form=None):
104
105
106
107
108
109
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
110
111
        if users is not None:
            for user in users:
112
113
                packages.extend(iter_user_packages(user))
        packages = list(set(packages))
114
        config['packages'] = {pkg: {} for pkg in set(packages)}
115

116
117
118
119
120
121
        # Processing columns
        if columns is None:
            columns = []
        columns = list(itertools.chain(*[item.split(",") for item in columns]))
        if columns:
            config['columns'] = columns
122

Louis's avatar
Louis committed
123
        return cls(config, form=form)
124
125
126
127

    def __iter__(self):
        yield from self.packages

Louis's avatar
Louis committed
128
129
130
131
132
133
    def values(self):
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
    @staticmethod
    def _get_environment():
        TEMPLATEDIR = os.path.join(
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

        env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATEDIR))
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
        env.filters['color'] = random_color
        return env

    def render(self):
        env = self._get_environment()
        template = env.get_template("index.html")
Louis's avatar
Louis committed
151
        return template.render(
152
153
            config=self.config,
            columns=self.columns,
154
            packages=self.packages,
Louis's avatar
Louis committed
155
            form=self.form,
Louis's avatar
Louis committed
156
            )
157
158


159
160
class Package:

Louis's avatar
Louis committed
161
    docs_api = {
162
        'readthedocs': slumber.API(base_url="http://readthedocs.io/api/v1/"),
Louis's avatar
Louis committed
163
164
        }

165
    def __init__(self, name):
Louis's avatar
Louis committed
166
        sys.stderr.write("Retrieving information about pypi package '{}'...\n".format(name))
167
168
169
170
        try:
            self.rawdata = json.loads(requests.get("http://pypi.python.org/pypi/{}/json".format(name)).text)
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
171
        self.name = name
172
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
173
        self.releases = {}
174
175
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
176
                continue
177
178
179
180
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
181
            self.releases[date] = {}
182
183
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
184
185
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
186
187
188
189
190
191
192
193
194
195
196
197
198

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    def iter_dates(self):
        yield from self.releases.keys()

    @property
    def total_downloads(self):
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']
199

Louis's avatar
Louis committed
200
201
202
203
204
205
206
207
208
    def get_documentation(self):
        # Readthedocs
        if self.docs_api['readthedocs'].project.get(slug=self.name.lower())['objects']:
            return (
                'readthedocs', 
                None,
                )

        # Pypi docs
209
        if self.info['docs_url'] is not None:
Louis's avatar
Louis committed
210
            return ("pypi", None)
Louis's avatar
Louis committed
211
212

        # No documentation
Louis's avatar
Louis committed
213
        return (None, None)