__init__.py 6.31 KB
Newer Older
Louis's avatar
Louis committed
1
2
3
#!/usr/bin/env python3

import collections
4
import colorsys
Louis's avatar
Louis committed
5
import datetime
6
import functools
7
8
import itertools
import jinja2
9
10
11
12
import json
import operator
import os
import pkg_resources
13
import random
14
import requests
Louis's avatar
Louis committed
15
import slumber
Louis's avatar
Louis committed
16
import sys
Louis's avatar
Louis committed
17
import urllib
18
import xmlrpc.client as xmlrpclib
19
import yaml
Louis's avatar
Louis committed
20

21
22
from pypimonitor.column import load_column_plugins

23
24
VERSION = "0.2.0-dev"

25
_client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class Counter:
    def __init__(self):
        self.value = 0

    def count(self):
        self.value += 1
        return self.value

def as_id(string):
    return "".join(list(filter(str.isalnum, string)))

def random_color(seed, saturation, value):
    old_seed = random.random()
    random.seed(seed)
    rgb = colorsys.hsv_to_rgb(random.random(), saturation, value)
    random.seed(old_seed)
    return ("#%02x%02x%02x" % tuple([int(value * 255) for value in rgb]))

def iter_user_packages(user):
    for _, package in _client.user_packages(user):
        try:
            yield package
        except urllib.error.HTTPError as error:
            sys.stderr.write("Ignoring user '{}': {}.".format(
                user,
                str(error),
                ))

class Renderer:

    def __init__(self, config):
58
        # Processing config
59
        self.config = config
60
61
62
        if "columns" not in self.config:
            self.config['columns'] = list(set(itertools.chain(*[
                    package.keys()
63
                    for package
64
65
                    in self.config['packages'].values()
                ])))
66
67
68
69
70
71
72
        for package in self.config['packages']:
            if self.config['packages'][package] is None:
                self.config['packages'][package] = {}
            for column in self.config['columns']:
                if self.config['packages'][package].get(column, None) is None:
                    self.config['packages'][package][column] = {}
                self.config['packages'][package][column]['column'] = self.config['packages'][package][column].get('column', column)
73
74

        # Processing packages
Louis's avatar
Louis committed
75
        self.packages = {}
76
        for name in sorted(config['packages']):
Louis's avatar
Louis committed
77
78
            try:
                self.packages[name] = Package(name)
79
80
            except Exception as error:
                sys.stderr.write("Ignoring package '{}': {}.\n".format(name, error))
Louis's avatar
Louis committed
81
                continue
82

83
84
85
86
87
88
89
        # Loading plugins
        self.columns = {
            plugin.keyword: plugin(self)
            for plugin
            in load_column_plugins()
            }

90
91
92
93
94
95
    @classmethod
    def from_yaml(cls, filename):
        with open(filename, 'r') as stream:
            return cls(yaml.safe_load(stream.read()))

    @classmethod
96
    def from_args(cls, packages=None, columns=None, users=None):
97
98
99
100
101
102
        config = {}

        # Processing packages
        if packages is None:
            packages = []
        packages = list(itertools.chain(*[item.split(",") for item in packages]))
Louis's avatar
Louis committed
103
104
        if users is not None:
            for user in users:
105
106
                packages.extend(iter_user_packages(user))
        packages = list(set(packages))
107
        config['packages'] = {pkg: {} for pkg in set(packages)}
108

109
110
111
112
113
114
        # Processing columns
        if columns is None:
            columns = []
        columns = list(itertools.chain(*[item.split(",") for item in columns]))
        if columns:
            config['columns'] = columns
115
116

        return cls(config)
117
118
119
120

    def __iter__(self):
        yield from self.packages

Louis's avatar
Louis committed
121
122
123
124
125
126
    def values(self):
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

127
128
129
130
131
132
133
134
135
136
137
    @staticmethod
    def _get_environment():
        TEMPLATEDIR = os.path.join(
            pkg_resources.resource_filename(__name__, 'data'),
            'templates',
            )

        env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATEDIR))
        env.globals['counter'] = Counter().count
        env.globals['now'] = datetime.datetime.now()
        env.filters['datetime'] = operator.methodcaller('strftime', "%Y-%m-%d %H:%M:%S")
Louis's avatar
TODOs    
Louis committed
138
        env.filters['as_id'] = as_id # TODO: Remove?
139
140
141
142
143
144
        env.filters['color'] = random_color
        return env

    def render(self):
        env = self._get_environment()
        template = env.get_template("index.html")
Louis's avatar
Louis committed
145
        return template.render(
146
147
            config=self.config,
            columns=self.columns,
148
            packages=self.packages,
Louis's avatar
Louis committed
149
            )
150
151


152
153
class Package:

Louis's avatar
Louis committed
154
    docs_api = {
155
        'readthedocs': slumber.API(base_url="http://readthedocs.io/api/v1/"),
Louis's avatar
Louis committed
156
157
        }

158
    def __init__(self, name):
Louis's avatar
Louis committed
159
        sys.stderr.write("Retrieving information about pypi package '{}'...\n".format(name))
160
161
162
163
        try:
            self.rawdata = json.loads(requests.get("http://pypi.python.org/pypi/{}/json".format(name)).text)
        except Exception as error:
            raise Exception("Error while getting data for package '{}': {}.".format(name, error))
164
        self.name = name
165
        self.info = self.rawdata["info"]
Louis's avatar
Louis committed
166
        self.releases = {}
167
168
        for version, files in self.rawdata['releases'].items():
            if not files:
Louis's avatar
Louis committed
169
                continue
170
171
172
173
            date = datetime.datetime.strptime(
                min((url['upload_time'] for url in files)),
                "%Y-%m-%dT%H:%M:%S",
                )
Louis's avatar
Louis committed
174
            self.releases[date] = {}
175
176
            self.releases[date]['downloads'] = sum((url['downloads'] for url in files))
            self.releases[date]['version'] = version
Louis's avatar
Louis committed
177
178
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
179
180
181
182
183
184
185
186
187
188
189
190
191

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

    def iter_dates(self):
        yield from self.releases.keys()

    @property
    def total_downloads(self):
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']
192

Louis's avatar
Louis committed
193
194
195
196
197
198
199
200
201
    def get_documentation(self):
        # Readthedocs
        if self.docs_api['readthedocs'].project.get(slug=self.name.lower())['objects']:
            return (
                'readthedocs', 
                None,
                )

        # Pypi docs
202
        if self.info['docs_url'] is not None:
Louis's avatar
Louis committed
203
            return ("pypi", None)
Louis's avatar
Louis committed
204
205

        # No documentation
Louis's avatar
Louis committed
206
        return (None, None)