__init__.py 4.42 KB
Newer Older
Louis's avatar
Louis committed
1
2
#!/usr/bin/env python3

3
from pkgtools import pypi
Louis's avatar
Louis committed
4
5
import collections
import datetime
6
7
8
import itertools
import jinja2
import pkg_resources
9
import random
Louis's avatar
Louis committed
10
import slumber
Louis's avatar
Louis committed
11
import sys
Louis's avatar
Louis committed
12
import urllib
Louis's avatar
Louis committed
13
14
15
16
17
18

try:
    import xmlrpclib
except ImportError:
    import xmlrpc.client as xmlrpclib

19
20
21
22
_client = xmlrpclib.ServerProxy("http://pypi.python.org/pypi")

class PackageList:
    def __init__(self, user=None, pkg=None):
Louis's avatar
Louis committed
23
        self.packages = {}
24
25
26
27
        package_list = list(self._get_packages(user, pkg))
        if not package_list:
            package_list = random.sample([package for package, _ in _client.top_packages(1000)], 10)
        for name in sorted(package_list):
Louis's avatar
Louis committed
28
29
30
31
32
            try:
                self.packages[name] = Package(name)
            except ValueError:
                sys.stderr.write("Ignoring package '{}' with no releases.\n".format(name))
                continue
33

Louis's avatar
Louis committed
34
35
36
37
38
39
40
41
    @staticmethod
    def _get_packages(users, packages):
        if users is not None:
            for user in users:
                for _, package in _client.user_packages(user):
                    try:
                        yield package
                    except urllib.error.HTTPError as error:
Louis's avatar
Louis committed
42
                        sys.stderr.write("Ignoring user '{}': {}.".format(
Louis's avatar
Louis committed
43
44
45
46
47
                            user,
                            str(error),
                            ))
        if packages is not None:
            yield from packages
48
49
50
51

    def __iter__(self):
        yield from self.packages

Louis's avatar
Louis committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    def values(self):
        yield from self.packages.values()

    def __getitem__(self, key):
        return self.packages[key]

    def iter_dated_downloads(self, name=None):
        if name is None:
            packages = list(self.values())
        else:
            packages = [self[name]]
        for date in set(itertools.chain(*(package.iter_dates() for package in packages))):
            downloads = []
            for package in packages:
                try:
                    downloads.append((package.name, package.releases[date]['previous']))
                except KeyError:
                    pass
            yield (date, downloads)
        yield (
            datetime.datetime.now(),
            [(package.name, package.total_downloads) for package in packages],
            )

    def iter_ykeys(self, name=None):
        if name is None:
            yield from self
        else:
            yield name

def DateTime2datetime(date):
    timetuple = date.timetuple()
    return datetime.datetime(
            timetuple.tm_year,
            timetuple.tm_mon,
            timetuple.tm_mday,
            timetuple.tm_hour,
            timetuple.tm_min,
            timetuple.tm_sec,
            )

93
94
class Package:

Louis's avatar
Louis committed
95
96
97
98
    docs_api = {
        'readthedocs': slumber.API(base_url="http://readthedocs.org/api/v1/"),
        }

99
    def __init__(self, name):
Louis's avatar
Louis committed
100
        sys.stderr.write("Retrieving information about pypi package '{}'...\n".format(name))
101
        self.name = name
Louis's avatar
Louis committed
102
103
104
105
106
107
108
109
110
        self.releases = {}
        for release in _client.package_releases(name, True):
            release_urls = _client.release_urls(name, release)
            if not release_urls:
                continue
            date = DateTime2datetime(min((url['upload_time'] for url in release_urls)))
            self.releases[date] = {}
            self.releases[date]['downloads'] = sum((url['downloads'] for url in release_urls))
            self.releases[date]['version'] = release
Louis's avatar
Louis committed
111
112
        if not self.releases:
            raise ValueError("Package '{}' does not exist or haven't any release.".format(name))
Louis's avatar
Louis committed
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

        downloads = 0
        for release in sorted(self.releases):
            self.releases[release]['previous'] = downloads
            downloads += self.releases[release]['downloads']

        self.data = _client.release_data(name, self.releases[max(sorted(self.releases))]['version'])

    def iter_dates(self):
        yield from self.releases.keys()

    @property
    def total_downloads(self):
        last = sorted(self.releases.keys())[-1]
        return self.releases[last]['downloads'] + self.releases[last]['previous']
128

Louis's avatar
Louis committed
129
130
131
132
133
134
135
136
137
138
139
140
141
142
    def get_documentation(self):
        # Readthedocs
        if self.docs_api['readthedocs'].project.get(slug=self.name.lower())['objects']:
            return (
                'readthedocs', 
                None,
                )

        # Pypi docs
        if self.data['docs_url'] is not None:
            return ("pypi", self.name)

        # No documentation
        return (None, {})