stats.py 4.54 KB
Newer Older
Louis's avatar
Louis committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
#!/usr/bin/env python3

import collections
import datetime
import pprint
import sys

try:
    import xmlrpclib
except ImportError:
    import xmlrpc.client as xmlrpclib

USER = "spalax"

client = xmlrpclib.ServerProxy("http://pypi.python.org/pypi")

def process_package(package):
    data = {}
    sys.stderr.write("Processing {}...\n".format(package))
    for release in client.package_releases(package, True):
        sys.stderr.write("Processing version {}...".format(release))
        data[release] = {'downloads': 0}
        for urls in client.release_urls(package, release):
            data[release]['downloads'] += urls['downloads']
            data[release]['upload_time'] = urls['upload_time']
        sys.stderr.write(" Done\n")
    return data

def get_release_date(data):
    try:
        return data[1]['upload_time']
    except KeyError:
        # Will be ignored later
        return datetime.datetime.fromtimestamp(0)

def shift(data):
    date = [{'downloads': 0}]
    for release, _ in sorted(data.items(), key=get_release_date):
        try:
            datetuple = data[release]['upload_time'].timetuple()
        except KeyError:
            continue
        date[-1].update({
            'date': datetime.date(datetuple.tm_year, datetuple.tm_mon, datetuple.tm_mday),
             'release': release,
        })
        date.append({
            'downloads': data[release]['downloads'] + date[-1]['downloads'],
        })
    date[-1].update({
        'date': datetime.date.today(),
        'release': "",
    })
    return date

def bydate(package, data):
    dateddata = collections.defaultdict(list)
    for release in data:
        dateddata[release['date']].append({
            'downloads': release['downloads'],
            'release': release['release'],
            'package': package,
            })
    return dateddata

def compile_dates(package_data):
    data = collections.defaultdict(list)
    for package in package_data:
        for date, release in package.items():
            data[date].extend(release)
    return data

HTML_BEGIN = """
<!DOCTYPE html>
<html lang="en">
  <head>
  <meta charset="utf-8">
      <title>Pypi statistics</title>
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/morris.js/0.5.1/morris.css">
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.9.0/jquery.min.js"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/raphael/2.1.0/raphael-min.js"></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/morris.js/0.5.1/morris.min.js"></script>
                    </head>
                      <body>
                      """

HTML_GRAPHDIV = """
<div id="pystats{counter}" style="height: 250px;"></div>
"""

HTML_SCRIPT = """
<script>
new Morris.Line({{
  element: 'pystats{counter}',
  data: [
    {yvalues}
  ],
  xkey: 'date',
  ykeys: [{ykeys}],
  labels: [{ykeys}]
}});
</script>
"""

HTML_END = """
</body>
</html>
"""

def make_id(string):
    new = ""
    for char in string:
        if char.isidentifier():
            new += char
    return new

def print_y(data):
    for date, releaselist in data.items():
        values = ", ".join((
            "{}: {}".format(make_id(release['package']), release['downloads'])
            for release in releaselist
            ))
        yield "{{ date: '{}', {}}}".format(str(date), values)

def print_ykeys(data):
    packages = set()
    for releaselist in data.values():
        for release in releaselist:
            packages.add(make_id(release['package']))
    for package in packages:
        yield "'{}'".format(package)

class Counter:
    def __init__(self):
        self.total = 0
    def count(self):
        self.total += 1
        return self.total
counter = Counter()

def get_compiled(packages):
    compiled = compile_dates((
        bydate(package, shift(process_package(package)))
        for package in packages
       ))

    count = counter.count()
    return (
            HTML_SCRIPT.format(
                ykeys=", ".join(print_ykeys(compiled)),
                yvalues=",\n".join(print_y(compiled)),
                counter= count,
                ),
            HTML_GRAPHDIV.format(counter=count),
            )

packages = [package for role, package in client.user_packages(USER)]

graphs = []
graphs.append(get_compiled(packages))
for package in packages:
    graphs.append(get_compiled([package]))

print(HTML_BEGIN)
for _, div in graphs:
    print(div)
for script, _ in graphs:
    print(script)
print(HTML_END)