Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Louis
pypimonitor
Commits
885f2b5e
Commit
885f2b5e
authored
May 27, 2016
by
Louis
Browse files
Pylint compliant
parent
f7e05ed5
Changes
14
Hide whitespace changes
Inline
Side-by-side
pylintrc
0 → 100644
View file @
885f2b5e
[VARIABLES]
dummy-variables-rgx=_|dummy
[MESSAGES CONTROL]
disable= logging-format-interpolation, I
pypimonitor/__init__.py
View file @
885f2b5e
"""Gather data about packages, and render them as an HTML page."""
import
colorsys
import
datetime
import
functools
import
itertools
import
jinja2
import
json
import
logging
import
operator
import
os
import
pkg_resources
import
random
import
requests
import
slumber
import
sys
import
urllib
import
xmlrpc.client
as
xmlrpclib
import
yaml
import
jinja2
import
pkg_resources
import
requests
import
slumber
from
pypimonitor.column
import
load_column_plugins
VERSION
=
"0.2.0-dev"
LOGGER
=
logging
.
getLogger
(
__name__
)
_client
=
xmlrpclib
.
ServerProxy
(
'https://pypi.python.org/pypi'
)
CLIENT
=
xmlrpclib
.
ServerProxy
(
'https://pypi.python.org/pypi'
)
class
Counter
:
"""Each call to :meth:`Counter.count` increments a counter, and return its value."""
# pylint: disable=too-few-public-methods
def
__init__
(
self
):
self
.
value
=
0
def
count
(
self
):
"""Increments counter, and return counter value."""
self
.
value
+=
1
return
self
.
value
def
as_id
(
string
):
return
""
.
join
(
list
(
filter
(
str
.
isalnum
,
string
)))
def
random_color
(
seed
,
saturation
,
value
):
"""Return a pseudo-random color."""
old_seed
=
random
.
random
()
random
.
seed
(
seed
)
rgb
=
colorsys
.
hsv_to_rgb
(
random
.
random
(),
saturation
,
value
)
random
.
seed
(
old_seed
)
return
(
"#%02x%02x%02x"
%
tuple
([
int
(
value
*
255
)
for
value
in
rgb
])
)
return
"#%02x%02x%02x"
%
tuple
([
int
(
value
*
255
)
for
value
in
rgb
])
def
iter_user_packages
(
user
):
for
_
,
package
in
_client
.
user_packages
(
user
):
"""Iterate over packages owned by a given user."""
for
_
,
package
in
CLIENT
.
user_packages
(
user
):
yield
package
class
Renderer
:
"""Get information about packages, and produce an HTML dashboard to monitor them."""
def
__init__
(
self
,
config
,
form
=
None
):
self
.
errors
=
set
()
...
...
@@ -54,9 +61,9 @@ class Renderer:
self
.
config
=
config
if
"columns"
not
in
self
.
config
:
self
.
config
[
'columns'
]
=
list
(
set
(
itertools
.
chain
(
*
[
package
.
keys
()
for
package
in
self
.
config
[
'packages'
].
values
()
package
.
keys
()
for
package
in
self
.
config
[
'packages'
].
values
()
])))
for
package
in
self
.
config
[
'packages'
]:
if
self
.
config
[
'packages'
][
package
]
is
None
:
...
...
@@ -68,14 +75,15 @@ class Renderer:
for
key
in
self
.
config
.
get
(
'default'
,
{}):
if
key
not
in
self
.
config
[
'packages'
][
package
]:
self
.
config
[
'packages'
][
package
][
key
]
=
self
.
config
[
'default'
][
key
]
self
.
config
[
'packages'
][
package
][
column
][
'column'
]
=
self
.
config
[
'packages'
][
package
][
column
].
get
(
'column'
,
column
)
if
'column'
not
in
self
.
config
[
'packages'
][
package
][
column
]:
self
.
config
[
'packages'
][
package
][
column
][
'column'
]
=
column
# Processing packages
self
.
packages
=
{}
for
name
in
sorted
(
config
[
'packages'
]):
try
:
self
.
packages
[
name
]
=
Package
(
name
)
except
Exception
as
error
:
except
Exception
as
error
:
# pylint: disable=broad-except
self
.
log_error
(
"Ignoring package '{}': {}"
.
format
(
name
,
error
))
continue
...
...
@@ -83,7 +91,9 @@ class Renderer:
self
.
form
=
form
if
self
.
form
is
not
None
:
for
key
in
[
'pkg'
,
'user'
,
'col'
]:
self
.
form
[
key
]
=
","
.
join
(
set
(
itertools
.
chain
(
*
[
item
.
split
(
","
)
for
item
in
self
.
form
.
get
(
key
,
[])])))
self
.
form
[
key
]
=
","
.
join
(
set
(
itertools
.
chain
(
*
[
item
.
split
(
","
)
for
item
in
self
.
form
.
get
(
key
,
[])])
))
# Loading plugins
self
.
columns
=
{
...
...
@@ -94,11 +104,13 @@ class Renderer:
@
classmethod
def
from_yaml
(
cls
,
filename
,
form
=
None
):
"""Factory to return a :class:`Renderer` from a yaml file."""
with
open
(
filename
,
'r'
)
as
stream
:
return
cls
(
yaml
.
safe_load
(
stream
.
read
()),
form
=
form
)
@
classmethod
def
from_args
(
cls
,
packages
=
None
,
columns
=
None
,
users
=
None
,
form
=
None
):
"""Factory to return a :class:`Renderer` from list of arguments."""
config
=
{}
# Processing packages
...
...
@@ -127,9 +139,11 @@ class Renderer:
return
cls
(
config
,
form
=
form
)
def
__iter__
(
self
):
"""Return iterator to `self.packages`."""
yield
from
self
.
packages
def
values
(
self
):
"""Yield `self.packages` values."""
yield
from
self
.
packages
.
values
()
def
__getitem__
(
self
,
key
):
...
...
@@ -137,12 +151,12 @@ class Renderer:
@
staticmethod
def
_get_environment
():
TEMPLATEDIR
=
os
.
path
.
join
(
templatedir
=
os
.
path
.
join
(
pkg_resources
.
resource_filename
(
__name__
,
'data'
),
'templates'
,
)
env
=
jinja2
.
Environment
(
loader
=
jinja2
.
FileSystemLoader
(
TEMPLATEDIR
))
env
=
jinja2
.
Environment
(
loader
=
jinja2
.
FileSystemLoader
(
templatedir
))
env
.
globals
[
'counter'
]
=
Counter
().
count
env
.
globals
[
'now'
]
=
datetime
.
datetime
.
now
()
env
.
filters
[
'datetime'
]
=
operator
.
methodcaller
(
'strftime'
,
"%Y-%m-%d %H:%M:%S"
)
...
...
@@ -150,9 +164,8 @@ class Renderer:
return
env
def
render
(
self
):
env
=
self
.
_get_environment
()
template
=
env
.
get_template
(
"index.html"
)
return
template
.
render
(
"""Return the HTML code."""
return
self
.
_get_environment
().
get_template
(
"index.html"
).
render
(
# pylint: disable=no-member
config
=
self
.
config
,
columns
=
self
.
columns
,
packages
=
self
.
packages
,
...
...
@@ -161,20 +174,23 @@ class Renderer:
)
def
log_error
(
self
,
message
):
"""Log error, both to HTML page and console."""
LOGGER
.
error
(
message
)
self
.
errors
.
add
(
message
)
class
Package
:
docs_api
=
{
'readthedocs'
:
slumber
.
API
(
base_url
=
"http://readthedocs.io/api/v1/"
),
}
"""Gather information about a single package."""
# pylint: disable=too-few-public-methods
def
__init__
(
self
,
name
):
LOGGER
.
info
(
"Retrieving information about pypi package '{}'...
\n
"
.
format
(
name
))
try
:
self
.
rawdata
=
json
.
loads
(
requests
.
get
(
"http://pypi.python.org/pypi/{}/json"
.
format
(
name
)).
text
)
self
.
rawdata
=
json
.
loads
(
requests
.
get
(
"http://pypi.python.org/pypi/{}/json"
.
format
(
name
)
).
text
)
except
Exception
as
error
:
raise
Exception
(
"Error while getting data for package '{}': {}."
.
format
(
name
,
error
))
self
.
name
=
name
...
...
@@ -198,25 +214,8 @@ class Package:
self
.
releases
[
release
][
'previous'
]
=
downloads
downloads
+=
self
.
releases
[
release
][
'downloads'
]
def
iter_dates
(
self
):
yield
from
self
.
releases
.
keys
()
@
property
def
total_downloads
(
self
):
"""Return this package's total downloads (all versions combined)."""
last
=
sorted
(
self
.
releases
.
keys
())[
-
1
]
return
self
.
releases
[
last
][
'downloads'
]
+
self
.
releases
[
last
][
'previous'
]
def
get_documentation
(
self
):
# Readthedocs
if
self
.
docs_api
[
'readthedocs'
].
project
.
get
(
slug
=
self
.
name
.
lower
())[
'objects'
]:
return
(
'readthedocs'
,
None
,
)
# Pypi docs
if
self
.
info
[
'docs_url'
]
is
not
None
:
return
(
"pypi"
,
None
)
# No documentation
return
(
None
,
None
)
pypimonitor/__main__.py
View file @
885f2b5e
"""Gather python packages information, and render them as an HTML page."""
import
argparse
import
sys
import
logging
...
...
@@ -55,14 +57,18 @@ def commandline_parser():
return
parser
def
_flatten
(
list_of_lists
):
"""Given a list of comma-separated lists, iterate over items."""
for
llist
in
list_of_lists
:
for
item
in
llist
:
yield
from
item
.
split
(
","
)
if
__name__
==
"__main__"
:
def
main
():
"""Main function: called when calling this module from command line."""
arguments
=
commandline_parser
().
parse_args
()
if
((
arguments
.
yaml
is
not
None
)
and
(
arguments
.
column
or
arguments
.
package
or
arguments
.
user
)):
LOGGER
.
error
(
"Configuration file and --column, --package and --user arguments are incompatible."
)
if
(
arguments
.
yaml
is
not
None
)
and
(
arguments
.
column
or
arguments
.
package
or
arguments
.
user
):
LOGGER
.
error
(
"Configuration file and --column, --package and --user arguments are incompatible."
)
sys
.
exit
(
1
)
if
arguments
.
yaml
is
None
:
print
(
pypimonitor
.
Renderer
.
from_args
(
...
...
@@ -72,3 +78,6 @@ if __name__ == "__main__":
).
render
())
else
:
print
(
pypimonitor
.
Renderer
.
from_yaml
(
arguments
.
yaml
).
render
())
if
__name__
==
"__main__"
:
main
()
pypimonitor/column/__init__.py
View file @
885f2b5e
# TODO Copyright
"""Generic definitions to implement column plugins."""
import
contextlib
import
logging
...
...
@@ -6,17 +6,18 @@ import os
from
straight
import
plugin
import
jinja2
import
pypimonitor
LOGGER
=
logging
.
getLogger
(
pypimonitor
.
__name__
)
LOGGER
=
logging
.
getLogger
(
"
pypimonitor
"
)
def
load_column_plugins
():
"""Iterator over the column plugins."""
for
column
in
set
(
plugin
.
load
(
'pypimonitor.column'
,
subclasses
=
Column
)):
if
column
.
keyword
is
not
None
:
yield
column
@
contextlib
.
contextmanager
def
temp_update
(
source
,
update
):
"""Temporary update of a dictionary."""
bak
=
{}
new
=
set
()
for
key
in
update
:
...
...
@@ -36,6 +37,8 @@ def temp_update(source, update):
del
source
[
key
]
class
Column
:
"""Render some piece of information about a package as HTML code."""
keyword
=
None
title
=
""
default
=
{}
...
...
@@ -57,21 +60,37 @@ class Column:
if
key
not
in
filled
:
missing
.
add
(
key
)
if
missing
:
return
self
.
render_error
(
context
,
self
.
keyword
,
package
,
"Missing argument(s) {}."
.
format
(
", "
.
join
([
"'{}'"
.
format
(
key
)
for
key
in
missing
])))
return
self
.
render_error
(
context
,
self
.
keyword
,
package
,
"Missing argument(s) {}."
.
format
(
", "
.
join
([
"'{}'"
.
format
(
key
)
for
key
in
missing
])
),
)
return
self
.
render
(
context
,
package
,
filled
)
def
render
(
self
,
context
,
package
,
column
):
"""Return the HTML code corresponding to this column."""
raise
NotImplementedError
()
def
render_error
(
self
,
context
,
column
,
package
,
message
):
@
staticmethod
def
render_error
(
context
,
column
,
package
,
message
):
"""Return the HTML code corresponding to an error."""
full_message
=
"Package '{}', column '{}': {}"
.
format
(
package
,
column
,
message
)
LOGGER
.
error
(
full_message
)
return
context
.
environment
.
get_template
(
os
.
path
.
join
(
"columns"
,
"cellerror.html"
)).
render
(
message
=
full_message
)
return
(
context
.
environment
.
get_template
(
os
.
path
.
join
(
"columns"
,
"cellerror.html"
))
.
render
(
message
=
full_message
)
)
class
Jinja2
(
Column
):
"""Generic class for columns that are barely more than a template."""
def
render
(
self
,
context
,
package
,
column
):
"""Return the HTML code corresponding to this column."""
with
temp_update
(
dict
(
context
),
{
'package'
:
package
,
'pypi'
:
self
.
renderer
[
package
],
...
...
@@ -81,4 +100,5 @@ class Jinja2(Column):
@
property
def
template
(
self
):
"""Return template path."""
return
os
.
path
.
join
(
"columns"
,
"{}.html"
.
format
(
self
.
keyword
))
pypimonitor/column/base.py
View file @
885f2b5e
"""Some base column plugins."""
from
pypimonitor.column
import
Column
,
Jinja2
class
Empty
(
Column
):
"""Empty column."""
keyword
=
"empty"
def
render
(
self
,
context
,
package
,
column
):
return
""
class
Html
(
Column
):
"""Render raw HTML code as the column."""
keyword
=
'html'
required
=
[
'html'
]
...
...
@@ -14,6 +20,8 @@ class Html(Column):
return
column
[
'html'
]
class
Error
(
Column
):
"""Render an error."""
keyword
=
'error'
def
render
(
self
,
context
,
package
,
column
):
...
...
@@ -25,6 +33,8 @@ class Error(Column):
)
class
Link
(
Jinja2
):
"""Render a link."""
keyword
=
'link'
required
=
[
'href'
]
...
...
@@ -34,4 +44,6 @@ class Link(Jinja2):
return
super
().
render
(
context
,
package
,
column
)
class
Color
(
Jinja2
):
"""Render a color square."""
keyword
=
'color'
pypimonitor/column/gitlab.py
View file @
885f2b5e
"""Columns related to http://gitlab.com"""
import
os
from
pypimonitor.column
import
Jinja2
class
Gitlab
(
Jinja2
):
"""Columns related to http://gitlab.com"""
@
property
def
template
(
self
):
return
os
.
path
.
join
(
"columns"
,
"gitlab"
,
"{}.html"
.
format
(
self
.
keyword
))
class
CI
(
Gitlab
):
"""Gitlab-ci badge."""
keyword
=
"gitlabci"
title
=
"GitlabCI"
default
=
{
...
...
pypimonitor/column/pypi.py
View file @
885f2b5e
"""Columns rendering data from the pypi API."""
from
pypimonitor.column
import
Jinja2
class
Homepage
(
Jinja2
):
"""Link to the package home page."""
keyword
=
'homepage'
title
=
'Home'
pypimonitor/column/readthedocs.py
View file @
885f2b5e
"""Column plugins related to http://readthedocs.org"""
from
pypimonitor.column
import
Jinja2
class
Readthedocs
(
Jinja2
):
"""Readthedocs.org build badge."""
keyword
=
'readthedocs'
title
=
'Doc'
default
=
{
...
...
pypimonitor/column/
badge
s.py
→
pypimonitor/column/
shield
s.py
View file @
885f2b5e
"""Badges from http://shields.io"""
import
os
from
pypimonitor.column
import
Jinja2
class
BadgesIO
(
Jinja2
):
class
ShieldsIO
(
Jinja2
):
"""Generic plugin, that sets up template directory."""
@
property
def
template
(
self
):
return
os
.
path
.
join
(
"columns"
,
"badgesio"
,
"{}.html"
.
format
(
self
.
keyword
))
return
os
.
path
.
join
(
"columns"
,
"shieldsio"
,
"{}.html"
.
format
(
self
.
keyword
))
class
PypiVersion
(
ShieldsIO
):
"""Pypi version."""
class
PypiVersion
(
BadgesIO
):
keyword
=
"pypiversion"
title
=
"Version"
class
PypiWeeklyDownloads
(
BadgesIO
):
class
PypiWeeklyDownloads
(
ShieldsIO
):
"""Pypi weekly downloads."""
keyword
=
"pypiwdownloads"
title
=
"Weekly"
class
PypiDailyDownloads
(
BadgesIO
):
class
PypiDailyDownloads
(
ShieldsIO
):
"""Pypi daily downloads."""
keyword
=
"pypiddownloads"
title
=
"Daily"
class
PypiMonthlyDownloads
(
BadgesIO
):
class
PypiMonthlyDownloads
(
ShieldsIO
):
"""Pypi monthly downloads."""
keyword
=
"pypimdownloads"
title
=
"Monthly"
pypimonitor/column/travis.py
View file @
885f2b5e
import
os
"""Column plugins related to http://travis-ci.org/"""
from
pypimonitor.column
import
Jinja2
class
TravisCI
(
Jinja2
):
"""Travis badge."""
keyword
=
"travisci"
required
=
[
'user'
]
pypimonitor/data/templates/columns/
badge
sio/pypiddownloads.html
→
pypimonitor/data/templates/columns/
shield
sio/pypiddownloads.html
View file @
885f2b5e
File moved
pypimonitor/data/templates/columns/
badge
sio/pypimdownloads.html
→
pypimonitor/data/templates/columns/
shield
sio/pypimdownloads.html
View file @
885f2b5e
File moved
pypimonitor/data/templates/columns/
badge
sio/pypiversion.html
→
pypimonitor/data/templates/columns/
shield
sio/pypiversion.html
View file @
885f2b5e
File moved
pypimonitor/data/templates/columns/
badge
sio/pypiwdownloads.html
→
pypimonitor/data/templates/columns/
shield
sio/pypiwdownloads.html
View file @
885f2b5e
File moved
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment