Commit b28f1c26 authored by Julien Lepiller's avatar Julien Lepiller

Initial commit

parents
*.swp
cache
public
fdroid.manifest
fdroiddata
# i18n monitoring website.
#
# Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
MYDATE=$(shell date +'%Y.%m.%d')
RESULTS=$(shell ls public/data) results.$(MYDATE).json
JSON_RESULTS=$(addprefix public/data/, $(RESULTS))
DAY_IMAGES=$(addsuffix .png, $(addprefix public/images/, $(RESULTS)))
FDROIDDATA=$(shell ls fdroiddata/metadata/*.txt fdroiddata/metadata/*.yml)
TRANSMON=/root/.config/guix/current/bin/guix environment -L modules --ad-hoc \
python transmon nss-certs -- transmon
QUIET=@
FDROIDDATA=fdroiddata
all: $(DAY_IMAGES)
$(MAKE) public/i18n.css public/i18n.html public/i18n-data.js
fdroid-update:
@if [ ! -d $(FDROIDDATA) ]; then \
echo [GIT CLONE] $(FDROIDDATA); \
git clone https://gitlab.com/fdroid/fdroiddata.git $(FDROIDDATA); \
fi
@echo [GIT PULL] $(FDROIDDATA)
$(QUIET)cd $(FDROIDDATA); git pull
cache/i18n-data-languages.js: $(shell ls public/images/evo-*)
@if [ ! -d cache ]; then mkdir cache; fi
@echo [GEN] $@
$(QUIET)echo "surveyed_languages=[" > $@
$(QUIET)for f in public/images/evo*; do \
echo "'$$(basename $$f | cut -c5- | rev | cut -c5- | rev)'," >> $@; \
done
$(QUIET)echo "];" >> $@
cache/i18n-data-dates.js: $(shell ls public/images/results*)
@if [ ! -d cache ]; then mkdir cache; fi
@echo [GEN] $@
$(QUIET)echo "survey_dates=[" > $@
$(QUIET)for f in public/images/results*; do \
echo "'$$(basename $$f | cut -c9- | rev | cut -c10- | rev)'," >> $@; \
done
$(QUIET)echo "];" >> $@
public/i18n-data.js: cache/i18n-data-languages.js cache/i18n-data-dates.js
@if [ ! -d public ]; then mkdir public; fi
@echo [GEN] $@
$(QUIET)cat $^ > $@
public/images/results.%.png: public/data/results.% one-lang.scm
@if [ ! -d public/images ]; then mkdir -p public/images; fi
@echo [ONE-DAY] $@
$(QUIET)/root/.config/guix/current/bin/guix environment --ad-hoc guile guile-json \
gnuplot -- guile -L modules one-day.scm $<
public/i18n.html: static/i18n.html
@if [ ! -d public ]; then mkdir public; fi
cp $< $@
public/i18n.css: static/i18n.css
@if [ ! -d public ]; then mkdir public; fi
cp $< $@
evo: $(JSON_RESULTS)
@if [ ! -d public/images ]; then mkdir -p public/images; fi
@echo [EXEC] one-lang.scm
$(QUIET)/root/.config/guix/current/bin/guix environment --ad-hoc guile guile-json \
gnuplot -- guile -L modules one-lang.scm $(RESULTS)
public/data/results.$(MYDATE).json: fdroid.manifest
@if [ ! -d public/data ]; then mkdir -p public/data; fi
@echo [TRANSMON] $@
$(QUIET)$(TRANSMON) $< > $@
fdroid.manifest: $(FDROIDDATA)
@echo [FDROIDDATA] $@
$(QUIET)/root/.config/guix/current/bin/guix environment --ad-hoc \
guile guile-json -- guile generate-fdroid.scm $^ > $@
Internationalization Monitoring
===============================
This repository contains scripts and programs used to generate the website at
[i18n.lepiller.eu](https://i18n.lepiller.eu/i18n.html). Every day, a cron task
updates repositories (currently only fdroid), analyses them to find a list of
free software projects, and analyses these projects to find their translation
status. Then, the website is generated with graphs and explanations.
Scripts and Usage
-----------------
In order to run the website and analysis, the following script is needed inside
a cron task:
```bash
cd /path/to/this/repository
make fdroid-update
make
```
This creates a website inside the `public` directory.
The Makefile is the main entry point to the analysis. The `modules` directory
contains important scheme modules used by some of the scripts. `data-analysis.scm`
contains common procedures for every analysis, while `guix-packages.scm` contains
[guix](https://guix.gnu.org) package definitions. The `static` directory contains
static website elements: a css and an html file.
Data Provenance
--------------
The set of free software analysed by this repository comes from the sources
below:
### Fdroidata
We analyze the content of the [fdroiddata](https://gitlab.com/fdroid/fdroiddata.git)
repository to find a list of free software android applications. The
`generate-fdroid.scm` script is used to generate a json file that can be
passed to transmon.
Currently, the script will analyze every file and generate a json object for
every file that contains at least this information: repository url, home page,
name and type of repository. Since transmon doesn't yet support other types
of repositories, we filter repository types and keep only git repositories
(the vast majority of applications use a git repository).
Data Collection
---------------
Everyday, we collect data from the generated json files: it's passed to transmon
which updates its cache or fetches new repositories. It then analyses each
application and returns another json file, named `public/data/results.<date>.json`.
This file is publicly available for download. See
[transmon](https://framagit.org/tyreunom/transmon)'s repository for more details
on how it works.
Data Analysis
-------------
The last step is data analysis. There are two scripts run to analyse the data
and generate graphs:
### one-day.scm
This script is responsible for generating a summary of a day's data. It generates
two graphs in `public/images` that represent the percentage of translated
applications in every language. It also sorts languages by a weighted average
of the number of apps in each translation status category.
### one-lang.scm
This script is responsible for generating a summary of a language status evolution.
It generates multiple graphs in `public/images` that each represent the evolution
of a language over a long period of time.
Website Generation
------------------
The website itself is not very complicated. The only generated part is a small
script that contains two variables that correspond to the availability of some
graphs and data, `i18n-data.js`.
;;; i18n monitoring website.
;;;
;;; Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
;;;
;;; This program is free software: you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation, either version 3 of the License, or
;;; (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program. If not, see <http://www.gnu.org/licenses/>.
(use-modules (json) (ice-9 match) (ice-9 rdelim))
(define fdroid-files (cdr (command-line)))
(define (generate-data data)
`(("name" . ,(assoc-ref data "name"))
("variants" .
,(list->array 1
`((("fetcher" . "git")
("uri" . ,(assoc-ref data "url"))
("tags" . ,(list->array 1 '("master")))
("name" . "git-master")))))
("tags" . ,(list->array 1 '("android" "fdroid")))
("home-page" . ,(assoc-ref data "home"))))
(define (getstring line from)
(string-trim-both (substring line from)))
(define (parse-txt file)
(call-with-input-file file
(lambda (port)
(let loop ((result '()))
(let ((s (read-line port)))
(cond
((eof-object? s) result)
((eq? (string-contains s "RepoType:") 0)
(let ((type (getstring s 9)))
(if (not (equal? type "git"))
#f
(loop (cons `("type" . "git") result)))))
((eq? (string-contains s "Repo Type:") 0)
(let ((type (getstring s 10)))
(if (not (equal? type "git"))
#f
(loop (cons `("type" . "git") result)))))
((eq? (string-contains s "AutoName:") 0)
(loop (cons `("name" . ,(getstring s 9)) result)))
((eq? (string-contains s "Auto Name:") 0)
(loop (cons `("name" . ,(getstring s 10)) result)))
((eq? (string-contains s "WebSite:") 0)
(let ((site (getstring s 8)))
(if (equal? site "")
(loop result)
(loop (cons* `("home" . ,site) `("website" . #t) result)))))
((eq? (string-contains s "Web Site:") 0)
(let ((site (getstring s 9)))
(if (equal? site "")
(loop result)
(loop (cons* `("home" . ,site) `("website" . #t) result)))))
((eq? (string-contains s "SourceCode:") 0)
(if (assoc-ref result "website")
(loop result)
(loop (cons `("home" . ,(getstring s 11)) result))))
((eq? (string-contains s "Source Code:") 0)
(if (assoc-ref result "website")
(loop result)
(loop (cons `("home" . ,(getstring s 12)) result))))
((eq? (string-contains s "Repo:") 0)
(loop (cons `("url" . ,(getstring s 5)) result)))
(else (loop result))))))))
(define (parse-yml file)
;; It's actually very similar :)
(parse-txt file))
(display
(scm->json-string
(list->array 1
(map
generate-data
(filter (lambda (a)
(and (assoc-ref a "url") (assoc-ref a "type")
(assoc-ref a "home") (assoc-ref a "name")))
(map parse-txt fdroid-files))))))
(newline)
;;; i18n monitoring website.
;;;
;;; Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
;;;
;;; This program is free software: you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation, either version 3 of the License, or
;;; (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program. If not, see <http://www.gnu.org/licenses/>.
(define-module (data-analysis)
#:use-module (srfi srfi-1)
#:use-module (ice-9 match)
#:use-module (json)
#:export (uniq
project-data
categorize
count-cat
get-langs
generate-lang-data))
(define (uniq lst)
"Return only uniq data from lst"
(let loop ((result '()) (lst lst))
(if (null? lst)
result
(loop (if (member (car lst) result) result (cons (car lst) result)) (cdr lst)))))
(define (project-data lang stats)
"Return stats only for one language"
(map (lambda (project) (assoc-ref project lang)) stats))
(define (categorize n)
"Return the category of a fraction"
(if n
(if (eq? n 1)
1
(if (> n (/ 90 100))
2
(if (> n (/ 70 100))
3
(if (> n (/ 30 100))
4
(if (eq? n 0) #f 5)))))
#f))
(define (count-cat cat-stats n)
"Return the count of projects that are at least in the category @var{n}"
(exact->inexact
(/ (length (filter (lambda (cat) (and cat (>= cat n))) cat-stats))
(length cat-stats))))
(define (any l)
"Return #t if any value in list @var{l} is true"
(fold (lambda (a b) (or a b)) #f l))
(define (get-langs stats)
"Return every language found in stats, ignoring weird names"
(let ((langs (uniq (apply append (map (lambda (project) (map car project)) stats)))))
(filter
(lambda (l)
(and (not (string-index l char-numeric?))
(not (string-contains l "keyshidden"))
(not (string-contains l "keysexposed"))
(not (any (map (lambda (s) (string-contains l s))
'("normal" "xlarge" "small" "large"
"hdpi"))))))
langs)))
(define* (generate-lang-data stats #:key (count-cat count-cat))
(let* ((langs (get-langs stats)))
(map (lambda (lang)
(let* ((lang-data (project-data lang stats))
(lang-count
(map
(lambda (s)
(if s
(let ((num (array-ref s 0))
(den (array-ref s 1)))
(if (eq? den 0) #f (/ num den)))
#f))
lang-data))
(lang-cat (map categorize lang-count)))
(list lang (count-cat lang-cat 1)
(count-cat lang-cat 2)
(count-cat lang-cat 3)
(count-cat lang-cat 4)
(count-cat lang-cat 5))))
langs)))
;;; i18n monitoring website.
;;;
;;; Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
;;;
;;; This program is free software: you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation, either version 3 of the License, or
;;; (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program. If not, see <http://www.gnu.org/licenses/>.
(define-module (guix-packages)
#:use-module ((guix licenses) #:prefix license:)
#:use-module (guix packages)
#:use-module (guix build-system python)
#:use-module (guix git-download)
#:use-module (guix download)
#:use-module (gnu packages check)
#:use-module (gnu packages python)
#:use-module (gnu packages python-xyz)
#:use-module (gnu packages sphinx)
#:use-module (gnu packages version-control))
(define-public python-check-manifest
(package
(name "python-check-manifest")
(version "0.37")
(source
(origin
(method url-fetch)
(uri (pypi-uri "check-manifest" version))
(sha256
(base32
"0lk45ifdv2cpkl6ayfyix7jwmnxa1rha7xvb0ih5999k115wzqs4"))))
(build-system python-build-system)
(native-inputs
`(("python-mock" ,python-mock)
("git" ,git)))
(home-page
"https://github.com/mgedmin/check-manifest")
(synopsis
"Check MANIFEST.in in a Python source package for completeness")
(description
"Check MANIFEST.in in a Python source package for completeness")
(license license:expat)))
(define-public python-codacy-coverage
(package
(name "python-codacy-coverage")
(version "1.3.11")
(source
(origin
(method url-fetch)
(uri (pypi-uri "codacy-coverage" version))
(sha256
(base32
"1g0c0w56xdkmqb8slacyw5qhzrkp814ng3ddh2lkiij58y9m2imr"))))
(build-system python-build-system)
(arguments
;; No tests
`(#:tests? #f))
(propagated-inputs
`(("python-check-manifest" ,python-check-manifest)))
(home-page
"https://github.com/codacy/python-codacy-coverage")
(synopsis "Codacy coverage reporter for Python")
(description
"Codacy coverage reporter for Python")
(license license:expat)))
(define-public python-translation-finder
(package
(name "python-translation-finder")
(version "1.0")
(source
(origin
(method url-fetch)
(uri (pypi-uri "translation-finder" version))
(sha256
(base32
"1bldziyp0kn93g9gg0wjibpcjwg6xcp8zqwch7bnplg0zwxs1y8b"))))
(build-system python-build-system)
(propagated-inputs
`(("python-chardet" ,python-chardet)
("python-pathlib2;" ,python-pathlib2)
("python-six" ,python-six)))
(native-inputs
`(("python-codecov" ,python-codecov)
("python-codacy-coverage" ,python-codacy-coverage)
("python-pytest-cov" ,python-pytest-cov)
("python-pytest-runner" ,python-pytest-runner)
("python-twine" ,python-twine)))
(home-page "https://weblate.org/")
(synopsis
"A translation file finder for Weblate, translation tool with tight version control integration")
(description
"A translation file finder for Weblate, translation tool with tight version control integration")
(license license:gpl3+)))
(define-public python-android-stringslib
(let ((commit "599eeb83858fec5292a0f1b87b91c6894eec3ab5")
(revision "1"))
(package
(name "python-android-stringslib")
(version (git-version "0.1.1" revision commit))
(source (origin
(method git-fetch)
(uri (git-reference
(url "https://framagit.org/tyreunom/python-android-strings-lib")
(commit commit)))
(file-name (git-file-name name version))
(sha256
(base32
"003cw8hz2dhwb1n09jny7n94rxks1cxzrsack5rndah2is7zgf3v"))))
(build-system python-build-system)
(arguments
`(#:tests? #f)); no tests
(native-inputs
`(("python-sphinx" ,python-sphinx)))
(home-page "https://framagit.org/tyreunom/python-android-strings-lib")
(synopsis "")
(description "")
(license license:expat))))
(define-public transmon
(let ((commit "da2f6e891b15cd7b7f26ecc6ece93bfd88ea655a")
(revision "0"))
(package
(name "transmon")
(version (git-version "0.1.1" revision commit))
(source (origin
(method git-fetch)
(uri (git-reference
(url "https://framagit.org/tyreunom/transmon")
(commit commit)))
(file-name (git-file-name name version))
(sha256
(base32
"0d9vwqqfl0khpp6jsnd41r88b1qx07bwkfxizvri6hq7rl09wyjd"))))
(build-system python-build-system)
(arguments
`(#:tests? #f)); no tests
(propagated-inputs
`(("python-android-stringslib" ,python-android-stringslib)
("python-polib" ,python-polib)
("python-pygit2" ,python-pygit2)
("python-translation-finder" ,python-translation-finder)))
(home-page "https://framagit.org/tyreunom/transmon")
(synopsis "")
(description "")
(license license:agpl3+))))
;;; i18n monitoring website.
;;;
;;; Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
;;;
;;; This program is free software: you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation, either version 3 of the License, or
;;; (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program. If not, see <http://www.gnu.org/licenses/>.
(use-modules (data-analysis) (ice-9 match) (json))
(define data-file (cadr (command-line)))
(define full-output-file (string-append "public/images/full-" (basename data-file) ".png"))
(define truncated-output-file (string-append "public/images/" (basename data-file) ".png"))
(define (generate-graph column-count data-file output-file)
(let ((plot-file (tmpnam)))
(with-output-to-file plot-file
(lambda _
(format #t "set style data histograms~%")
(format #t "set style fill solid~%")
(format #t "set xtics rotate~%")
(format #t "set terminal png size ~a,600~%" (+ 100 (* column-count 14)))
(format #t "set key at graph 0.05, 0.9~%")
(format #t "set output '~a'~%" output-file)
(format #t "plot '~a' using 2:xtic(1) with boxes title 'done (100%)' linecolor rgb '#88ee88',"
data-file)
(format #t " '' using 3 with boxes title 'almost done (> 90%)' linecolor rgb '#aaff99',")
(format #t " '' using 4 with boxes title 'good progress (> 70%)' linecolor rgb '#eedd99',")
(format #t " '' using 5 with boxes title 'in progress (> 30%)' linecolor rgb '#eecc88',")
(format #t " '' using 6 with boxes title 'started (> 0%)' linecolor rgb '#ffaa99'~%")))
(system* "gnuplot" plot-file)
(delete-file plot-file)))
(define (points r)
(match r
((lang c1 c2 c3 c4 c5)
(+ (- c1 c2) (* 0.9 (- c2 c3)) (* 0.7 (- c3 c4)) (* 0.3 (- c4 c5)) (* 0.1 c5)))))
(define (generate-data-file formated-data-file lang-data)
(with-output-to-file formated-data-file
(lambda _
(format #t "#lang any 90 70 30 0~%")
(for-each
(lambda (lang)
(match lang
((lang c1 c2 c3 c4 c5)
(format #t "~a ~a ~a ~a ~a ~a~%" lang c1 c2 c3 c4 c5))))
(sort lang-data (lambda (r1 r2)
(> (points r1) (points r2))))))))
;; Create a file for one day, every lang in X
(catch #t
(lambda ()
(let* ((tmp-full (tmpnam))
(tmp-truncated (tmpnam))
(data (with-input-from-file data-file json->scm))
(states (map (lambda (d) (array->list (assoc-ref d "state")))
(array->list data)))
;; We can have more than one translation state per project, so we
;; flatten them here:
(states (apply append states))
(stats (map (lambda (s) (assoc-ref s "state")) states))
(stats-truncated (filter (lambda (s) (and s (not (null? s)))) stats))
(full-lang (generate-lang-data stats))
(truncated-lang (generate-lang-data stats-truncated)))
(generate-data-file tmp-full full-lang)
(generate-data-file tmp-truncated truncated-lang)
(generate-graph (length full-lang) tmp-full full-output-file)
(generate-graph (length truncated-lang) tmp-truncated truncated-output-file)
(delete-file tmp-full)
(delete-file tmp-truncated)))
(lambda _
(format #t "Unable to produce graphs from ~a. Skipping.~%" data-file)))
;;; i18n monitoring website.
;;;
;;; Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
;;;
;;; This program is free software: you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation, either version 3 of the License, or
;;; (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program. If not, see <http://www.gnu.org/licenses/>.
(use-modules (data-analysis) (ice-9 match) (json))
(define data-files (cdr (command-line)))
(define (count-cat cat-stats n)
(length (filter (lambda (cat) (and cat (>= cat n))) cat-stats)))
(define (generate-graph data-file output-file)
(let ((plot-file (tmpnam)))
(with-output-to-file plot-file
(lambda _
(format #t "set style data histograms~%")
(format #t "set style fill solid~%")
(format #t "set xtics rotate~%")
(format #t "set terminal png size 800,600~%")
(format #t "set key outside~%")
(format #t "set xtics auto~%")
(format #t "set output '~a'~%" output-file)
(format #t "plot '~a' using 2:xtic(int($0)%3 == 0 ? stringcolumn(1) : '') with boxes title 'done (100%)' linecolor rgb '#88ee88',"
data-file)
(format #t " '' using 3 with boxes title 'almost done (> 90%)' linecolor rgb '#aaff99',")
(format #t " '' using 4 with boxes title 'good progress (> 70%)' linecolor rgb '#eedd99',")
(format #t " '' using 5 with boxes title 'in progress (> 30%)' linecolor rgb '#eecc88',")
(format #t " '' using 6 with boxes title 'started (> 0%)' linecolor rgb '#ffaa99'~%")))
(system* "gnuplot" plot-file)
(delete-file plot-file)))
(define (generate-data-file formated-data-file lang-data)
(with-output-to-file formated-data-file
(lambda _
(format #t "#lang any 90 70 30 0~%")
(for-each
(lambda (lang)
(match lang
((lang c1 c2 c3 c4 c5)
(format #t "~a ~a ~a ~a ~a ~a~%" lang c1 c2 c3 c4 c5))
((lang . #f)
(format #t "~a 0 0 0 0 0~%" lang))))
lang-data))))
(define (read-data-file file)
(catch #t
(lambda ()
(let* ((data (with-input-from-file file json->scm))
(states (map (lambda (d) (array->list (assoc-ref d "state")))
(array->list data)))
(states (apply append states))
(stats (map (lambda (s) (assoc-ref s "state")) states)))
(filter (lambda (s) (and s (not (null? s)))) stats)))
;; any error? the file is broken
(lambda _
'())))
(define (name->date f)
(substring (basename f) 8 18))