Commit 9bc25112 authored by rgaudin's avatar rgaudin
Browse files

replaced python-requests based downloader with aria2c-bin one

parent 50c76b05
...@@ -20,3 +20,4 @@ qemu-system-arm ...@@ -20,3 +20,4 @@ qemu-system-arm
xz-*.tar.gz xz-*.tar.gz
xz-* xz-*
etcher-cli etcher-cli
aria2c
...@@ -31,6 +31,11 @@ build-pibox-image: ...@@ -31,6 +31,11 @@ build-pibox-image:
- mkdir -p etcher-cli - mkdir -p etcher-cli
- tar xf balena-etcher-cli-1.4.8-linux-x64.tar.gz -C etcher-cli --strip-components=1 - tar xf balena-etcher-cli-1.4.8-linux-x64.tar.gz -C etcher-cli --strip-components=1
# download aria2c (not used but packaged)
- wget http://download.kiwix.org/dev/aria2c-linux64.zip
- unzip aria2c-linux64.zip
- rm aria2c-linux64.zip
# Install python dependancies # Install python dependancies
- pip3 install -U pip - pip3 install -U pip
- pip3 install -r requirements-linux.txt - pip3 install -r requirements-linux.txt
......
...@@ -61,6 +61,11 @@ matrix: ...@@ -61,6 +61,11 @@ matrix:
- mkdir -p etcher-cli - mkdir -p etcher-cli
- tar xf balena-etcher-cli-1.4.8-darwin-x64.tar.gz -C etcher-cli --strip-components=1 - tar xf balena-etcher-cli-1.4.8-darwin-x64.tar.gz -C etcher-cli --strip-components=1
# Bundle aria2c
- wget http://download.kiwix.org/dev/aria2c-darwin.zip
- unzip aria2c-darwin.zip
- rm aria2c-darwin.zip
# Run PyInstaller # Run PyInstaller
- pyinstaller --log-level=DEBUG kiwix-hotspot-macos.spec - pyinstaller --log-level=DEBUG kiwix-hotspot-macos.spec
...@@ -150,6 +155,11 @@ matrix: ...@@ -150,6 +155,11 @@ matrix:
- mkdir -p etcher-cli - mkdir -p etcher-cli
- tar xf balena-etcher-cli-1.4.8-linux-x64.tar.gz -C etcher-cli --strip-components=1 - tar xf balena-etcher-cli-1.4.8-linux-x64.tar.gz -C etcher-cli --strip-components=1
# Bundle aria2c
- wget http://download.kiwix.org/dev/aria2c-linux64.zip
- unzip aria2c-linux64.zip
- rm aria2c-linux64.zip
# Install python dependancies # Install python dependancies
- pip3 install -r requirements-linux.txt - pip3 install -r requirements-linux.txt
......
2.0.9 2.0.9
* Using aria2 to download content (metalink and resume)
* Using mirror.download.kiwix.org for retrieving catalog (gzip'd)
* system locale to en_US (was en_GB*) * system locale to en_US (was en_GB*)
* Updated kiwix-serve to 3.0.1 * Updated kiwix-serve to 3.0.1
* Updated to raspbian 2019-07-10 (buster) * Updated to raspbian 2019-07-10 (buster)
......
...@@ -119,6 +119,13 @@ install: ...@@ -119,6 +119,13 @@ install:
- 7z.exe x etcher-cli.zip - 7z.exe x etcher-cli.zip
- del etcher-cli.zip - del etcher-cli.zip
# Download aria2
- cd C:\projects\kiwix-hotspot
- if %platform%==x86 appveyor DownloadFile "http://mirror.download.kiwix.org/dev/aria2c-win32.zip" -FileName "aria2c.zip"
- if %platform%==x64 appveyor DownloadFile "http://mirror.download.kiwix.org/dev/aria2c-win64.zip" -FileName "aria2c.zip"
- 7z.exe x aria2c.zip
- del aria2c.zip
# Run pyinstaller and show warning # Run pyinstaller and show warning
- cd C:\projects\kiwix-hotspot - cd C:\projects\kiwix-hotspot
- if %platform%==x86 C:\Python34\python.exe C:\Python34\Scripts\pyinstaller-script.py --log-level=DEBUG kiwix-hotspot-win32.spec - if %platform%==x86 C:\Python34\python.exe C:\Python34\Scripts\pyinstaller-script.py --log-level=DEBUG kiwix-hotspot-win32.spec
......
...@@ -10,6 +10,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'], ...@@ -10,6 +10,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('contents.json', '.'), ('contents.json', '.'),
('etcher.gif', '.'), ('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'), ('kiwix-hotspot-logo.png', '.'),
('aria2c', '.'),
('ansiblecube', 'ansiblecube'), ('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'), ('etcher-cli', 'etcher-cli'),
('mbr.img', '.'), ('mbr.img', '.'),
......
...@@ -17,6 +17,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'], ...@@ -17,6 +17,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('contents.json', '.'), ('contents.json', '.'),
('etcher.gif', '.'), ('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'), ('kiwix-hotspot-logo.png', '.'),
('aria2c', '.'),
('ansiblecube', 'ansiblecube'), ('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'), ('etcher-cli', 'etcher-cli'),
('mbr.img', '.'), ('mbr.img', '.'),
......
...@@ -26,6 +26,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'], ...@@ -26,6 +26,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'), ('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'), ('kiwix-hotspot-logo.png', '.'),
('mbr.img', '.'), ('mbr.img', '.'),
('aria2c.exe', '.'),
('ansiblecube', 'ansiblecube'), ('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot'), ('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'), ('C:\Program Files\qemu', 'qemu'),
......
...@@ -26,6 +26,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'], ...@@ -26,6 +26,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'), ('etcher.gif', '.'),
('mbr.img', '.'), ('mbr.img', '.'),
('kiwix-hotspot-logo.png', '.'), ('kiwix-hotspot-logo.png', '.'),
('aria2c.exe', '.'),
('ansiblecube', 'ansiblecube'), ('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot'), ('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'), ('C:\Program Files\qemu', 'qemu'),
......
...@@ -13,7 +13,7 @@ CATALOGS = [ ...@@ -13,7 +13,7 @@ CATALOGS = [
{ {
"name": "Kiwix", "name": "Kiwix",
"description": "Kiwix ZIM Content", "description": "Kiwix ZIM Content",
"url": "http://download.kiwix.org/library/ideascube.yml", "url": "http://mirror.download.kiwix.org/library/ideascube.yml",
} }
] ]
...@@ -27,11 +27,11 @@ def fetch_catalogs(logger): ...@@ -27,11 +27,11 @@ def fetch_catalogs(logger):
try: try:
for catalog in CATALOGS: for catalog in CATALOGS:
tmpfile = tempfile.NamedTemporaryFile(suffix=".yml", delete=False) tmpfile = tempfile.NamedTemporaryFile(suffix=".yml", delete=False)
tmpfile.close()
dlf = download_file(catalog.get("url"), tmpfile.name, logger) dlf = download_file(catalog.get("url"), tmpfile.name, logger)
tmpfile.seek(0) # reset as we'll read it right away
if dlf.successful: if dlf.successful:
catalogs.append(yaml.load(tmpfile.read())) with open(tmpfile.name, "r") as fp:
tmpfile.close() catalogs.append(yaml.load(fp.read()))
os.unlink(tmpfile.name) os.unlink(tmpfile.name)
else: else:
raise ValueError("Unable to download {}".format(catalog.get("url"))) raise ValueError("Unable to download {}".format(catalog.get("url")))
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
# vim: ai ts=4 sts=4 et sw=4 nu # vim: ai ts=4 sts=4 et sw=4 nu
import os import os
import io
import re import re
import sys import sys
import shutil import shutil
...@@ -12,12 +11,17 @@ import subprocess ...@@ -12,12 +11,17 @@ import subprocess
import requests import requests
from data import data_dir, http_proxy_test_url, https_proxy_test_url from data import data_dir, http_proxy_test_url, https_proxy_test_url
from util import ReportHook, get_checksum, get_cache, get_prefs from util import get_checksum, get_cache, get_prefs
from backend.util import subprocess_pretty_check_call, startup_info_args from backend.util import subprocess_pretty_check_call, startup_info_args
PROXIES = None
FAILURE_RETRIES = 6 FAILURE_RETRIES = 6
szip_exe = os.path.join(data_dir, "7za.exe") szip_exe = os.path.join(data_dir, "7za.exe")
PROXIES = None
if sys.platform == "win32":
aria2_exe = os.path.join(data_dir, "aria2c.exe")
else:
aria2_exe = os.path.join(data_dir, "aria2c")
def read_proxies(load_env=True): def read_proxies(load_env=True):
...@@ -112,82 +116,88 @@ class RequestedFile(object): ...@@ -112,82 +116,88 @@ class RequestedFile(object):
) )
def stream( def download_file(url, fpath, logger, checksum=None):
url,
write_to=None,
callback=None,
block_size=1024,
proxies=None,
only_first_block=False,
):
""" download an URL without blocking
- retries download on failure (with increasing wait delay)
- feeds a callback to provide progress indication """
# prepare adapter so it retries on failure
session = requests.Session()
# retries up-to FAILURE_RETRIES whichever kind of listed error
retries = requests.packages.urllib3.util.retry.Retry(
total=FAILURE_RETRIES, # total number of retries
connect=FAILURE_RETRIES, # connection errors
read=FAILURE_RETRIES, # read errors
status=2, # failure HTTP status (only those bellow)
redirect=False, # don't fail on redirections
backoff_factor=30, # sleep factor between retries
status_forcelist=[413, 429, 500, 502, 503, 504],
)
retry_adapter = requests.adapters.HTTPAdapter(max_retries=retries)
session.mount("http", retry_adapter) # tied to http and https
req = session.get(
url, stream=True, proxies=proxies if proxies is not None else PROXIES
)
total_size = int(req.headers.get("content-length", 0))
# adjust if we are only requesting first block
if only_first_block and total_size > block_size:
total_size = block_size
total_downloaded = 0 """ download an URL into a named path and reports progress to logger
if write_to is not None:
fd = open(write_to, "wb") download is externalized to aria2c binary and progress extracted
from periodic summary
downloads are resumed if possible
supports metalink. if link is metalink, downloads both then replace
actual target (aria2 doesn't allow setting target for metalink
as it can be multiple files) """
output_dir, fname = os.path.split(fpath)
args = [aria2_exe, f"--dir={output_dir}", f"--out={fname}"]
args += [
"--connect-timeout=60",
"--max-file-not-found=5",
"--max-tries=5",
"--retry-wait=60",
"--timeout=60",
"--follow-metalink=mem",
"--allow-overwrite=true",
"--always-resume=false",
"--max-resume-failure-tries=1",
"--auto-file-renaming=false",
"--download-result=full",
"--log-level=error",
"--console-log-level=error",
"--summary-interval=1", # display a line with progress every X seconds
"--human-readable={}".format(str(logger.on_tty).lower()),
]
args += ["--http-accept-gzip=true"] # only for catalog?
args += [url]
aria2c = subprocess.Popen(args, stdout=subprocess.PIPE, text=True)
# logger.std(" ".join(args))
if not logger.on_tty:
logger.ascii_progressbar(0, 100)
metalink_target = None
for line in iter(aria2c.stdout.readline, ""):
line = line.strip()
# [#915371 5996544B/90241109B(6%) CN:4 DL:1704260B ETA:49s]
if line.startswith("[#") and line.endswith("]"): # quick check, no re
if logger.on_tty:
logger.flash(line + " ")
else: else:
fd = io.BytesIO() try:
downloaded_size, total_size = [
for data in req.iter_content(block_size): int(x)
callback(data, block_size, total_size) for x in list(
total_downloaded += len(data) re.search(r"\s([0-9]+)B\/([0-9]+)B", line).groups()
if write_to: )
fd.write(data) ]
except Exception:
downloaded_size, total_size = 1, -1
logger.ascii_progressbar(downloaded_size, total_size)
if metalink_target is None and line.startswith("FILE:"):
metalink_target = line.split(":")[-1].strip()
# stop downloading/reading if we're just testing first block if aria2c.poll() is None:
if only_first_block: try:
break aria2c.wait(timeout=2)
except subprocess.TimeoutError:
aria2c.terminate()
if write_to: logger.std("") # clear last \r
fd.close()
else:
fd.seek(0)
if total_size != 0 and total_downloaded != total_size: if not aria2c.returncode == 0:
raise AssertionError( return RequestedFile.from_failure(
"Downloaded size ({}) is different than expected ({})".format( url,
total_downloaded, total_size fpath,
) ValueError("aria2c returned {}".format(aria2c.returncode)),
checksum,
) )
return total_downloaded, write_to if write_to else fd if metalink_target is not None:
shutil.move(metalink_target, fpath)
return RequestedFile.from_download(url, fpath, os.path.getsize(fpath))
def download_file(url, fpath, logger, checksum=None):
""" downloads expected URL+sum to file while showing progress to logger """
hook = ReportHook(logger.raw_std).reporthook
try:
size, path = stream(url, fpath, callback=hook)
except Exception as exp:
return RequestedFile.from_failure(url, fpath, exp, checksum)
return RequestedFile.from_download(url, fpath, size)
def download_if_missing(url, fpath, logger, checksum=None): def download_if_missing(url, fpath, logger, checksum=None):
...@@ -207,19 +217,13 @@ def download_if_missing(url, fpath, logger, checksum=None): ...@@ -207,19 +217,13 @@ def download_if_missing(url, fpath, logger, checksum=None):
def test_connection(proxies=None): def test_connection(proxies=None):
for kind, url in (("HTTP", http_proxy_test_url), ("HTTPS", https_proxy_test_url)):
try: try:
hook = ReportHook(print).reporthook req = requests.head(url, proxies=proxies or PROXIES, timeout=20)
for kind, url in ( req.raise_for_status()
("HTTP", http_proxy_test_url),
("HTTPS", https_proxy_test_url),
):
size, fd = stream(
url, proxies=proxies, callback=hook, only_first_block=True
)
except Exception: except Exception:
return False, kind return False, kind
else: return True, None
return size > 0, None
def get_content_cache(content, folder, is_cache_folder=False): def get_content_cache(content, folder, is_cache_folder=False):
...@@ -289,7 +293,7 @@ def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger): ...@@ -289,7 +293,7 @@ def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger):
[szip_exe, "x", "-so", archive_fpath], [szip_exe, "x", "-so", archive_fpath],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
**startup_info_args() **startup_info_args(),
) )
logger.std("Call: " + str(uncompress.args)) logger.std("Call: " + str(uncompress.args))
...@@ -299,7 +303,7 @@ def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger): ...@@ -299,7 +303,7 @@ def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger):
stdin=uncompress.stdout, stdin=uncompress.stdout,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
**startup_info_args() **startup_info_args(),
) )
logger.std("Call: " + str(untar.args)) logger.std("Call: " + str(untar.args))
......
...@@ -97,6 +97,37 @@ class Logger(ProgressHelper): ...@@ -97,6 +97,37 @@ class Logger(ProgressHelper):
self.stg_tag = self.text_buffer.create_tag("stg", foreground="purple") self.stg_tag = self.text_buffer.create_tag("stg", foreground="purple")
self.run_pulse() self.run_pulse()
@property
def on_tty(self):
return False
def flash(self, line):
pass
def ascii_progressbar(self, current, total):
width = 60
avail_dots = width - 2
if total == -1:
line = "unknown size"
elif current >= total:
line = "[" + "." * avail_dots + "] 100%\n"
else:
ratio = min(float(current) / total, 1.)
shaded_dots = min(int(ratio * avail_dots), avail_dots)
percent = min(int(ratio * 100), 100)
line = (
"["
+ "." * shaded_dots
+ " " * (avail_dots - shaded_dots)
+ "] "
+ str(percent)
+ "%\r"
)
if getattr(self, "_last_progress_line", None) != line:
self.raw_std(line)
setattr(self, "_last_progress_line", line)
def step(self, step): def step(self, step):
GLib.idle_add(self.main_thread_step, step) GLib.idle_add(self.main_thread_step, step)
......
...@@ -339,45 +339,17 @@ def human_readable_size(size, binary=True): ...@@ -339,45 +339,17 @@ def human_readable_size(size, binary=True):
return output return output
class ReportHook:
def __init__(self, writter):
self._current_size = 0
self.width = 60
self._last_line = None
self._writter = writter
self.reporthook(0, 0, 100) # display empty bar as we start
def reporthook(self, chunk, chunk_size, total_size):
if chunk != 0:
self._current_size += chunk_size
avail_dots = self.width - 2
if total_size == -1:
line = "unknown size"
elif self._current_size >= total_size:
line = "[" + "." * avail_dots + "] 100%\n"
else:
ratio = min(float(self._current_size) / total_size, 1.)
shaded_dots = min(int(ratio * avail_dots), avail_dots)
percent = min(int(ratio * 100), 100)
line = (
"["
+ "." * shaded_dots
+ " " * (avail_dots - shaded_dots)
+ "] "
+ str(percent)
+ "%\r"
)
if line != self._last_line:
self._last_line = line
self._writter(line)
class CLILogger(ProgressHelper): class CLILogger(ProgressHelper):
def __init__(self): def __init__(self):
super(CLILogger, self).__init__() super(CLILogger, self).__init__()
@property
def on_tty(self):
return True
def flash(self, line):
self.raw_std(line + "\r")
def step(self, step, end=None): def step(self, step, end=None):
self.p("--> {}".format(step), color="34", end=end) self.p("--> {}".format(step), color="34", end=end)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment