Commit 9bc25112 authored by rgaudin's avatar rgaudin
Browse files

replaced python-requests based downloader with aria2c-bin one

parent 50c76b05
......@@ -20,3 +20,4 @@ qemu-system-arm
xz-*.tar.gz
xz-*
etcher-cli
aria2c
......@@ -31,6 +31,11 @@ build-pibox-image:
- mkdir -p etcher-cli
- tar xf balena-etcher-cli-1.4.8-linux-x64.tar.gz -C etcher-cli --strip-components=1
# download aria2c (not used but packaged)
- wget http://download.kiwix.org/dev/aria2c-linux64.zip
- unzip aria2c-linux64.zip
- rm aria2c-linux64.zip
# Install python dependancies
- pip3 install -U pip
- pip3 install -r requirements-linux.txt
......
......@@ -61,6 +61,11 @@ matrix:
- mkdir -p etcher-cli
- tar xf balena-etcher-cli-1.4.8-darwin-x64.tar.gz -C etcher-cli --strip-components=1
# Bundle aria2c
- wget http://download.kiwix.org/dev/aria2c-darwin.zip
- unzip aria2c-darwin.zip
- rm aria2c-darwin.zip
# Run PyInstaller
- pyinstaller --log-level=DEBUG kiwix-hotspot-macos.spec
......@@ -150,6 +155,11 @@ matrix:
- mkdir -p etcher-cli
- tar xf balena-etcher-cli-1.4.8-linux-x64.tar.gz -C etcher-cli --strip-components=1
# Bundle aria2c
- wget http://download.kiwix.org/dev/aria2c-linux64.zip
- unzip aria2c-linux64.zip
- rm aria2c-linux64.zip
# Install python dependancies
- pip3 install -r requirements-linux.txt
......
2.0.9
* Using aria2 to download content (metalink and resume)
* Using mirror.download.kiwix.org for retrieving catalog (gzip'd)
* system locale to en_US (was en_GB*)
* Updated kiwix-serve to 3.0.1
* Updated to raspbian 2019-07-10 (buster)
......
......@@ -119,6 +119,13 @@ install:
- 7z.exe x etcher-cli.zip
- del etcher-cli.zip
# Download aria2
- cd C:\projects\kiwix-hotspot
- if %platform%==x86 appveyor DownloadFile "http://mirror.download.kiwix.org/dev/aria2c-win32.zip" -FileName "aria2c.zip"
- if %platform%==x64 appveyor DownloadFile "http://mirror.download.kiwix.org/dev/aria2c-win64.zip" -FileName "aria2c.zip"
- 7z.exe x aria2c.zip
- del aria2c.zip
# Run pyinstaller and show warning
- cd C:\projects\kiwix-hotspot
- if %platform%==x86 C:\Python34\python.exe C:\Python34\Scripts\pyinstaller-script.py --log-level=DEBUG kiwix-hotspot-win32.spec
......
......@@ -10,6 +10,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('contents.json', '.'),
('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'),
('aria2c', '.'),
('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'),
('mbr.img', '.'),
......
......@@ -17,6 +17,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('contents.json', '.'),
('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'),
('aria2c', '.'),
('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'),
('mbr.img', '.'),
......
......@@ -26,6 +26,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'),
('mbr.img', '.'),
('aria2c.exe', '.'),
('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'),
......
......@@ -26,6 +26,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'),
('mbr.img', '.'),
('kiwix-hotspot-logo.png', '.'),
('aria2c.exe', '.'),
('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'),
......
......@@ -13,7 +13,7 @@ CATALOGS = [
{
"name": "Kiwix",
"description": "Kiwix ZIM Content",
"url": "http://download.kiwix.org/library/ideascube.yml",
"url": "http://mirror.download.kiwix.org/library/ideascube.yml",
}
]
......@@ -27,11 +27,11 @@ def fetch_catalogs(logger):
try:
for catalog in CATALOGS:
tmpfile = tempfile.NamedTemporaryFile(suffix=".yml", delete=False)
tmpfile.close()
dlf = download_file(catalog.get("url"), tmpfile.name, logger)
tmpfile.seek(0) # reset as we'll read it right away
if dlf.successful:
catalogs.append(yaml.load(tmpfile.read()))
tmpfile.close()
with open(tmpfile.name, "r") as fp:
catalogs.append(yaml.load(fp.read()))
os.unlink(tmpfile.name)
else:
raise ValueError("Unable to download {}".format(catalog.get("url")))
......
......@@ -2,7 +2,6 @@
# vim: ai ts=4 sts=4 et sw=4 nu
import os
import io
import re
import sys
import shutil
......@@ -12,12 +11,17 @@ import subprocess
import requests
from data import data_dir, http_proxy_test_url, https_proxy_test_url
from util import ReportHook, get_checksum, get_cache, get_prefs
from util import get_checksum, get_cache, get_prefs
from backend.util import subprocess_pretty_check_call, startup_info_args
PROXIES = None
FAILURE_RETRIES = 6
szip_exe = os.path.join(data_dir, "7za.exe")
PROXIES = None
if sys.platform == "win32":
aria2_exe = os.path.join(data_dir, "aria2c.exe")
else:
aria2_exe = os.path.join(data_dir, "aria2c")
def read_proxies(load_env=True):
......@@ -112,82 +116,88 @@ class RequestedFile(object):
)
def stream(
url,
write_to=None,
callback=None,
block_size=1024,
proxies=None,
only_first_block=False,
):
""" download an URL without blocking
- retries download on failure (with increasing wait delay)
- feeds a callback to provide progress indication """
# prepare adapter so it retries on failure
session = requests.Session()
# retries up-to FAILURE_RETRIES whichever kind of listed error
retries = requests.packages.urllib3.util.retry.Retry(
total=FAILURE_RETRIES, # total number of retries
connect=FAILURE_RETRIES, # connection errors
read=FAILURE_RETRIES, # read errors
status=2, # failure HTTP status (only those bellow)
redirect=False, # don't fail on redirections
backoff_factor=30, # sleep factor between retries
status_forcelist=[413, 429, 500, 502, 503, 504],
)
retry_adapter = requests.adapters.HTTPAdapter(max_retries=retries)
session.mount("http", retry_adapter) # tied to http and https
req = session.get(
url, stream=True, proxies=proxies if proxies is not None else PROXIES
)
total_size = int(req.headers.get("content-length", 0))
# adjust if we are only requesting first block
if only_first_block and total_size > block_size:
total_size = block_size
total_downloaded = 0
if write_to is not None:
fd = open(write_to, "wb")
else:
fd = io.BytesIO()
for data in req.iter_content(block_size):
callback(data, block_size, total_size)
total_downloaded += len(data)
if write_to:
fd.write(data)
# stop downloading/reading if we're just testing first block
if only_first_block:
break
if write_to:
fd.close()
else:
fd.seek(0)
def download_file(url, fpath, logger, checksum=None):
if total_size != 0 and total_downloaded != total_size:
raise AssertionError(
"Downloaded size ({}) is different than expected ({})".format(
total_downloaded, total_size
)
""" download an URL into a named path and reports progress to logger
download is externalized to aria2c binary and progress extracted
from periodic summary
downloads are resumed if possible
supports metalink. if link is metalink, downloads both then replace
actual target (aria2 doesn't allow setting target for metalink
as it can be multiple files) """
output_dir, fname = os.path.split(fpath)
args = [aria2_exe, f"--dir={output_dir}", f"--out={fname}"]
args += [
"--connect-timeout=60",
"--max-file-not-found=5",
"--max-tries=5",
"--retry-wait=60",
"--timeout=60",
"--follow-metalink=mem",
"--allow-overwrite=true",
"--always-resume=false",
"--max-resume-failure-tries=1",
"--auto-file-renaming=false",
"--download-result=full",
"--log-level=error",
"--console-log-level=error",
"--summary-interval=1", # display a line with progress every X seconds
"--human-readable={}".format(str(logger.on_tty).lower()),
]
args += ["--http-accept-gzip=true"] # only for catalog?
args += [url]
aria2c = subprocess.Popen(args, stdout=subprocess.PIPE, text=True)
# logger.std(" ".join(args))
if not logger.on_tty:
logger.ascii_progressbar(0, 100)
metalink_target = None
for line in iter(aria2c.stdout.readline, ""):
line = line.strip()
# [#915371 5996544B/90241109B(6%) CN:4 DL:1704260B ETA:49s]
if line.startswith("[#") and line.endswith("]"): # quick check, no re
if logger.on_tty:
logger.flash(line + " ")
else:
try:
downloaded_size, total_size = [
int(x)
for x in list(
re.search(r"\s([0-9]+)B\/([0-9]+)B", line).groups()
)
]
except Exception:
downloaded_size, total_size = 1, -1
logger.ascii_progressbar(downloaded_size, total_size)
if metalink_target is None and line.startswith("FILE:"):
metalink_target = line.split(":")[-1].strip()
if aria2c.poll() is None:
try:
aria2c.wait(timeout=2)
except subprocess.TimeoutError:
aria2c.terminate()
logger.std("") # clear last \r
if not aria2c.returncode == 0:
return RequestedFile.from_failure(
url,
fpath,
ValueError("aria2c returned {}".format(aria2c.returncode)),
checksum,
)
return total_downloaded, write_to if write_to else fd
if metalink_target is not None:
shutil.move(metalink_target, fpath)
def download_file(url, fpath, logger, checksum=None):
""" downloads expected URL+sum to file while showing progress to logger """
hook = ReportHook(logger.raw_std).reporthook
try:
size, path = stream(url, fpath, callback=hook)
except Exception as exp:
return RequestedFile.from_failure(url, fpath, exp, checksum)
return RequestedFile.from_download(url, fpath, size)
return RequestedFile.from_download(url, fpath, os.path.getsize(fpath))
def download_if_missing(url, fpath, logger, checksum=None):
......@@ -207,19 +217,13 @@ def download_if_missing(url, fpath, logger, checksum=None):
def test_connection(proxies=None):
try:
hook = ReportHook(print).reporthook
for kind, url in (
("HTTP", http_proxy_test_url),
("HTTPS", https_proxy_test_url),
):
size, fd = stream(
url, proxies=proxies, callback=hook, only_first_block=True
)
except Exception:
return False, kind
else:
return size > 0, None
for kind, url in (("HTTP", http_proxy_test_url), ("HTTPS", https_proxy_test_url)):
try:
req = requests.head(url, proxies=proxies or PROXIES, timeout=20)
req.raise_for_status()
except Exception:
return False, kind
return True, None
def get_content_cache(content, folder, is_cache_folder=False):
......@@ -289,7 +293,7 @@ def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger):
[szip_exe, "x", "-so", archive_fpath],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**startup_info_args()
**startup_info_args(),
)
logger.std("Call: " + str(uncompress.args))
......@@ -299,7 +303,7 @@ def win_unarchive_compressed_tar_pipe(archive_fpath, dest_folder, logger):
stdin=uncompress.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**startup_info_args()
**startup_info_args(),
)
logger.std("Call: " + str(untar.args))
......
......@@ -97,6 +97,37 @@ class Logger(ProgressHelper):
self.stg_tag = self.text_buffer.create_tag("stg", foreground="purple")
self.run_pulse()
@property
def on_tty(self):
return False
def flash(self, line):
pass
def ascii_progressbar(self, current, total):
width = 60
avail_dots = width - 2
if total == -1:
line = "unknown size"
elif current >= total:
line = "[" + "." * avail_dots + "] 100%\n"
else:
ratio = min(float(current) / total, 1.)
shaded_dots = min(int(ratio * avail_dots), avail_dots)
percent = min(int(ratio * 100), 100)
line = (
"["
+ "." * shaded_dots
+ " " * (avail_dots - shaded_dots)
+ "] "
+ str(percent)
+ "%\r"
)
if getattr(self, "_last_progress_line", None) != line:
self.raw_std(line)
setattr(self, "_last_progress_line", line)
def step(self, step):
GLib.idle_add(self.main_thread_step, step)
......
......@@ -339,45 +339,17 @@ def human_readable_size(size, binary=True):
return output
class ReportHook:
def __init__(self, writter):
self._current_size = 0
self.width = 60
self._last_line = None
self._writter = writter
self.reporthook(0, 0, 100) # display empty bar as we start
def reporthook(self, chunk, chunk_size, total_size):
if chunk != 0:
self._current_size += chunk_size
avail_dots = self.width - 2
if total_size == -1:
line = "unknown size"
elif self._current_size >= total_size:
line = "[" + "." * avail_dots + "] 100%\n"
else:
ratio = min(float(self._current_size) / total_size, 1.)
shaded_dots = min(int(ratio * avail_dots), avail_dots)
percent = min(int(ratio * 100), 100)
line = (
"["
+ "." * shaded_dots
+ " " * (avail_dots - shaded_dots)
+ "] "
+ str(percent)
+ "%\r"
)
if line != self._last_line:
self._last_line = line
self._writter(line)
class CLILogger(ProgressHelper):
def __init__(self):
super(CLILogger, self).__init__()
@property
def on_tty(self):
return True
def flash(self, line):
self.raw_std(line + "\r")
def step(self, step, end=None):
self.p("--> {}".format(step), color="34", end=end)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment