Commit be0bc782 authored by rgaudin's avatar rgaudin

aria2 downloader fixes

- disable --http-accept-gzip if filename looks like a *zip
- Using a provided ca-certificates file for aria2 downloads
- using in-memory metalink download and renaming target if necessary
- concat metalink fname with path to ensure proper windows-friendly paths
- display move paths and sleep
- make sure we don't catch [MEMORY] line as metalink target
parent 0e7eac85
......@@ -65,6 +65,7 @@ matrix:
- wget http://download.kiwix.org/dev/aria2c-darwin.zip
- unzip aria2c-darwin.zip
- rm aria2c-darwin.zip
- wget http://download.kiwix.org/dev/ca-certificates.crt
# Run PyInstaller
- pyinstaller --log-level=DEBUG kiwix-hotspot-macos.spec
......@@ -159,6 +160,7 @@ matrix:
- wget http://download.kiwix.org/dev/aria2c-linux64.zip
- unzip aria2c-linux64.zip
- rm aria2c-linux64.zip
- wget http://download.kiwix.org/dev/ca-certificates.crt
# Install python dependancies
- pip3 install -r requirements-linux.txt
......
......@@ -125,6 +125,7 @@ install:
- if %platform%==x64 appveyor DownloadFile "http://mirror.download.kiwix.org/dev/aria2c-win64.zip" -FileName "aria2c.zip"
- 7z.exe x aria2c.zip
- del aria2c.zip
- appveyor DownloadFile "http://mirror.download.kiwix.org/dev/ca-certificates.crt" -FileName "ca-certificates.crt"
# Run pyinstaller and show warning
- cd C:\projects\kiwix-hotspot
......
......@@ -11,6 +11,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'),
('aria2c', '.'),
('ca-certificates.crt', '.'),
('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'),
('mbr.img', '.'),
......
......@@ -18,6 +18,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'),
('aria2c', '.'),
('ca-certificates.crt', '.'),
('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'),
('mbr.img', '.'),
......
......@@ -27,6 +27,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('kiwix-hotspot-logo.png', '.'),
('mbr.img', '.'),
('aria2c.exe', '.'),
('ca-certificates.crt', '.'),
('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'),
......
......@@ -27,6 +27,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('mbr.img', '.'),
('kiwix-hotspot-logo.png', '.'),
('aria2c.exe', '.'),
('ca-certificates.crt', '.'),
('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'),
......
......@@ -30,7 +30,7 @@ def fetch_catalogs(logger):
for catalog in CATALOGS:
tmp_fpath = os.path.join(tmp_dir, "{}.catalog".format(catalog["name"]))
dlf = download_file(catalog.get("url"), tmp_fpath, logger)
dlf = download_file(catalog.get("url"), tmp_fpath, logger, debug=False)
if dlf.successful:
with open(tmp_fpath, "r") as fp:
catalogs.append(yaml.load(fp.read()))
......
......@@ -4,6 +4,7 @@
import os
import re
import sys
import time
import shutil
import zipfile
import subprocess
......@@ -116,7 +117,7 @@ class RequestedFile(object):
)
def download_file(url, fpath, logger, checksum=None):
def download_file(url, fpath, logger, checksum=None, debug=False):
""" download an URL into a named path and reports progress to logger
......@@ -147,14 +148,18 @@ def download_file(url, fpath, logger, checksum=None):
"--console-log-level=error",
"--summary-interval=1", # display a line with progress every X seconds
"--human-readable={}".format(str(logger.on_tty).lower()),
"--ca-certificate={}".format(os.path.join(data_dir, "ca-certificates.crt")),
]
args += ["--http-accept-gzip=true"] # only for catalog?
# zip* files might send incorrect gzip header and get auto-extracted by aria2
if not fname.endswith("zip"):
args += ["--http-accept-gzip=true"]
args += [url]
aria2c = subprocess.Popen(
args, stdout=subprocess.PIPE, universal_newlines=True, **startup_info_args()
)
# logger.std(" ".join(args))
if debug:
logger.std(" ".join(args))
if not logger.on_tty:
logger.ascii_progressbar(0, 100)
......@@ -177,8 +182,14 @@ def download_file(url, fpath, logger, checksum=None):
except Exception:
downloaded_size, total_size = 1, -1
logger.ascii_progressbar(downloaded_size, total_size)
if metalink_target is None and line.startswith("FILE:"):
metalink_target = line.split(":")[-1].strip()
if (
metalink_target is None
and line.startswith("FILE:")
and "[MEMORY]" not in line
):
metalink_target = os.path.join(
output_dir, os.path.basename(line.split(":")[-1].strip())
)
if aria2c.poll() is None:
try:
......@@ -196,8 +207,10 @@ def download_file(url, fpath, logger, checksum=None):
checksum,
)
if metalink_target is not None:
shutil.move(metalink_target, fpath)
if metalink_target is not None and metalink_target != fpath:
logger.std("mv {src} {dst}".format(src=metalink_target, dst=fpath))
time.sleep(5)
os.replace(metalink_target, fpath)
return RequestedFile.from_download(url, fpath, os.path.getsize(fpath))
......@@ -215,7 +228,7 @@ def download_if_missing(url, fpath, logger, checksum=None):
elif os.path.exists(fpath):
return RequestedFile.from_disk(url, fpath)
return download_file(url, fpath, logger, checksum)
return download_file(url, fpath, logger, checksum, debug=True)
def test_connection(proxies=None):
......
......@@ -163,7 +163,6 @@ def generate_homepage(logger, options):
kiwix_fqdn = "kiwix.{fqdn}".format(fqdn=fqdn)
for package_id in options["packages"]:
package = get_package(logger, package_id)
print(package)
cards.append(
{
"url": "http://{fqdn}/{id}".format(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment