Commit b496b52f authored by rgaudin's avatar rgaudin
Browse files

Use a proxies configuration for downloads

- introduced a preferences JSON file (stored in homedir) via get_prefs() and save_prefs()
- preferences can contain HTTP_PROXY and HTTPS_PROXY values
- introduced a proxies variable via get_proxies()
- proxies conf has "http" and "https" keys.
those replicates the preference ones and contains full proxy URI: http://username:pass@server:port
- preferences and proxies are read on startup
- when present, proxies are superseeded by environment variables (HTTP_PROXY and HTTPS_PROXY)
- proxy URI in prefs file or env can be socks:// (useful for tests using SSH)
- test_connection() tests one HTTP and one HTTPS URLs
parent 4bd7df19
......@@ -11,12 +11,40 @@ import subprocess
import requests
from data import data_dir
from util import ReportHook, get_checksum, get_cache
from data import data_dir, http_proxy_test_url, https_proxy_test_url
from util import ReportHook, get_checksum, get_cache, get_prefs
from backend.util import subprocess_pretty_check_call, startup_info_args
FAILURE_RETRIES = 6
szip_exe = os.path.join(data_dir, "7za.exe")
PROXIES = None
def read_proxies(load_env=True):
""" read proxy configuration from pref file or ENV """
proxies = {}
if get_prefs().get("HTTP_PROXY", None):
proxies.update({"http": get_prefs().get("HTTP_PROXY")})
if get_prefs().get("HTTPS_PROXY", None):
proxies.update({"https": get_prefs().get("HTTPS_PROXY")})
if load_env:
# environment variables overwrites preferences
if os.getenv("HTTP_PROXY", None):
proxies.update({"http": os.getenv("HTTP_PROXY")})
if os.getenv("HTTPS_PROXY", None):
proxies.update({"https": os.getenv("HTTPS_PROXY")})
return proxies
def get_proxies(load_env=True, force_reload=False):
""" cached-shortcut to PROXIES """
global PROXIES
if PROXIES is None or force_reload:
PROXIES = read_proxies()
return PROXIES
class RequestedFile(object):
......@@ -84,7 +112,7 @@ class RequestedFile(object):
)
def stream(url, write_to=None, callback=None, block_size=1024):
def stream(url, write_to=None, callback=None, block_size=1024, proxies=None):
""" download an URL without blocking
- retries download on failure (with increasing wait delay)
......@@ -104,7 +132,9 @@ def stream(url, write_to=None, callback=None, block_size=1024):
)
retry_adapter = requests.adapters.HTTPAdapter(max_retries=retries)
session.mount("http", retry_adapter) # tied to http and https
req = session.get(url, stream=True)
req = session.get(
url, stream=True, proxies=proxies if proxies is not None else PROXIES
)
total_size = int(req.headers.get("content-length", 0))
total_downloaded = 0
......@@ -125,7 +155,11 @@ def stream(url, write_to=None, callback=None, block_size=1024):
fd.seek(0)
if total_size != 0 and total_downloaded != total_size:
raise AssertionError("Downloaded size is different than expected")
raise AssertionError(
"Downloaded size ({}) is different than expected ({})".format(
total_downloaded, total_size
)
)
return total_downloaded, write_to if write_to else fd
......@@ -157,6 +191,20 @@ def download_if_missing(url, fpath, logger, checksum=None):
return download_file(url, fpath, logger, checksum)
def test_connection(proxies=None):
try:
hook = ReportHook(print).reporthook
for kind, url in (
("HTTP", http_proxy_test_url),
("HTTPS", https_proxy_test_url),
):
size, fd = stream(url, proxies=proxies, callback=hook)
except Exception:
return False, kind
else:
return size > 0, None
def get_content_cache(content, folder, is_cache_folder=False):
""" shortcut to content's fpath from build_folder or cache_folder """
......
......@@ -26,6 +26,9 @@ ansiblecube_path = os.path.join(data_dir, "ansiblecube")
mirror = "http://download.kiwix.org"
sdcard_sizes = (8, 16, 32, 64, 128, 200, 256)
http_proxy_test_url = "http://download.kiwix.org/favicon.ico"
https_proxy_test_url = "https://download.kiwix.org/favicon.ico"
ideascube_languages = [
("am", u"አማርኛ"),
("ar", u"\u0627\u0644\u0639\u0631\u0628\u064a\u0651\u0629"),
......
......@@ -4,6 +4,7 @@
import os
import re
import sys
import json
import data
import signal
import base64
......@@ -16,6 +17,7 @@ import tempfile
import datetime
import threading
import collections
from urllib.parse import urlparse
try:
scandir_func = os.scandir
......@@ -28,10 +30,14 @@ import pytz
from path import Path
import humanfriendly
if sys.platform == "win32":
from win32com.shell import shellcon, shell
ONE_MiB = 2 ** 20
ONE_GiB = 2 ** 30
ONE_GB = int(1e9)
EXFAT_FORBIDDEN_CHARS = ["/", "\\", ":", "*", "?", '"', "<", ">", "|"]
PREFERENCES = None
STAGES = collections.OrderedDict(
......@@ -533,3 +539,70 @@ def get_folder_size(path):
elif entry.is_dir():
total += get_folder_size(entry.path)
return total
def split_proxy(proxy_url):
""" return a (username:password@host, port) tuple from proxy URL """
try:
url = urlparse(proxy_url)
if url.port:
return url.netloc[: -(len(str(url.port)) + 1)], str(url.port)
return url.netloc, ""
except Exception:
parts = proxy_url.rsplit(":", 1)
return parts[0], parts[1] if len(parts) > 1 else ""
def get_prefs_path():
""" full path to our preferences JSON file """
fname = "kiwix-hotspot.prefs"
homedir = os.path.expanduser("~")
if sys.platform == "win32":
try:
prefsdir = shell.SHGetFolderPath(0, shellcon.CSIDL_LOCAL_APPDATA, 0, 0)
except Exception:
prefsdir = homedir
elif sys.platform == "linux":
prefsdir = os.path.join(homedir, ".config")
elif sys.platform == "darwin":
prefsdir = os.path.join(homedir, "Library", "Application Support")
else:
raise NotImplementedError("OS Not Supported")
os.makedirs(prefsdir, exist_ok=True) # create folder should it not exist
return os.path.join(prefsdir, fname)
def read_preferences():
""" read and parse JSON preference file """
fpath = get_prefs_path()
if os.path.exists(fpath):
try:
with open(fpath, "r") as fd:
return json.load(fd)
except Exception as exp:
print("Failed to parse prefs at {}: {}".format(fpath, exp))
return {}
def get_prefs(force_reload=False):
""" cached-shortcut to PREFERENCES """
global PREFERENCES
if PREFERENCES is None or force_reload:
PREFERENCES = read_preferences()
return PREFERENCES
def save_prefs(prefs, auto_reload=True):
""" save a passed preferences dict to the preferences file on disk """
fpath = get_prefs_path()
try:
with open(fpath, "w") as fd:
json.dump(prefs, fd, indent=4)
except Exception as exp:
print("Failed to save prefs to {}: {}".format(fpath, exp))
return False
if auto_reload:
get_prefs(force_reload=True)
return True
......@@ -17,5 +17,6 @@ path.py==11.0
psutil==5.4.5
humanfriendly==4.12
requests==2.18.4
requests[socks]==2.18.4
xlib==0.21
scandir==1.9.0
......@@ -16,4 +16,5 @@ path.py==11.0
psutil==5.4.5
humanfriendly==4.12
requests==2.18.4
requests[socks]==2.18.4
scandir==1.9.0
......@@ -16,4 +16,5 @@ path.py==11.0
psutil==5.4.5
humanfriendly==4.12
requests==2.18.4
requests[socks]==2.18.4
scandir==1.9.0
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment