Commit 8832c2af authored by rgaudin's avatar rgaudin
Browse files

Fixed #241: Added a Cache Folder management tool

- command line tools:
	- kiwix-hotspot cache --build-dir . show
		list all files in cache, showing which ones are to be kept (latest version)
	- kiwix-hotspot cache --build-dir . clean
		remove all obsolete (non latest or alien) files from cache
	- kiwix-hotspot cache --build-dir . reset
		removes the cache folder completely (option to keep latest master)

- GUI interface
	- Clean button appearing next to build-folder selector when build-dir is set
	- Opens a popup displaying usage and offering Clean or Reset (no option) actions
	- Async, cancellable actions
	- Displays new disk usage and free space after operation completed
parent af82f867
......@@ -12,7 +12,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('kiwix-hotspot-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot')],
hiddenimports=['gui', 'cli', 'image'],
hiddenimports=['gui', 'cli', 'image', 'cache'],
hookspath=['additional-hooks'],
runtime_hooks=[],
excludes=[],
......
......@@ -19,7 +19,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('kiwix-hotspot-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('vexpress-boot', 'vexpress-boot')],
hiddenimports=['gui', 'cli', 'image'],
hiddenimports=['gui', 'cli', 'image', 'cache'],
hookspath=['additional-hooks'],
runtime_hooks=[],
excludes=[],
......
......@@ -32,7 +32,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('C:\Program Files\\7zextra\\7za.dll', '.'),
('C:\Program Files\\7zextra\\7za.exe', '.'),
('C:\Program Files\\7zextra\\7zxa.dll', '.')],
hiddenimports=['gui', 'cli', 'image'],
hiddenimports=['gui', 'cli', 'image', 'cache'],
hookspath=['additional-hooks'],
runtime_hooks=[],
excludes=[],
......
......@@ -32,7 +32,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('C:\Program Files\\7zextra\\x64\\7za.dll', '.'),
('C:\Program Files\\7zextra\\x64\\7za.exe', '.'),
('C:\Program Files\\7zextra\\x64\\7zxa.dll', '.')],
hiddenimports=['gui', 'cli', 'image'],
hiddenimports=['gui', 'cli', 'image', 'cache'],
hookspath=['additional-hooks'],
runtime_hooks=[],
excludes=[],
......
......@@ -5,6 +5,9 @@
import argparse
import sys
import runpy
from multiprocessing import freeze_support
freeze_support() # needed for Windows to support multiprocessing once frozen
if len(sys.argv) == 1:
runpy.run_module("gui")
......@@ -14,9 +17,15 @@ elif sys.argv[1] == "cli":
elif sys.argv[1] == "image":
sys.argv.pop(1)
runpy.run_module("image")
elif sys.argv[1] == "cache":
sys.argv.pop(1)
from cache import main
main()
else:
parser = argparse.ArgumentParser(description="Kiwix Hotspot creation tool")
sub_parser = parser.add_subparsers()
sub_parser.add_parser("cli", help="run it on the console")
sub_parser.add_parser("image", help="prepare a master image")
sub_parser.add_parser("cache", help="manage cache folder to reclaim disk space")
parser.parse_args()
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu
import os
import shutil
from util import human_readable_size
from backend.content import CONTENTS
from backend.content import get_content
from backend.catalog import YAML_CATALOGS
from util import get_cache, get_folder_size, get_free_space_in_dir, get_checksum
def package_is_latest_version(fpath, fname):
""" whether a package (ZIM or ZIP) is in the current catalog """
for catalog in YAML_CATALOGS:
for (package_id, package) in catalog["all"].items():
package.update({"ext": "zip" if package["type"] != "zim" else "zim"})
package.update({"langid": package.get("langid") or package_id})
rfname = "package_{langid}-{version}.{ext}".format(**package)
if rfname == fname and get_checksum(fpath) == package["sha256sum"]:
return "{ext}: {fname}".format(
fname=package["langid"], ext=package["ext"].upper()
)
return False
def is_latest_version(fpath, fname):
""" whether the filename is a usable content """
if fname.startswith("package_"):
return package_is_latest_version(fpath, fname)
for key, content in CONTENTS.items():
if not content["name"] == fname:
continue
if get_checksum(fpath) == content["checksum"]:
return key
return False
def get_cache_file_details(logger, cache_folder, fname):
""" analyzed cache file details (dict) """
fpath = os.path.join(cache_folder, fname)
isdir = os.path.isdir(fpath)
size = get_folder_size(fpath) if isdir else os.path.getsize(fpath)
alien = False
if fname.endswith(".zim") and not fname.startswith("package_"): # alien ZIM
alien = True
if isdir:
alien = True # our cache contains only files
latest = is_latest_version(fpath, fname)
return {
"fname": fname,
"isdir": isdir,
"size": size,
"alien": alien,
"latest": latest,
"keep": alien or latest,
}
def get_analyzed_cache_files(logger, cache_folder):
""" generator for the detailed file dict of cache files """
for fname in os.listdir(cache_folder):
yield get_cache_file_details(logger, cache_folder, fname)
def get_cache_size_and_free_space(build_folder, cache_folder):
""" shortcut to query both cache folder size and build-dir free space """
return (
get_folder_size(cache_folder),
len(os.listdir(cache_folder)),
get_free_space_in_dir(cache_folder),
)
def display_cache_and_free_space(
logger, build_folder, cache_folder, old_cache_size=None, old_free_space=None
):
""" display cache size and free space shortcut """
cache_size, nb_files, free_space = get_cache_size_and_free_space(
build_folder, cache_folder
)
if old_cache_size:
logger.std(
"NEW CACHE SIZE: {} (removed {})".format(
human_readable_size(cache_size),
human_readable_size(old_cache_size - cache_size),
)
)
else:
logger.std(
"CACHE SIZE: {} ({} files)".format(
human_readable_size(cache_size), nb_files
)
)
if old_free_space:
logger.std(
"NEW FREE SPACE: {} (reclaimed {})".format(
human_readable_size(free_space),
human_readable_size(free_space - old_free_space),
)
)
else:
logger.std("FREE SPACE: {}".format(human_readable_size(free_space)))
return cache_size, free_space
def reset_cache(logger, build_folder, cache_folder, **kwargs):
""" wipe out the cache folder, optionnaly keeping latest master """
logger.step("Reseting cache folder: {}".format(cache_folder))
cache_size, free_space = display_cache_and_free_space(
logger, build_folder, cache_folder
)
logger.std("-------------")
if kwargs.get("keep_master"):
master = get_content("pibox_base_image")
master_fpath = os.path.join(cache_folder, master["name"])
if (
os.path.exists(master_fpath)
and get_checksum(master_fpath) == master["checksum"]
):
# latest master to be moved temporarly to build-dir
tmp_master_fpath = os.path.join(
build_folder, ".__tmp--{}".format(master["name"])
)
logger.std("Keeping your latest master aside: {}".format(master["name"]))
try:
shutil.move(master_fpath, tmp_master_fpath)
except Exception as exp:
logger.err("Unable to move your latest master into build-dir. Exiting.")
return 1
logger.std("Removing cache...", end="")
try:
shutil.rmtree(cache_folder)
except Exception as exp:
logger.err("FAILED ({}).".format(exp))
else:
logger.succ("OK.")
logger.std("Recreating cache placeholder.")
cache_folder = get_cache(build_folder)
if kwargs.get("keep_master"):
logger.std("Restoring your latest master.")
try:
shutil.move(tmp_master_fpath, master_fpath)
except Exception as exp:
logger.err("Unable to move back your master file into fresh cache.")
logger.err("Please find your master at: {}".format(tmp_master_fpath))
return 1
logger.std("-------------")
display_cache_and_free_space(
logger, build_folder, cache_folder, cache_size, free_space
)
return 0
def clean_cache(logger, build_folder, cache_folder, **kwargs):
""" remove all obsolete files (keep=False in analyzed detail) from cache """
logger.step("Starting cache cleaner for: {}".format(cache_folder))
cfiles = get_analyzed_cache_files(logger, cache_folder)
cache_size, free_space = display_cache_and_free_space(
logger, build_folder, cache_folder
)
logger.std("-------------")
for cfile in cfiles:
if cfile["keep"]:
logger.std("SKIPPING `{}`".format(cfile["fname"]))
continue
logger.std("REMOVING `{}`... ".format(cfile["fname"]), end="")
try:
os.unlink(os.path.join(cache_folder, cfile["fname"]))
except Exception as exp:
logger.err("FAILED ({}).".format(exp))
else:
logger.succ("OK.")
logger.std("-------------")
display_cache_and_free_space(
logger, build_folder, cache_folder, cache_size, free_space
)
return 0
def list_cache_files(logger, build_folder, cache_folder, **kwargs):
""" colored list of all files in cache with legend (to Keep or to Remove) """
logger.step("Listing cache content for: {}".format(cache_folder))
cfiles = get_analyzed_cache_files(logger, cache_folder)
cache_size, nb_files, free_space = get_cache_size_and_free_space(
build_folder, cache_folder
)
logger.std("-------------\nLEGEND\n-------------")
logger.std("K: to be kept (latest content or alien folder)")
logger.std("R: to be removed (obsolete, damaged or alien file)")
logger.std("F: is a file")
logger.std("D: is a directory")
logger.std("-------------")
claimable = 0
for cfile in cfiles:
if cfile["latest"]:
fmt = logger.succ
elif not cfile["alien"]:
fmt = logger.err
claimable += cfile["size"]
else:
fmt = logger.std
fmt(
" {a} [{s}]\t\t{t} {f}{sf}".format(
a="K" if cfile["keep"] else "R",
s=human_readable_size(cfile["size"]),
f=cfile["fname"],
t="D" if cfile["isdir"] else "F",
sf=" --- [{}]".format(cfile["latest"]) if cfile["latest"] else "",
)
)
logger.std("-------------")
logger.std("TOTAL USED SPACE in cache: {}".format(human_readable_size(cache_size)))
logger.std("TOTAL FREE SPACE in cache: {}".format(human_readable_size(free_space)))
logger.std(
"TOTAL CLAIMABLE SPACE in cache: {}".format(human_readable_size(claimable))
)
return 0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: ai ts=4 sts=4 et sw=4 nu
""" manages the cache folder of a supplied build-dir
- show contents and their status (usable or not)
- clean all not usable contents
- reset the cache folder completely
"""
import os
import sys
import argparse
from backend.content import CONTENTS
from util import CLILogger, get_cache
from backend.catalog import YAML_CATALOGS
from backend.cache import list_cache_files, clean_cache, reset_cache
def init(logger):
""" verify that the CATALOGS and CONTENTS list are populated (not empty) """
logger.step("initializing...", end="")
nb_contents = len(CONTENTS)
nb_packages = sum([len(c["all"]) for c in YAML_CATALOGS])
nums = "({}+{})".format(nb_contents, nb_packages)
if not nb_contents or not nb_packages:
logger.err("FAILED. Check your internet connection. {}".format(nums))
sys.exit(1)
logger.succ("OK {}".format(nums))
def main():
def default(logger, build_folder, cache_folder, **kwargs):
parser.parse_args(["--help"])
parser = argparse.ArgumentParser(description="Cache Folder management tool")
parser.add_argument(
"--build", help="Build Folder containing the cache one", required=True
)
parser.set_defaults(func=default)
subparsers = parser.add_subparsers()
parser_show = subparsers.add_parser("show", help="List files in cache")
parser_show.set_defaults(func=list_cache_files)
parser_clean = subparsers.add_parser(
"clean", help="Remove obsolete files from cache"
)
parser_clean.set_defaults(func=clean_cache)
parser_reset = subparsers.add_parser("reset", help="Reset cache folder completely")
parser_reset.set_defaults(func=reset_cache)
parser_reset.add_argument(
"--keep-master",
help="Keep the latest master image if it exists",
action="store_true",
)
# defaults to help
args = parser.parse_args(["--help"] if len(sys.argv) < 2 else None)
logger = CLILogger()
build_folder = args.build
if not os.path.exists(build_folder) or not os.path.isdir(build_folder):
logger.err("Build folder is not a directory.")
sys.exit(1)
cache_folder = get_cache(build_folder)
# ensure we have a proper list of contents and packages to match against
init(logger)
sys.exit(args.func(logger, build_folder, cache_folder, **args.__dict__))
......@@ -7,6 +7,7 @@ import json
import platform
import tempfile
import threading
import multiprocessing
from collections import OrderedDict
import gi
......@@ -28,6 +29,8 @@ from backend.content import (
import data
import sd_card_info
from util import relpathto
from util import get_cache
from util import CLILogger
from util import check_user_inputs
from version import get_version_str
from util import b64encode, b64decode
......@@ -38,6 +41,8 @@ from util import CancelEvent, ProgressHelper
from run_installation import run_installation
from backend.mount import open_explorer_for_imdisk
from util import human_readable_size, ONE_GB, ONE_MiB
from backend.cache import clean_cache, reset_cache
from backend.cache import get_cache_size_and_free_space
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk, Gdk, GLib, GdkPixbuf, GObject
......@@ -281,6 +286,9 @@ class Application:
# etcher
self.component.menu_etcher.connect("activate", self.activate_menu_etcher)
# cache
self.component.clean_cache_button.connect("clicked", self.activate_menu_cache)
# wifi password
self.component.wifi_password_switch.connect(
"notify::active",
......@@ -316,6 +324,9 @@ class Application:
self.component.timezone_combobox.pack_start(renderer, True)
self.component.timezone_combobox.add_attribute(renderer, "text", 0)
# build-path
self.component.build_path_chooser.connect("file-set", self.changed_build_path)
# output
self.component.sd_card_combobox.connect(
"changed", lambda _: self.update_free_space()
......@@ -777,6 +788,160 @@ class Application:
webbrowser.open(data.etcher_url)
dialog.close()
def changed_build_path(self, widget):
""" display Clean cache button only if build-path is set """
self.component.clean_cache_button.set_visible(
bool(self.component.build_path_chooser.get_filename().strip())
)
def activate_menu_cache(self, widget):
build_folder = self.component.build_path_chooser.get_filename()
cache_folder = get_cache(build_folder)
cache_size, nb_files, free_space = get_cache_size_and_free_space(
build_folder, cache_folder
)
class CacheDialog(Gtk.Dialog):
WIPE_CODE = 2111
def __init__(self, parent):
Gtk.Dialog.__init__(
self,
"Reclaim space by cleaning-up your cache",
parent,
0,
(
"Wipe Cache (quick)",
self.WIPE_CODE,
"Clean Cache (slow)",
Gtk.ResponseType.OK,
"Close",
Gtk.ResponseType.CANCEL,
),
)
self.parent = parent
self.set_default_size(300, 100)
label = Gtk.Label()
label.set_markup(
"\nKiwix Hotspot maintains <b>a cache of all downloaded files</b> "
"and reuse them on future runs.\n"
"\nYou can either <b>wipe the cache completely</b> "
"or <b>only remove obsolete files</b>.\n"
"Obsoletes files are previous version of ZIMs or content packs.\n\n"
"Wiping is almost instantaneous.\n"
"Cleaning takes several minutes as "
"it analyzes files to determine which ones should be kept.\n\n"
"Your cache folder is: <i>{cache}</i>.\n"
"<u>Cache Disk Usage</u>: <b>{du}</b> ({nb} files)\n"
"<u>Free Space</u>: <b>{df}</b>\n".format(
cache=cache_folder,
du=human_readable_size(cache_size),
df=human_readable_size(free_space),
nb=nb_files,
)
)
label.set_alignment(0, 0.5)
self.thread = None
self.run_progressbar = Gtk.ProgressBar()
self.cancel_button = Gtk.Button("Cancel")
self.cancel_button.connect("clicked", self.stop_cache_operation)
box = self.get_content_area()
box.add(label)
box.add(self.run_progressbar)
box.add(self.cancel_button)
box.add(Gtk.Label("")) # spacer
self.show_all()
self.run_progressbar.set_visible(False)
self.cancel_button.set_visible(False)
def start_cache_operation(self, is_wipe):
# do nothing if the thread is running
if self.thread is not None and self.thread.is_alive():
return
# show progress bar and cancel button
self.run_progressbar.set_visible(True)
self.cancel_button.set_label(
"Cancel {}".format("Wiping" if is_wipe else "Cleaning...")
)
self.cancel_button.set_visible(True)
# start progress bar animation
self.timeout_id = GObject.timeout_add(50, self.on_timeout)
self.thread = multiprocessing.Process(
target=reset_cache if is_wipe else clean_cache,
args=(CLILogger(), build_folder, cache_folder),
)
self.thread.start()
def on_timeout(self):
# display post-thread dialog on cancelled thread
if self.thread is not None and not self.thread.is_alive():
self.display_post_cache_operation_dialog()
return False
elif self.thread is None: # stop progress anim if thread not running
return False
new_value = self.run_progressbar.get_fraction() + 0.035
# inverse direction if end reached
if new_value > 1:
new_value = 0
# switch from left-to-right to right-to-left at bounds
self.run_progressbar.set_inverted(
not self.run_progressbar.get_inverted()
)
self.run_progressbar.set_fraction(new_value)
return True # returns True so it continues to get called
def stop_cache_operation(self, *args, **kwargs):
if self.thread is not None and self.thread.is_alive():
self.thread.terminate()
self.thread = None
self.cancel_button.set_visible(False)
self.run_progressbar.set_visible(False)
self.close()
def display_post_cache_operation_dialog(self):
msg_box = Gtk.MessageDialog(
self.parent,
None,
Gtk.MessageType.INFO,
Gtk.ButtonsType.OK,
"Cache Operation Completed",
)
cache_size, nb_files, free_space = get_cache_size_and_free_space(
build_folder, cache_folder
)
content = (
"Cache folder: {cache}.\n"
"Cache Disk Usage: {du} ({nb} files)\n"
"Free Space: {df}\n\n".format(
cache=cache_folder,
du=human_readable_size(cache_size),
df=human_readable_size(free_space),
nb=nb_files,
)
)
msg_box.format_secondary_text(content)
msg_box.set_modal(True)
msg_box.run()
msg_box.destroy()
self.close()
dialog = CacheDialog(self.component.window)
ret = dialog.run()
if ret == CacheDialog.WIPE_CODE:
dialog.start_cache_operation(True)
elif ret == Gtk.ResponseType.OK:
dialog.start_cache_operation(False)
else:
dialog.close()
def installation_done(self, error):
ok = error is None
validate_label(self.component.done_label, ok)
......
......@@ -17,6 +17,13 @@ import datetime
import threading
import collections
try:
scandir_func = os.scandir
except AttributeError:
import scandir
scandir_func = scandir.scandir
import pytz
from path import Path
import humanfriendly
......@@ -357,14 +364,14 @@ class CLILogger(ProgressHelper):
def __init__(self):
super(CLILogger, self).__init__()
def step(self, step):
self.p("--> {}".format(step), color="34")
def step(self, step, end=None):