Commit b44dd354 authored by rgaudin's avatar rgaudin
Browse files

SD-card writing: speed (macOS), verify, cancelable

Use /dev/rdiskX on macOS to increase speed (defered sync)
Using a cancellable thread for writing
Issue #221: Writing 512b chunks on windows (safer, slower)
parent 1e62101a
2.0-rc7
* Fixed catalog update in Qemu (stage 5)
* Better SD-card writer (faster, error-handling, cancellable)
2.0-rc6
* JSON Exports's properties and sub-properties saved in a fixed order
* Failed process raises failed dialog properly [regression]
......
......@@ -143,59 +143,18 @@ def close_handles(image_fd, device_fd):
pass
def write_image_on_card(image_fd, device_fd, logger):
''' copy image content onto device (raw) '''
logger.step("Copy image to sd card")
total_size = os.lseek(image_fd, 0, os.SEEK_END)
os.lseek(image_fd, 0, os.SEEK_SET)
buffer_size = 25 * ONE_MiB
steps = total_size // buffer_size
for step in range(0, steps):
# only update logger every 4 steps (100MiB)
if step % 4 == 0:
logger.progress(step, steps)
logger.std(
"Copied {copied} of {total} ({pc:.2f}%)"
.format(copied=human_readable_size(step * buffer_size),
total=human_readable_size(total_size),
pc=step / steps * 100))
try:
os.write(device_fd, os.read(image_fd, buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
raise
if total_size % buffer_size:
logger.std("Writing last chunk...")
try:
os.write(device_fd, os.read(image_fd, total_size % buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
raise
logger.progress(1)
logger.step("sync")
os.fsync(device_fd)
close_handles(image_fd, device_fd)
def ensure_card_written(image_fd, device_fd, logger):
def ensure_card_written(image_fpath, device_fpath, logger):
''' asserts image and device content is same (reads rand 4MiB from both '''
logger.step("Verify data on SD card")
image_fd, device_fd = open_handles(image_fpath, device_fpath, read_only=True)
# read a 4MiB random part from the image
buffer_size = 4 * ONE_MiB
total_size = os.lseek(image_fd, 0, os.SEEK_END)
offset = random.randint(0, int((total_size - buffer_size) * .8)) // 512
offset = random.randint(0, int((total_size - buffer_size) * .8))
offset -= (offset % 512)
logger.std("reading {n}b from offset {s} out of {t}b."
.format(n=buffer_size, s=offset, t=total_size))
......@@ -203,27 +162,76 @@ def ensure_card_written(image_fd, device_fd, logger):
# read same part from the SD card and compare
os.lseek(image_fd, offset, os.SEEK_SET)
os.lseek(device_fd, offset, os.SEEK_SET)
assert os.read(image_fd, buffer_size) == os.read(device_fd, buffer_size)
except AssertionError:
raise
except Exception as exp:
logger.std("Exception during read: {}".format(exp))
if not os.read(image_fd, buffer_size) == os.read(device_fd, buffer_size):
raise ValueError("Image and SD-card challenge do not match.")
except Exception:
raise
finally:
close_handles(image_fd, device_fd)
def run_sdcard_thread(image_fpath, device_fpath, verify_step, cancel_event, logger):
''' starts a cancelable thread for SD-card I/O operations '''
image_fd, device_fd = open_handles(image_fpath, device_fpath, read_only=verify_step)
thread = threading.Thread(
target=ensure_card_written if verify_step else write_image_on_card,
args=(image_fd, device_fd, logger))
thread.start()
cancel_event.register_thread(
thread=thread, callback=close_handles, callback_args=(image_fd, device_fd))
thread.join() # wait for target to complete
cancel_event.unregister_thread()
class ImageWriterThread(threading.Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._should_stop = False # stop flag
self.exp = None # exception to be re-raised by caller
def stop(self):
self._should_stop = True
def run(self):
image_fpath, device_fpath, logger = self._args
logger.step("Copy image to sd card")
image_fd, device_fd = open_handles(image_fpath, device_fpath)
total_size = os.lseek(image_fd, 0, os.SEEK_END)
os.lseek(image_fd, 0, os.SEEK_SET)
if os.name == "nt":
buffer_size = 512 # safer on windows
logger_break = 1000
else:
buffer_size = 25 * ONE_MiB
logger_break = 4
steps = total_size // buffer_size
for step in range(0, steps):
if self._should_stop:
break
# only update logger every 4 steps (100MiB)
if step % logger_break == 0:
logger.progress(step, steps)
logger.std(
"Copied {copied} of {total} ({pc:.2f}%)"
.format(copied=human_readable_size(step * buffer_size),
total=human_readable_size(total_size),
pc=step / steps * 100))
try:
os.write(device_fd, os.read(image_fd, buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
self.exp = exp
raise
if not self._should_stop and total_size % buffer_size:
logger.std("Writing last chunk...")
try:
os.write(device_fd, os.read(image_fd, total_size % buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
self.exp = exp
raise
logger.progress(1)
logger.step("sync")
if not self._should_stop:
os.fsync(device_fd)
close_handles(image_fd, device_fd)
......@@ -7,7 +7,7 @@ from backend.download import download_content, unzip_file
from backend.mount import mount_data_partition, unmount_data_partition, test_mount_procedure, format_data_partition, guess_next_loop_device
from backend.mount import can_write_on, allow_write_on, restore_mode
from backend.util import subprocess_pretty_check_call, subprocess_pretty_call
from backend.util import run_sdcard_thread
from backend.util import ensure_card_written, ImageWriterThread
from backend.sysreq import host_matches_requirements, requirements_url
import data
from util import human_readable_size, get_cache, ensure_zip_exfat_compatible, EXFAT_FORBIDDEN_CHARS
......@@ -298,8 +298,33 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
logger.stage('write')
logger.step("Writting image to SD-card ({})".format(sd_card))
try:
run_sdcard_thread(image_final_path, sd_card,
False, cancel_event, logger)
imwriter = ImageWriterThread(
args=(image_final_path, sd_card, logger))
cancel_event.register_thread(thread=imwriter)
imwriter.start()
imwriter.join(timeout=2) # make sure it started
while imwriter.is_alive():
pass
imwriter.join(timeout=2)
cancel_event.unregister_thread()
if imwriter.exp is not None:
raise imwriter.exp
logger.std("Done writing ; preparing for verification.")
time.sleep(5)
ensure_card_written(image_final_path, sd_card, logger)
except ValueError:
logger.succ("Image created successfuly.")
logger.err(
"SD-card content is different than that of image.\n"
"Please check the content of your card and "
"verify that the card is not damaged ("
"often turns read-only silently).\n"
"Alternatively, use Etcher (see File > Flash) to "
"flash image onto SD-card and validate transfer."
)
raise Exception("SD-card content verification failed")
except Exception:
logger.succ("Image created successfuly.")
logger.err(
......@@ -309,20 +334,6 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
)
raise Exception("Failed to write Image to SD-card")
# check that SD card was properly written
try:
run_sdcard_thread(image_final_path, sd_card,
True, cancel_event, logger)
except AssertionError:
logger.err(
"SD-card content is different than that of image.\n"
"Please check the content of your card and "
"verify that the card is not damaged ("
"often turns read-only silently).\n"
"Alternatively, use Etcher (see File > Flash) to "
"flash image onto SD-card and validate transfer.")
raise Exception("SD-card content differs from image.")
except Exception as e:
logger.failed(str(e))
......
......@@ -96,7 +96,7 @@ elif sys.platform == "darwin":
devices.append({
"bus_protocol": plist["BusProtocol"],
"device_identifier": plist["DeviceIdentifier"],
"device": plist["DeviceNode"],
"device": plist["DeviceNode"].replace('/dev/disk', '/dev/rdisk'),
"io_registry_entry_name": plist["IORegistryEntryName"],
"media_name": plist["MediaName"],
"media_type": plist["MediaType"],
......
......@@ -246,10 +246,8 @@ class CancelEvent:
def lock(self):
return _CancelEventRegister(self._lock, self._pids)
def register_thread(self, thread, callback, callback_args):
def register_thread(self, thread):
self.thread = thread
self.callback = callback
self.callback_args = callback_args
def unregister_thread(self):
self.thread = None
......@@ -258,8 +256,8 @@ class CancelEvent:
def cancel(self):
if self.thread is not None:
self.thread.join(timeout=0)
self.callback(*self.callback_args)
self.thread.stop()
self.thread.join(timeout=5) # allow proper release of handles
self.unregister_thread()
self._lock.acquire()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment