Commit c892639b authored by rgaudin's avatar rgaudin
Browse files

Issue #214: Improve SD card writing

- add a delay after diskpart to prevent Bad file descriptor error
- speed up process by writing chunks of 25MiB
- properly close device and image on error
- additional logging
- fixed SD challenge offset
- copy is properly cancellable
- dont consider failed-to-write as a complete failure (use success image name)
- in case of failure to write, invite user to use Etcher via File menu entry
parent 5021db51
......@@ -8,6 +8,7 @@ a = Analysis(['pibox-installer/__main__.py'],
binaries=[('qemu-system-arm', '.'), ('qemu-img', '.')],
datas=[('ui.glade', '.'),
('contents.json', '.'),
('etcher.gif', '.'),
('pibox-installer-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('pibox-installer-vexpress-boot', 'pibox-installer-vexpress-boot')],
......
......@@ -15,6 +15,7 @@ a = Analysis(['pibox-installer/__main__.py'],
binaries=[("qemu/" + file, '.') for file in files],
datas=[('ui.glade', '.'),
('contents.json', '.'),
('etcher.gif', '.'),
('pibox-installer-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('pibox-installer-vexpress-boot', 'pibox-installer-vexpress-boot')],
......
......@@ -23,6 +23,7 @@ a = Analysis(['pibox-installer/__main__.py'],
datas=[('ui.glade', '.'),
('contents.json', '.'),
('imdisk.png', '.'),
('etcher.gif', '.'),
('pibox-installer-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('pibox-installer-vexpress-boot', 'pibox-installer-vexpress-boot'),
......
......@@ -23,6 +23,7 @@ a = Analysis(['pibox-installer/__main__.py'],
datas=[('ui.glade', '.'),
('contents.json', '.'),
('imdisk.png', '.'),
('etcher.gif', '.'),
('pibox-installer-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('pibox-installer-vexpress-boot', 'pibox-installer-vexpress-boot'),
......
......@@ -121,80 +121,6 @@ class Emulator:
def resize_image(self, size):
subprocess_pretty_check_call([qemu_img_exe_path, "resize", "-f", "raw", self._image, "{}".format(size)], self._logger)
def copy_image(self, device_name):
self._logger.step("Copy image to sd card")
if os.name == "posix":
image = os.open(self._image, os.O_RDONLY)
device = os.open(device_name, os.O_WRONLY)
elif os.name == "nt":
image = os.open(self._image, os.O_RDONLY | os.O_BINARY)
device = os.open(device_name, os.O_WRONLY | os.O_BINARY)
else:
self._logger.err("platform not supported")
return
total_size = os.lseek(image, 0, os.SEEK_END)
os.lseek(image, 0, os.SEEK_SET)
current_percentage = 0.0
refresh_every = 100 * ONE_MiB
while True:
current_size = os.lseek(image, 0, os.SEEK_CUR)
new_percentage = (100 * current_size) / total_size
if new_percentage != current_percentage:
current_percentage = new_percentage
if current_size % refresh_every == 0:
self._logger.progress(current_percentage / 100)
self._logger.std(
"Copied {copied} of {total} ({pc:.2f}%)"
.format(copied=human_readable_size(current_size),
total=human_readable_size(total_size),
pc=current_percentage))
buf = os.read(image, 4096)
if buf == b"":
break
os.write(device, buf)
self._logger.progress(1)
os.close(image)
self._logger.step("Sync")
os.fsync(device)
os.close(device)
def ensure_written(self, device_name):
self._logger.step("Verify data on SD card")
# reopen image and device (read-only)
if os.name == "posix":
image = os.open(self._image, os.O_RDONLY)
device = os.open(device_name, os.O_RDONLY)
elif os.name == "nt":
image = os.open(self._image, os.O_RDONLY | os.O_BINARY)
device = os.open(device_name, os.O_RDONLY | os.O_BINARY)
else:
self._logger.err("platform not supported")
return
# read a 4MiB random part from the image
buffer_size = ONE_MiB * 4
total_size = os.lseek(image, 0, os.SEEK_END)
offset = random.randint(0, total_size - buffer_size) * .8
os.lseek(image, offset, os.SEEK_SET)
challenge = os.read(image, buffer_size)
os.close(image)
# read same part from the SD card and compare
os.lseek(device, offset, os.SEEK_SET)
sd_challenge = os.read(device, buffer_size)
os.close(device)
assert sd_challenge == challenge
class _RunningInstance:
_emulation = None
_qemu = None
......
import os
import sys
import ctypes
import random
import threading
import subprocess
from util import CLILogger
from util import CLILogger, ONE_MiB, human_readable_size
class CheckCallException(Exception):
......@@ -117,3 +119,111 @@ def get_admin_command(command, from_gui):
.format(command=" ".join(command))]
if sys.platform == "linux":
return ["pkexec"] + command
def open_handles(image_fpath, device_fpath, read_only=False):
img_flag = os.O_RDONLY
dev_flag = os.O_RDONLY if read_only else os.O_WRONLY
if os.name == "posix":
return (os.open(image_fpath, img_flag),
os.open(device_fpath, dev_flag))
elif os.name == "nt":
dev_flag = dev_flag | os.O_BINARY
return (os.open(image_fpath, img_flag),
os.open(device_fpath, dev_flag))
else:
raise NotImplementedError("Platform not supported")
def close_handles(image_fd, device_fd):
try:
os.close(image_fd)
os.close(device_fd)
except Exception:
pass
def write_image_on_card(image_fd, device_fd, logger):
''' copy image content onto device (raw) '''
logger.step("Copy image to sd card")
total_size = os.lseek(image_fd, 0, os.SEEK_END)
os.lseek(image_fd, 0, os.SEEK_SET)
buffer_size = 25 * ONE_MiB
steps = total_size // buffer_size
for step in range(0, steps):
# only update logger every 4 steps (100MiB)
if step % 4 == 0:
logger.progress(step / steps)
logger.std(
"Copied {copied} of {total} ({pc:.2f}%)"
.format(copied=human_readable_size(step * buffer_size),
total=human_readable_size(total_size),
pc=step / steps * 100))
try:
os.write(device_fd, os.read(image_fd, buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
raise
if total_size % buffer_size:
logger.std("Writing last chunk...")
try:
os.write(device_fd, os.read(image_fd, total_size % buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
raise
logger.progress(1)
logger.step("sync")
os.fsync(device_fd)
close_handles(image_fd, device_fd)
def ensure_card_written(image_fd, device_fd, logger):
''' asserts image and device content is same (reads rand 4MiB from both '''
logger.step("Verify data on SD card")
# read a 4MiB random part from the image
buffer_size = 4 * ONE_MiB
total_size = os.lseek(image_fd, 0, os.SEEK_END)
offset = random.randint(0, int((total_size - buffer_size) * .8)) // 512
logger.std("reading {n}b from offset {s} out of {t}b."
.format(n=buffer_size, s=offset, t=total_size))
try:
# read same part from the SD card and compare
os.lseek(image_fd, offset, os.SEEK_SET)
os.lseek(device_fd, offset, os.SEEK_SET)
assert os.read(image_fd, buffer_size) == os.read(device_fd, buffer_size)
except AssertionError:
raise
except Exception as exp:
logger.std("Exception during read: {}".format(exp))
raise
finally:
close_handles(image_fd, device_fd)
def run_sdcard_thread(image_fpath, device_fpath, verify_step, cancel_event, logger):
''' starts a cancelable thread for SD-card I/O operations '''
image_fd, device_fd = open_handles(image_fpath, device_fpath, read_only=verify_step)
thread = threading.Thread(
target=ensure_card_written if verify_step else write_image_on_card,
args=(image_fd, device_fd, logger))
thread.start()
cancel_event.register_thread(
thread=thread, callback=close_handles, callback_args=(image_fd, device_fd))
thread.join() # wait for target to complete
cancel_event.unregister_thread()
......@@ -36,5 +36,6 @@ ideascube_languages = [
]
help_url = "https://framagit.org/ideascube/pibox-installer/wikis/faq"
etcher_url = "https://etcher.io/"
VERSION = "devel"
......@@ -9,6 +9,7 @@ import tzlocal
import os
import sys
import json
import platform
import tempfile
import threading
from util import CancelEvent, ProgressHelper
......@@ -21,6 +22,7 @@ from util import b64encode, b64decode
from util import check_user_inputs
import data
import iso639
import requests
import humanfriendly
import webbrowser
gi.require_version('Gtk', '3.0')
......@@ -74,6 +76,9 @@ class Logger(ProgressHelper):
def err(self, err):
GLib.idle_add(self.main_thread_err, err)
def succ(self, succ):
GLib.idle_add(self.main_thread_succ, succ)
def raw_std(self, std):
GLib.idle_add(self.main_thread_raw_std, std, end="")
......@@ -103,6 +108,9 @@ class Logger(ProgressHelper):
def main_thread_err(self, text):
self.main_thread_text(text, "\n", self.err_tag)
def main_thread_succ(self, text):
self.main_thread_text(text, "\n", self.succ_tag)
def main_thread_raw_std(self, text):
self.main_thread_text(text)
......@@ -143,7 +151,7 @@ class Logger(ProgressHelper):
def main_thread_complete(self):
super(Logger, self).complete()
self.main_thread_text("Installation succeded.", tag=self.succ_tag)
self.main_thread_succ("Installation succeded.")
self.component.run_step_label.set_markup("<b>Done.</b>")
self.progress(1)
......@@ -242,6 +250,10 @@ class Application:
self.component.menu_imdisk.connect(
"activate", self.activate_menu_imdisk)
# etcher
self.component.menu_etcher.connect(
"activate", self.activate_menu_etcher)
# wifi password
self.component.wifi_password_switch.connect("notify::active", lambda switch, state: self.component.wifi_password_revealer.set_reveal_child(not switch.get_active()))
......@@ -536,6 +548,69 @@ class Application:
open_explorer_for_imdisk(self.logger)
dialog.close()
def activate_menu_etcher(self, widget):
class EtcherDialog(Gtk.Dialog):
DL_CODE = 2111
def __init__(self, parent):
Gtk.Dialog.__init__(
self, "Use Etcher to Flash your SD-card", parent, 0,
("Download Latest Etcher", self.DL_CODE,
"Visit Website", Gtk.ResponseType.OK,
Gtk.STOCK_OK, Gtk.ResponseType.CANCEL))
self.set_default_size(500, 300)
label = Gtk.Label()
label.set_markup(
"\nUse <b>Etcher</b> to flash your image onto your SD-card"
". It will also <b>validate</b> that the SD-card "
"has been <b>successfuly written</b>.\n"
"You can even burn the same image "
"on <b>several SD-cards at once</b>.\n")
label.set_alignment(0, 0.5)
label2 = Gtk.Label()
label2.set_markup(
"\nPlease Download and Run <b>Etcher</b> separately.\n")
label2.set_alignment(0, 0.5)
image = Gtk.Image.new_from_file(
os.path.join(data.data_dir, 'etcher.gif'))
box = self.get_content_area()
box.add(label)
box.add(image)
box.add(label2)
self.show_all()
dialog = EtcherDialog(self.component.window)
ret = dialog.run()
if ret == EtcherDialog.DL_CODE:
try:
req = requests.get("https://img.shields.io/github/release"
"/resin-io/etcher.json")
version = req.json().get("value")
base_url = ("https://github.com/resin-io/etcher/releases/"
"download/{}/".format(version))
def get_fname():
if sys.platform == "linux":
if platform.machine() == 'x86_64':
return "etcher-electron-{v}-x86_64.AppImage"
return "etcher-electron-{v}-i386.AppImage"
elif sys.platform == "win32":
if platform.machine() == 'AMD64':
return "Etcher-Portable-{v}-x64.exe"
return "Etcher-Portable-{v}-x86.exe"
elif sys.platform == "darwin":
return "Etcher-{v}.dmg"
raise NotImplementedError("platform not supported")
etcher_dl_url = base_url + get_fname().format(v=version[1:])
except Exception as exp:
etcher_dl_url = data.etcher_url
webbrowser.open(etcher_dl_url)
elif ret == Gtk.ResponseType.OK:
webbrowser.open(data.etcher_url)
dialog.close()
def installation_done(self, error):
ok = error is None
validate_label(self.component.done_label, ok)
......
......@@ -7,6 +7,7 @@ from backend.download import download_content, unzip_file
from backend.mount import mount_data_partition, unmount_data_partition, test_mount_procedure, format_data_partition, guess_next_loop_device
from backend.mount import can_write_on, allow_write_on, restore_mode
from backend.util import subprocess_pretty_check_call, subprocess_pretty_call
from backend.util import run_sdcard_thread
from backend.sysreq import host_matches_requirements, requirements_url
import data
from util import human_readable_size, get_cache, ensure_zip_exfat_compatible, EXFAT_FORBIDDEN_CHARS
......@@ -14,6 +15,7 @@ from datetime import datetime
import os
import sys
import re
import time
import shutil
import traceback
......@@ -80,6 +82,8 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
os.close(w)
logger.std("diskpart select disk {} and clean".format(device_number))
subprocess_pretty_check_call(["diskpart"], logger, stdin=r)
logger.std("sleeping for 15s to acknowledge diskpart changes")
time.sleep(15)
# Download Base image
logger.stage('master')
......@@ -272,22 +276,6 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
logger.step("Re-run ansiblecube for move-content")
ansiblecube.run_phase_two(emulation, extra_vars, secret_keys)
# Write image to SD Card
if sd_card:
logger.stage('write')
logger.step("Writting image to SD-card ({})".format(sd_card))
emulator.copy_image(sd_card)
# check that SD card was properly written
try:
emulator.ensure_written(sd_card)
except AssertionError:
logger.err("SD-card content is different than that of image.\n"
"Please check the content of your card and "
"verify that the card is not damaged ("
"often turns read-only silently).")
raise Exception("SD-card content differs from image.")
except Exception as e:
logger.failed(str(e))
......@@ -304,11 +292,48 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
# Set final image filename
os.rename(image_building_path, image_final_path)
logger.complete()
error = None
try:
# Write image to SD Card
if sd_card:
logger.stage('write')
logger.step("Writting image to SD-card ({})".format(sd_card))
try:
run_sdcard_thread(image_final_path, sd_card,
False, cancel_event, logger)
except Exception:
logger.succ("Image created successfuly.")
logger.err(
"Writing your Image to your SD-card failed.\n"
"Please use a third party tool to flash your image "
"onto your SD-card. See File menu for links to Etcher."
)
raise Exception("Failed to write Image to SD-card")
# check that SD card was properly written
try:
run_sdcard_thread(image_final_path, sd_card,
True, cancel_event, logger)
except AssertionError:
logger.err(
"SD-card content is different than that of image.\n"
"Please check the content of your card and "
"verify that the card is not damaged ("
"often turns read-only silently).\n"
"Alternatively, use Etcher (see File > Flash) to "
"flash image onto SD-card and validate transfer.")
raise Exception("SD-card content differs from image.")
except Exception as e:
logger.failed(str(e))
# display traceback on logger
logger.std("\n--- Exception Trace ---\n{exp}\n---"
.format(exp=traceback.format_exc()))
error = e
else:
logger.complete()
error = None
finally:
if sys.platform == "linux" and loop_dev and previous_loop_mode:
logger.step("Restoring loop device ({}) mode".format(loop_dev))
restore_mode(loop_dev, previous_loop_mode, logger)
......
......@@ -230,11 +230,29 @@ class CancelEvent:
def __init__(self):
self._lock = threading.Lock()
self._pids = []
self.thread = None
self.callback = None
self.callback_args = None
def lock(self):
return _CancelEventRegister(self._lock, self._pids)
def register_thread(self, thread, callback, callback_args):
self.thread = thread
self.callback = callback
self.callback_args = callback_args
def unregister_thread(self):
self.thread = None
self.callback = None
self.callback_args = None
def cancel(self):
if self.thread is not None:
self.thread.join(timeout=0)
self.callback(*self.callback_args)
self.unregister_thread()
self._lock.acquire()
for pid in self._pids:
os.kill(pid, signal.SIGTERM)
......@@ -311,6 +329,9 @@ class CLILogger(ProgressHelper):
def err(self, err):
self.p(err, color="31")
def succ(self, succ):
self.p(succ, color="32")
def raw_std(self, std):
sys.stdout.write(std)
......@@ -324,7 +345,7 @@ class CLILogger(ProgressHelper):
def complete(self):
super(CLILogger, self).complete()
self.p("Installation succeded.", color="32")
self.succ("Installation succeded.")
def failed(self, error="?"):
super(CLILogger, self).failed()
......
......@@ -324,6 +324,14 @@
<property name="use_underline">True</property>
</object>
</child>
<child>
<object class="GtkMenuItem" id="menu_etcher">
<property name="label">Flash Image onto SD-card</property>
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="use_underline">True</property>
</object>
</child>
<child>
<object class="GtkMenuItem" id="menu_help">
<property name="label">Open Help / FAQ</property>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment