Commit 479ad4fb authored by rgaudin's avatar rgaudin

Fixed #220: using Etcher-cli to write and verify SD-card

- Using v1.4.5 on Linux and macOS
- Using 1.1.2 on Windows (later versions are buggy)
- etcher-cli used with auto-check and auto-unmount options
- etcher-cli requires sudo on macOS and linux (in win32, we already run as admin)
  so password is asked at writing stage
- no more password on macOS and linux to change SD card device mode on start
  linux still ask for loop device mode change though
parent c26b8d09
......@@ -19,3 +19,4 @@ qemu-img
qemu-system-arm
xz-*.tar.gz
xz-*
etcher-cli
......@@ -48,6 +48,11 @@ matrix:
- wget http://download.kiwix.org/dev/qemu-2.12.0_macOS.tar
- tar xf qemu-2.12.0_macOS.tar
# Bundle Etcher-cli
- wget http://download.kiwix.org/dev/etcher-cli-1.4.5-darwin-x64.tar.gz
- mkdir -p etcher-cli
- tar xf etcher-cli-1.4.5-darwin-x64.tar.gz -C etcher-cli --strip-components=1
# Run PyInstaller
- pyinstaller --log-level=DEBUG kiwix-hotspot-macos.spec
......@@ -120,6 +125,11 @@ matrix:
- wget http://download.kiwix.org/dev/qemu-2.12.0-linux-x86_64.tar.gz
- tar xf qemu-2.12.0-linux-x86_64.tar.gz
# Bundle Etcher-cli
- wget http://download.kiwix.org/dev/etcher-cli-1.4.5-linux-x64.tar.gz
- mkdir -p etcher-cli
- tar xf etcher-cli-1.4.5-linux-x64.tar.gz -C etcher-cli --strip-components=1
# Install python dependancies
- pip3 install -r requirements-linux.txt
......
......@@ -4,6 +4,7 @@
* Added Proxy Configuration Support
* Removed StaticSites catalog as it is no longer available online
* Changed JSON format for WiFi: single `wifi_password` entry
* Using etcher-cli to flash and verify SD-card (all platforms)
2.0-rc12
* macOS binary name (Kiwix Hotspot instead of kiwix-hotspot)
......
......@@ -112,6 +112,14 @@ install:
- del imdiskinst.exe
- dir "C:\Program Files\imdiskinst"
# Download Etcher-cli
- mkdir "C:\Program Files\etcher-cli"
- cd "C:\Program Files\etcher-cli"
- if %platform%==x86 appveyor DownloadFile "http://download.kiwix.org/dev/Etcher-cli-1.1.2-win32-x86.zip" -FileName "etcher-cli.zip"
- if %platform%==x64 appveyor DownloadFile "http://download.kiwix.org/dev/Etcher-cli-1.1.2-win32-x64.zip" -FileName "etcher-cli.zip"
- 7z.exe x etcher-cli.zip
- del etcher-cli.zip
# Run pyinstaller and show warning
- cd C:\projects\kiwix-hotspot
- if %platform%==x86 C:\Python34\python.exe C:\Python34\Scripts\pyinstaller-script.py --log-level=DEBUG kiwix-hotspot-win32.spec
......
......@@ -11,6 +11,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'),
('vexpress-boot', 'vexpress-boot')],
hiddenimports=['gui', 'cli', 'image', 'cache'],
hookspath=['additional-hooks'],
......
......@@ -18,6 +18,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('etcher.gif', '.'),
('kiwix-hotspot-logo.png', '.'),
('ansiblecube', 'ansiblecube'),
('etcher-cli', 'etcher-cli'),
('vexpress-boot', 'vexpress-boot')],
hiddenimports=['gui', 'cli', 'image', 'cache'],
hookspath=['additional-hooks'],
......
......@@ -29,6 +29,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'),
('C:\Program Files\imdiskinst', 'imdiskinst'),
('C:\Program Files\etcher-cli', 'etcher-cli'),
('C:\Program Files\\7zextra\\7za.dll', '.'),
('C:\Program Files\\7zextra\\7za.exe', '.'),
('C:\Program Files\\7zextra\\7zxa.dll', '.')],
......
......@@ -29,6 +29,7 @@ a = Analysis(['kiwix-hotspot/__main__.py'],
('vexpress-boot', 'vexpress-boot'),
('C:\Program Files\qemu', 'qemu'),
('C:\Program Files\imdiskinst', 'imdiskinst'),
('C:\Program Files\etcher-cli', 'etcher-cli'),
('C:\Program Files\\7zextra\\x64\\7za.dll', '.'),
('C:\Program Files\\7zextra\\x64\\7za.exe', '.'),
('C:\Program Files\\7zextra\\x64\\7zxa.dll', '.')],
......
......@@ -3,11 +3,15 @@
import os
import sys
import time
import signal
import ctypes
import random
import tempfile
import threading
import subprocess
import data
from util import CLILogger, ONE_MiB, human_readable_size
......@@ -40,9 +44,10 @@ def startup_info_args():
# distraction.
si = subprocess.STARTUPINFO()
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
cf = subprocess.CREATE_NEW_PROCESS_GROUP
else:
si = None
return {"startupinfo": si}
si = cf = None
return {"startupinfo": si, "creationflags": cf}
def subprocess_pretty_call(
......@@ -130,7 +135,7 @@ def run_as_win_admin(command, logger):
return rc
def get_admin_command(command, from_gui):
def get_admin_command(command, from_gui, log_to=None):
""" updated command to run it as root on macos or linux
from_gui: whether called via GUI. Using cli sudo if not """
......@@ -142,64 +147,16 @@ def get_admin_command(command, from_gui):
return [
"/usr/bin/osascript",
"-e",
'do shell script "{command} 2>&1" '
"with administrator privileges".format(command=" ".join(command)),
'do shell script "{command} 2>&1 {redir}" '
"with administrator privileges".format(
command=" ".join(command), redir=">{}".format(log_to) if log_to else ""
),
]
if sys.platform == "linux":
return ["pkexec"] + command
def open_handles(image_fpath, device_fpath, read_only=False):
img_flag = os.O_RDONLY
dev_flag = os.O_RDONLY if read_only else os.O_WRONLY
if os.name == "posix":
return (os.open(image_fpath, img_flag), os.open(device_fpath, dev_flag))
elif os.name == "nt":
dev_flag = dev_flag | os.O_BINARY
return (os.open(image_fpath, img_flag), os.open(device_fpath, dev_flag))
else:
raise NotImplementedError("Platform not supported")
def close_handles(image_fd, device_fd):
try:
os.close(image_fd)
os.close(device_fd)
except Exception:
pass
def ensure_card_written(image_fpath, device_fpath, logger):
""" asserts image and device content is same (reads rand 4MiB from both """
logger.step("Verify data on SD card")
image_fd, device_fd = open_handles(image_fpath, device_fpath, read_only=True)
# read a 4MiB random part from the image
buffer_size = 4 * ONE_MiB
total_size = os.lseek(image_fd, 0, os.SEEK_END)
offset = random.randint(0, int((total_size - buffer_size) * .8))
offset -= offset % 512
logger.std(
"reading {n}b from offset {s} out of {t}b.".format(
n=buffer_size, s=offset, t=total_size
)
)
try:
# read same part from the SD card and compare
os.lseek(image_fd, offset, os.SEEK_SET)
os.lseek(device_fd, offset, os.SEEK_SET)
if not os.read(image_fd, buffer_size) == os.read(device_fd, buffer_size):
raise ValueError("Image and SD-card challenge do not match.")
except Exception:
raise
finally:
close_handles(image_fd, device_fd)
class ImageWriterThread(threading.Thread):
class EtcherWriterThread(threading.Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._should_stop = False # stop flag
......@@ -208,63 +165,82 @@ class ImageWriterThread(threading.Thread):
def stop(self):
self._should_stop = True
def run(self):
def run(self,):
image_fpath, device_fpath, logger = self._args
logger.step("Copy image to sd card")
image_fd, device_fd = open_handles(image_fpath, device_fpath)
total_size = os.lseek(image_fd, 0, os.SEEK_END)
os.lseek(image_fd, 0, os.SEEK_SET)
logger.step("Copy image to sd card using etcher-cli")
if os.name == "nt":
buffer_size = 512 # safer on windows
logger_break = 1000
else:
buffer_size = 25 * ONE_MiB
logger_break = 4
steps = total_size // buffer_size
for step in range(0, steps):
from_cli = logger is None or type(logger) == CLILogger
# on macOS, GUI sudo captures stdout so we use a log file
log_to_file = not from_cli and sys.platform == "darwin"
if log_to_file:
log_file = tempfile.NamedTemporaryFile(suffix=".log")
cmd = get_admin_command(
[
os.path.join(data.data_dir, "etcher-cli", "etcher"),
"-c",
"-y",
"-u",
"-d",
'"{}"'.format(device_fpath)
if sys.platform == "win32" # \\.\PHYSICALDRIVE1 string needs quotes
else device_fpath,
image_fpath,
],
from_gui=not from_cli,
log_to=log_file.name if log_to_file else None,
)
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **startup_info_args()
)
logger.std("Starting Etcher: " + str(process.args))
if self._should_stop:
while process.poll() is None:
if self._should_stop: # on cancel
logger.std(". cancelling...")
break
# only update logger every 4 steps (100MiB)
if step % logger_break == 0:
logger.progress(step, steps)
logger.std(
"Copied {copied} of {total} ({pc:.2f}%)".format(
copied=human_readable_size(step * buffer_size),
total=human_readable_size(total_size),
pc=step / steps * 100,
)
)
if log_to_file:
with open(log_file.name, "r") as f:
content = f.read()
if content:
logger.raw_std(f.read())
else:
for line in process.stdout:
logger.raw_std(line.decode("utf-8", "ignore"))
time.sleep(2)
try:
os.write(device_fd, os.read(image_fd, buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
self.exp = exp
raise
if not self._should_stop and total_size % buffer_size:
logger.std("Writing last chunk...")
try:
os.write(device_fd, os.read(image_fd, total_size % buffer_size))
except Exception as exp:
logger.std("Exception during write: {}".format(exp))
close_handles(image_fd, device_fd)
self.exp = exp
raise
if log_to_file:
log_file.close()
try:
logger.std(". has process exited?")
process.wait(timeout=2)
except subprocess.TimeoutExpired:
logger.std(". process exited")
# send ctrl^c
if sys.platform == "win32":
logger.std(". sending ctrl^C")
process.send_signal(signal.CTRL_C_EVENT)
process.send_signal(signal.CTRL_BREAK_EVENT)
time.sleep(2)
if process.poll() is None:
logger.std(". sending SIGTERM")
process.terminate() # send SIGTERM
time.sleep(2)
if process.poll() is None:
logger.std(". sending SIGKILL")
process.kill() # send SIGKILL (SIGTERM again on windows)
time.sleep(2)
else:
logger.std(". process exited")
if not process.returncode == 0:
self.exp = CheckCallException(
"Process returned {}".format(process.returncode)
)
logger.std(". process done")
logger.progress(1)
logger.step("sync")
if not self._should_stop:
os.fsync(device_fd)
close_handles(image_fd, device_fd)
def prevent_sleep(logger):
......
......@@ -35,7 +35,7 @@ from backend.mount import (
format_data_partition,
guess_next_loop_device,
)
from backend.util import ensure_card_written, ImageWriterThread
from backend.util import EtcherWriterThread
from backend.mount import can_write_on, allow_write_on, restore_mode
from backend.util import (
subprocess_pretty_check_call,
......@@ -121,14 +121,11 @@ def run_installation(
# Prepare SD Card
if sd_card:
logger.step("Change SD-card device mode ({})".format(sd_card))
if sys.platform == "linux":
allow_write_on(sd_card, logger)
elif sys.platform == "darwin":
logger.step("Prepare SD-card ({})".format(sd_card))
if sys.platform == "darwin":
subprocess_pretty_check_call(
["diskutil", "unmountDisk", sd_card], logger
)
allow_write_on(sd_card, logger)
elif sys.platform == "win32":
logger.step("Format SD card {}".format(sd_card))
matches = re.findall(r"\\\\.\\PHYSICALDRIVE(\d*)", sd_card)
......@@ -397,39 +394,29 @@ def run_installation(
if sd_card:
logger.stage("write")
logger.step("Writting image to SD-card ({})".format(sd_card))
try:
imwriter = ImageWriterThread(
etcher_writer = EtcherWriterThread(
args=(image_final_path, sd_card, logger)
)
cancel_event.register_thread(thread=imwriter)
imwriter.start()
imwriter.join(timeout=2) # make sure it started
while imwriter.is_alive():
cancel_event.register_thread(thread=etcher_writer)
etcher_writer.start()
etcher_writer.join(timeout=2) # make sure it started
while etcher_writer.is_alive():
pass
imwriter.join(timeout=2)
logger.std("not alive")
etcher_writer.join(timeout=2)
cancel_event.unregister_thread()
if imwriter.exp is not None:
raise imwriter.exp
if etcher_writer.exp is not None:
raise etcher_writer.exp
logger.std("Done writing ; preparing for verification.")
logger.std("Done writing and verifying.")
time.sleep(5)
ensure_card_written(image_final_path, sd_card, logger)
except ValueError:
logger.succ("Image created successfuly.")
logger.err(
"SD-card content is different than that of image.\n"
"Please check the content of your card and "
"verify that the card is not damaged ("
"often turns read-only silently).\n"
"Alternatively, use Etcher (see File > Flash) to "
"flash image onto SD-card and validate transfer."
)
raise Exception("SD-card content verification failed")
except Exception:
logger.succ("Image created successfuly.")
logger.err(
"Writing your Image to your SD-card failed.\n"
"Writing or verification of Image to your SD-card failed.\n"
"Please use a third party tool to flash your image "
"onto your SD-card. See File menu for links to Etcher."
)
......@@ -456,13 +443,6 @@ def run_installation(
logger.step("Restoring loop device ({}) mode".format(loop_dev))
restore_mode(loop_dev, previous_loop_mode, logger)
if sd_card:
logger.step("Restoring SD-card device ({}) mode".format(sd_card))
if sys.platform == "linux":
restore_mode(sd_card, "0660", logger)
elif sys.platform == "darwin":
restore_mode(sd_card, "0660", logger)
# display durations summary
logger.summary()
......
......@@ -107,7 +107,7 @@ elif sys.platform == "darwin":
{
"bus_protocol": plist["BusProtocol"],
"device_identifier": plist["DeviceIdentifier"],
"device": plist["DeviceNode"].replace("/dev/disk", "/dev/rdisk"),
"device": plist["DeviceNode"],
"io_registry_entry_name": plist["IORegistryEntryName"],
"media_name": plist["MediaName"],
"media_type": plist["MediaType"],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment