Commit 3798d765 authored by rgaudin's avatar rgaudin
Browse files

Format data partition on host

	To prevent inconsistency in data partition mounting on host:
		1. backup /data on /home/data_part.tar after reconfigure (very small)
		2. mkfs data partition on host
		3. copy large content onto data partition
		4. restore data_part.tar in qemu
parent 8ddd7373
......@@ -14,6 +14,9 @@
roles:
- role: variables
- role: restore
tags: move-content
# master, setup (timezone, apt-update)
- role: system
......@@ -65,6 +68,9 @@
- role: wikifundi_content
when: ansible_local.config.wikifundi_languages | length
- role: backup
tags: reconfigure
- role: services
tags: seal
......
---
- name: create tar of {{ data_path }}
become: yes
become_user: "{{ username }}"
args:
chdir: "{{ data_path }}"
shell: tar --dereference --hard-dereference -cf /home/{{ username }}/data_part.tar *
tags: reconfigure
......@@ -160,5 +160,5 @@
state: directory
with_items:
- "{{ data_path }}/test"
tags: ['setup', 'reconfigure']
tags: always
---
- name: restore {{ data_path }} archive
become: yes
become_user: "{{ username }}"
shell: tar -m --no-same-permissions --no-same-owner --no-overwrite-dir -C {{ data_path }} -xf /home/{{ username }}/data_part.tar
ignore_errors: yes
tags: move-content
- name: ensure kiwix data folder is present
file:
path: "{{ data_path }}/ideascube/kiwix/data"
state: directory
tags: move-content
- name: remove backup archive
file:
path: /home/{{ username }}/data_part.tar
state: absent
tags: move-content
......@@ -134,7 +134,7 @@
- name: install newer exfat-utils from bower
apt:
deb: "{{ mirror }}/dev/exfat-utils_1.2.8-1_armhf.deb"
tags: ['master', 'resize']
tags: master
- name: Install avahi packages
apt:
......
......@@ -2,6 +2,10 @@
- debug: msg="Start playbook {{ ansible_date_time["date"] }} - {{ ansible_date_time["time"] }}"
tags: always
- name: DEBUG list {{ data_path }} content
command: ls -lh {{ data_path }}
tags: always
- name: ensure custom facts directory exists
file:
path: /etc/ansible/facts.d
......
......@@ -37,8 +37,10 @@ if sys.platform == "win32":
elif sys.platform == "linux":
udisksctl_exe = '/usr/bin/udisksctl'
udisks_nou = '--no-user-interaction'
mkfs_exe = '/sbin/mkfs.exfat'
elif sys.platform == "darwin":
hdiutil_exe = '/usr/bin/hdiutil'
diskutil_exe = '/usr/sbin/diskutil'
mount_exe = '/sbin/mount'
umount_exe = '/sbin/umount'
......@@ -213,8 +215,8 @@ def test_mount_procedure(image_fpath, logger=None, thorough=False):
pass # failed to unmount (outch)
def mount_data_partition(image_fpath, logger=None):
''' mount the QEMU image's 3rd part and return its mount point/drive '''
def get_virtual_device(image_fpath, logger=None):
''' create and return a loop device or drive letter we can format/mount '''
if sys.platform == "linux":
# find out offset for third partition from the root part size
......@@ -222,7 +224,6 @@ def mount_data_partition(image_fpath, logger=None):
disk_size = get_qemu_image_size(image_fpath, logger)
offset, size = get_start_offset(
base_image.get('root_partition_size'), disk_size)
# size = get_partition_size(image_fpath, offset, logger)
# prepare loop device
udisks_loop = subprocess_pretty_call(
......@@ -234,6 +235,82 @@ def mount_data_partition(image_fpath, logger=None):
target_dev = re.search(r"(\/dev\/loop[0-9]+)\.$",
udisks_loop).groups()[0]
return target_dev
elif sys.platform == "darwin":
# attach image to create loop devices
hdiutil_out = subprocess_pretty_call(
[hdiutil_exe, 'attach', '-nomount', image_fpath],
logger, check=True, decode=True)[0].strip()
target_dev = str(hdiutil_out.splitlines()[0].split()[0])
return target_dev
elif sys.platform == "win32":
# make sure we have imdisk installed
install_imdisk(logger)
# get an available letter
target_dev = get_avail_drive_letter(logger)
return target_dev
def format_data_partition(image_fpath, logger=None):
''' format the QEMU image's 3rd part in exfat on host '''
target_dev = get_virtual_device(image_fpath, logger)
if sys.platform == "linux":
# make sure it's not mounted (gnoe automounts)
subprocess_pretty_call(
[udisksctl_exe, 'unmount',
'--block-device', target_dev, udisks_nou], logger)
# format the data partition
try:
subprocess_pretty_check_call(
[mkfs_exe, '-n', 'data', target_dev], logger)
except Exception:
raise
finally:
# ensure we release the loop device on mount failure
unmount_data_partition(None, target_dev)
elif sys.platform == "darwin":
target_part = "{dev}s3".format(dev=target_dev)
try:
subprocess_pretty_check_call(
[diskutil_exe, 'eraseVolume', 'exfat', 'data', target_part], logger)
except Exception:
raise
finally:
# ensure we release the loop device on mount failure
unmount_data_partition(None, target_dev)
elif sys.platform == "win32":
# mount into specified path AND format
try:
subprocess_pretty_check_call(
[imdisk_exe, '-a', '-f', image_fpath,
'-o', 'rw', '-t', 'file',
'-v', '3',
'-p', '/fs:exfat /V:data /q /y',
'-m', target_dev], logger)
except Exception:
raise
finally:
# ensure we release the loop device on mount failure
unmount_data_partition(None, target_dev)
def mount_data_partition(image_fpath, logger=None):
''' mount the QEMU image's 3rd part and return its mount point/drive '''
target_dev = get_virtual_device(image_fpath, logger)
if sys.platform == "linux":
# mount the loop-device (udisksctl sets the mount point)
udisks_mount_ret, udisks_mount = subprocess_pretty_call(
[udisksctl_exe, 'mount',
......@@ -255,11 +332,6 @@ def mount_data_partition(image_fpath, logger=None):
return mount_point, target_dev
elif sys.platform == "darwin":
# attach image to create loop devices
hdiutil_out = subprocess_pretty_call(
[hdiutil_exe, 'attach', '-nomount', image_fpath],
logger, check=True, decode=True)[0].strip()
target_dev = str(hdiutil_out.splitlines()[0].split()[0])
target_part = "{dev}s3".format(dev=target_dev)
# create a mount point in /tmp
......@@ -274,11 +346,6 @@ def mount_data_partition(image_fpath, logger=None):
return mount_point, target_dev
elif sys.platform == "win32":
# make sure we have imdisk installed
install_imdisk(logger)
# get an available letter
target_dev = get_avail_drive_letter(logger)
mount_point = "{}\\".format(target_dev)
# mount into the specified drive
......@@ -292,30 +359,36 @@ def unmount_data_partition(mount_point, device, logger=None):
''' unmount data partition and free virtual resources '''
if sys.platform == "linux":
# sleep to prevent unmount failures
time.sleep(5)
# unmount using device path
subprocess_pretty_call(
[udisksctl_exe, 'unmount',
'--block-device', device, udisks_nou], logger)
try:
os.rmdir(mount_point)
except FileNotFoundError:
pass
if mount_point:
# unmount using device path
subprocess_pretty_call(
[udisksctl_exe, 'unmount',
'--block-device', device, udisks_nou], logger)
try:
os.rmdir(mount_point)
except FileNotFoundError:
pass
# delete the loop device (might have already been deletec)
subprocess_pretty_call(
[udisksctl_exe, 'loop-delete',
'--block-device', device, udisks_nou], logger)
elif sys.platform == "darwin":
# unmount
subprocess_pretty_call([umount_exe, mount_point], logger)
try:
os.rmdir(mount_point)
except FileNotFoundError:
pass
if mount_point:
# unmount
subprocess_pretty_call([umount_exe, mount_point], logger)
try:
os.rmdir(mount_point)
except FileNotFoundError:
pass
# detach image file (also unmounts if not already done)
subprocess_pretty_call([hdiutil_exe, 'detach', device], logger)
elif sys.platform == "win32":
# unmount using force (-D) as -d is not reliable
subprocess_pretty_call([imdisk_exe, '-D', '-m', device], logger)
......@@ -3,7 +3,7 @@ from backend import qemu
from backend.content import (get_collection, get_content,
get_all_contents_for, isremote)
from backend.download import download_content, unzip_file
from backend.mount import mount_data_partition, unmount_data_partition, test_mount_procedure
from backend.mount import mount_data_partition, unmount_data_partition, test_mount_procedure, format_data_partition
from backend.util import subprocess_pretty_check_call, subprocess_pretty_call
from backend.sysreq import host_matches_requirements, requirements_url
import data
......@@ -91,7 +91,7 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
.format(image_building_path))
logger.step("Testing mount procedure")
test_mount_procedure(image_building_path, logger)
test_mount_procedure(image_building_path, logger, thorough=True)
# harmonize options
packages = [] if zim_install is None else zim_install
......@@ -197,8 +197,11 @@ def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite,
# mount image's 3rd partition on host
logger.stage('copy')
logger.step("Mounting data partition on host")
logger.step("Formating data partition on host")
format_data_partition(image_building_path, logger)
logger.step("Mounting data partition on host")
# copy contents from cache to mount point
try:
mount_point, device = mount_data_partition(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment