run_installation.py 15.1 KB
Newer Older
thiolliere's avatar
thiolliere committed
1
2
from backend import ansiblecube
from backend import qemu
rgaudin's avatar
rgaudin committed
3
from backend.content import (get_collection, get_content,
4
5
                             get_all_contents_for, isremote, get_content_cache,
                             get_alien_content)
6
from backend.download import download_content, unzip_file
7
8
from backend.mount import mount_data_partition, unmount_data_partition, test_mount_procedure, format_data_partition, guess_next_loop_device
from backend.mount import can_write_on, allow_write_on, restore_mode
9
from backend.util import subprocess_pretty_check_call, subprocess_pretty_call
rgaudin's avatar
rgaudin committed
10
from backend.util import run_sdcard_thread
11
from backend.sysreq import host_matches_requirements, requirements_url
thiolliere's avatar
thiolliere committed
12
import data
13
from util import human_readable_size, get_cache, ensure_zip_exfat_compatible, EXFAT_FORBIDDEN_CHARS
thiolliere's avatar
thiolliere committed
14
from datetime import datetime
thiolliere's avatar
thiolliere committed
15
import os
thiolliere's avatar
thiolliere committed
16
import sys
17
import re
rgaudin's avatar
rgaudin committed
18
import time
rgaudin's avatar
rgaudin committed
19
import shutil
20
import traceback
thiolliere's avatar
thiolliere committed
21

22

rgaudin's avatar
rgaudin committed
23
def run_installation(name, timezone, language, wifi_pwd, admin_account, kalite, aflatoun, wikifundi, edupi, edupi_resources, zim_install, size, logger, cancel_event, sd_card, favicon, logo, css, done_callback=None, build_dir=".", qemu_ram="2G"):
24
25
26
27

    logger.start(bool(sd_card))

    logger.stage('init')
rgaudin's avatar
rgaudin committed
28
    cache_folder = get_cache(build_dir)
29

thiolliere's avatar
thiolliere committed
30
    try:
31
32
33
34
35
36
37
38
        logger.step("Check System Requirements")
        logger.std("Please read {} for details".format(requirements_url))
        sysreq_ok, missing_deps = host_matches_requirements(build_dir)
        if not sysreq_ok:
            raise SystemError(
                "Your system does not matches system requirements:\n{}".format(
                    "\n".join([" - {}".format(dep) for dep in missing_deps])))

39
40
        logger.step("Ensure user files are present")
        for user_fpath in (edupi_resources, favicon, logo, css):
41
42
            if user_fpath is not None and not isremote(user_fpath) \
                    and not os.path.exists(user_fpath):
43
44
45
                raise ValueError("Specified file is not available ({})"
                                 .format(user_fpath))

46
47
48
49
50
        logger.step("Prepare Image file")

        # set image names
        today = datetime.today().strftime('%Y_%m_%d-%H_%M_%S')

51
52
53
        image_final_path = os.path.join(build_dir, "plug-{}.img".format(today))
        image_building_path = os.path.join(build_dir, "plug-{}.BUILDING.img".format(today))
        image_error_path = os.path.join(build_dir, "plug-{}.ERROR.img".format(today))
54

55
56
57
58
59
60
61
62
63
        # loop device mode on linux (for mkfs in userspace)
        if sys.platform == "linux":
            loop_dev = guess_next_loop_device(logger)
            if loop_dev and not can_write_on(loop_dev):
                logger.step("Change loop device mode ({})".format(sd_card))
                previous_loop_mode = allow_write_on(loop_dev, logger)
            else:
                previous_loop_mode = None

thiolliere's avatar
thiolliere committed
64
        # Prepare SD Card
65
        if sd_card:
66
            logger.step("Change SD-card device mode ({})".format(sd_card))
67
            if sys.platform == "linux":
68
                allow_write_on(sd_card, logger)
69
            elif sys.platform == "darwin":
70
                subprocess_pretty_check_call(["diskutil", "unmountDisk", sd_card], logger)
71
                allow_write_on(sd_card, logger)
72
            elif sys.platform == "win32":
73
                logger.step("Format SD card {}".format(sd_card))
74
                matches = re.findall(r"\\\\.\\PHYSICALDRIVE(\d*)", sd_card)
75
76
77
78
79
80
81
82
                if len(matches) != 1:
                    raise ValueError("Error while getting physical drive number")
                device_number = matches[0]

                r,w = os.pipe()
                os.write(w, str.encode("select disk {}\n".format(device_number)))
                os.write(w, b"clean\n")
                os.close(w)
thiolliere's avatar
fix log    
thiolliere committed
83
                logger.std("diskpart select disk {} and clean".format(device_number))
84
                subprocess_pretty_check_call(["diskpart"], logger, stdin=r)
rgaudin's avatar
rgaudin committed
85
86
                logger.std("sleeping for 15s to acknowledge diskpart changes")
                time.sleep(15)
87

88
89
        # Download Base image
        logger.stage('master')
90
        logger.step("Retrieving base image file")
91
92
93
        base_image = get_content('pibox_base_image')
        rf = download_content(base_image, logger, build_dir)
        if not rf.successful:
94
            logger.err("Failed to download base image.\n{e}"
95
96
97
98
99
                       .format(e=rf.exception))
            sys.exit(1)
        elif rf.found:
            logger.std("Reusing already downloaded base image ZIP file")
        logger.progress(.5)
thiolliere's avatar
thiolliere committed
100

101
102
103
104
105
106
107
108
109
110
111
112
113
114
        # extract base image and rename
        logger.step("Extracting base image from ZIP file")
        unzip_file(archive_fpath=rf.fpath,
                   src_fname=base_image['name'].replace('.zip', ''),
                   build_folder=build_dir,
                   dest_fpath=image_building_path)
        logger.std("Extraction complete: {p}".format(p=image_building_path))
        logger.progress(.9)

        if not os.path.exists(image_building_path):
            raise IOError("image path does not exists: {}"
                          .format(image_building_path))

        logger.step("Testing mount procedure")
115
116
        if not test_mount_procedure(image_building_path, logger, True):
            raise ValueError("thorough mount procedure failed")
117
118
119
120
121
122

        # harmonize options
        packages = [] if zim_install is None else zim_install
        kalite_languages = [] if kalite is None else kalite
        wikifundi_languages = [] if wikifundi is None else wikifundi
        aflatoun_languages = ['fr', 'en'] if aflatoun else []
thiolliere's avatar
thiolliere committed
123

rgaudin's avatar
rgaudin committed
124
125
126
127
        if edupi_resources and not isremote(edupi_resources):
            logger.step("Copying EduPi resources into cache")
            shutil.copy(edupi_resources, cache_folder)

128
129
130
131
        # collection contains both downloads and processing callbacks
        # for all requested contents
        collection = get_collection(
            edupi=edupi,
rgaudin's avatar
rgaudin committed
132
            edupi_resources=edupi_resources,
133
134
135
136
            packages=packages,
            kalite_languages=kalite_languages,
            wikifundi_languages=wikifundi_languages,
            aflatoun_languages=aflatoun_languages)
thiolliere's avatar
thiolliere committed
137

138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
        # download contents into cache
        logger.stage('download')
        logger.step("Starting all content downloads")
        downloads = list(get_all_contents_for(collection))
        archives_total_size = sum([c['archive_size'] for c in downloads])
        retrieved = 0

        for dl_content in downloads:
            logger.step("Retrieving {name} ({size})".format(
                name=dl_content['name'],
                size=human_readable_size(dl_content['archive_size'])))

            rf = download_content(dl_content, logger, build_dir)
            if not rf.successful:
                logger.err("Error downloading {u} to {p}\n{e}"
                           .format(u=dl_content['url'],
                                   p=rf.fpath, e=rf.exception))
                raise rf.exception if rf.exception else IOError
            elif rf.found:
                logger.std("Reusing already downloaded {p}".format(p=rf.fpath))
            else:
                logger.std("Saved `{p}` successfuly: {s}"
                           .format(p=dl_content['name'],
                                   s=human_readable_size(rf.downloaded_size)))
            retrieved += dl_content['archive_size']
163
            logger.progress(retrieved, archives_total_size)
164

165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
        # check edupi resources compliance
        if edupi_resources:
            logger.step("Verifying EduPi resources file names")
            exfat_compat, exfat_errors = ensure_zip_exfat_compatible(get_content_cache(get_alien_content(edupi_resources),
                                                    cache_folder, True))
            if not exfat_compat:
                raise ValueError("Your EduPi resources archive is incorrect.\n"
                                 "It should be a ZIP file of a root folder "
                                 "in which all files have exfat-compatible "
                                 "names (no {chars})\n... {fnames}"
                                 .format(chars=" ".join(EXFAT_FORBIDDEN_CHARS),
                                         fnames="\n... ".join(exfat_errors)))
            else:
                logger.std("EduPi resources archive OK")

180
181
182
183
184
185
186
        # instanciate emulator
        logger.stage('setup')
        logger.step("Preparing qemu VM")
        emulator = qemu.Emulator(data.vexpress_boot_kernel,
                                 data.vexpress_boot_dtb,
                                 image_building_path, logger,
                                 ram=qemu_ram)
thiolliere's avatar
thiolliere committed
187
188

        # Resize image
189
190
        logger.step("Resizing image file to {s}"
                    .format(s=human_readable_size(emulator.get_image_size())))
thiolliere's avatar
thiolliere committed
191
192
193
        if size < emulator.get_image_size():
            logger.err("cannot decrease image size")
            raise ValueError("cannot decrease image size")
thiolliere's avatar
thiolliere committed
194

thiolliere's avatar
thiolliere committed
195
        emulator.resize_image(size)
thiolliere's avatar
thiolliere committed
196

197
198
199
200
201
202
203
204
        # prepare ansible options
        ansible_options = {
            'name': name,
            'timezone': timezone,
            'language': language,
            'language_name': dict(data.ideascube_languages)[language],

            'edupi': edupi,
rgaudin's avatar
rgaudin committed
205
            'edupi_resources': edupi_resources,
206
207
208
209
210
211
212
213
214
215
216
217
218
219
            'wikifundi_languages': wikifundi_languages,
            'aflatoun_languages': aflatoun_languages,
            'kalite_languages': kalite_languages,
            'packages': packages,

            'wifi_pwd': wifi_pwd,
            'admin_account': admin_account,

            'disk_size': emulator.get_image_size(),
            'root_partition_size': base_image.get('root_partition_size'),
        }
        extra_vars, secret_keys = ansiblecube.build_extra_vars(
            **ansible_options)

thiolliere's avatar
thiolliere committed
220
        # Run emulation
221
        logger.step("Starting-up VM")
thiolliere's avatar
thiolliere committed
222
        with emulator.run(cancel_event) as emulation:
223
224
225
226
227
228
229
            # copying ansiblecube again into the VM
            # should the master-version been updated
            logger.step("Copy ansiblecube")
            emulation.exec_cmd("sudo /bin/rm -rf {}".format(
                ansiblecube.ansiblecube_path))
            emulation.put_dir(data.ansiblecube_path,
                              ansiblecube.ansiblecube_path)
thiolliere's avatar
thiolliere committed
230

231
232
233
            logger.step("Run ansiblecube")
            ansiblecube.run_phase_one(emulation, extra_vars, secret_keys,
                                      logo=logo, favicon=favicon, css=css)
234

235
236
        # mount image's 3rd partition on host
        logger.stage('copy')
thiolliere's avatar
thiolliere committed
237

rgaudin's avatar
rgaudin committed
238
239
240
241
        logger.step("Formating data partition on host")
        format_data_partition(image_building_path, logger)

        logger.step("Mounting data partition on host")
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
        # copy contents from cache to mount point
        try:
            mount_point, device = mount_data_partition(
                image_building_path, logger)
            logger.step("Processing downloaded content onto data partition")
            expanded_total_size = sum([c['expanded_size'] for c in downloads])
            processed = 0

            for category, content_dl_cb, \
                    content_run_cb, cb_kwargs in collection:

                logger.step("Processing {cat}".format(cat=category))
                content_run_cb(cache_folder=cache_folder,
                               mount_point=mount_point,
                               logger=logger, **cb_kwargs)
                # size of expanded files for this category (for progress)
                processed += sum([c['expanded_size']
                                  for c in content_dl_cb(**cb_kwargs)])
260
                logger.progress(processed, expanded_total_size)
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
        except Exception as exp:
            try:
                unmount_data_partition(mount_point, device, logger)
            except NameError:
                pass  # if mount_point or device are not defined
            raise exp

        # unmount partition
        logger.step("Unmounting data partition")
        unmount_data_partition(mount_point, device, logger)

        # rerun emulation for discovery
        logger.stage('move')
        logger.step("Starting-up VM again for content-discovery")
        with emulator.run(cancel_event) as emulation:
            logger.step("Re-run ansiblecube for move-content")
            ansiblecube.run_phase_two(emulation, extra_vars, secret_keys)
thiolliere's avatar
rebrand    
thiolliere committed
278

thiolliere's avatar
thiolliere committed
279
    except Exception as e:
280
281
        logger.failed(str(e))

282
283
284
285
        # display traceback on logger
        logger.std("\n--- Exception Trace ---\n{exp}\n---"
                   .format(exp=traceback.format_exc()))

thiolliere's avatar
thiolliere committed
286
287
288
289
        # Set final image filename
        if os.path.isfile(image_building_path):
            os.rename(image_building_path, image_error_path)

thiolliere's avatar
thiolliere committed
290
291
        error = e
    else:
thiolliere's avatar
thiolliere committed
292
293
294
        # Set final image filename
        os.rename(image_building_path, image_final_path)

rgaudin's avatar
rgaudin committed
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
        try:
            # Write image to SD Card
            if sd_card:
                logger.stage('write')
                logger.step("Writting image to SD-card ({})".format(sd_card))
                try:
                    run_sdcard_thread(image_final_path, sd_card,
                                      False, cancel_event, logger)
                except Exception:
                    logger.succ("Image created successfuly.")
                    logger.err(
                        "Writing your Image to your SD-card failed.\n"
                        "Please use a third party tool to flash your image "
                        "onto your SD-card. See File menu for links to Etcher."
                    )
                    raise Exception("Failed to write Image to SD-card")

                # check that SD card was properly written
                try:
                    run_sdcard_thread(image_final_path, sd_card,
                                      True, cancel_event, logger)
                except AssertionError:
                    logger.err(
                        "SD-card content is different than that of image.\n"
                        "Please check the content of your card and "
                        "verify that the card is not damaged ("
                        "often turns read-only silently).\n"
                        "Alternatively, use Etcher (see File > Flash) to "
                        "flash image onto SD-card and validate transfer.")
                    raise Exception("SD-card content differs from image.")

        except Exception as e:
            logger.failed(str(e))

            # display traceback on logger
            logger.std("\n--- Exception Trace ---\n{exp}\n---"
                       .format(exp=traceback.format_exc()))
            error = e
        else:
            logger.complete()
            error = None
336
    finally:
337
338
339
340
        if sys.platform == "linux" and loop_dev and previous_loop_mode:
            logger.step("Restoring loop device ({}) mode".format(loop_dev))
            restore_mode(loop_dev, previous_loop_mode, logger)

341
        if sd_card:
342
            logger.step("Restoring SD-card device ({}) mode".format(sd_card))
343
            if sys.platform == "linux":
344
                restore_mode(sd_card, '0660', logger)
345
            elif sys.platform == "darwin":
346
                restore_mode(sd_card, '0660', logger)
347
348
349

        # display durations summary
        logger.summary()
350

thiolliere's avatar
thiolliere committed
351
    if done_callback:
thiolliere's avatar
thiolliere committed
352
353
354
        done_callback(error)

    return error