tasks.py 7.66 KB
Newer Older
1
2
3
4
"""Run using:
celery -A hkis worker
"""

5
6
import asyncio
from functools import partial
7
import json
8
from random import choice
9
from typing import Optional
10
11
import os
import tempfile
12
13
14
from subprocess import Popen, PIPE, run, STDOUT, TimeoutExpired, DEVNULL

from logging import getLogger
15
16
17

from celery import shared_task

18
19
20

logger = getLogger(__name__)

21
22
23
24
25
26
27
28
FIREJAIL_OPTIONS = [
    "--quiet",
    "--net=none",
    "--shell=none",
    "--x11=none",
    "--protocol=inet",
    "--private-dev",
    "--private-bin=python3",
29
    "--private-etc=group,hostname,localtime,nsswitch.conf,passwd,resolv.conf,alternatives,ssl",
30
31
32
33
34
35
36
37
38
39
40
41
42
    "--private-tmp",
    "--caps.drop=all",
    "--noprofile",
    "--nonewprivs",
    "--nosound",
    "--no3d",
    "--nogroups",
    "--noroot",
    "--seccomp",
    "--rlimit-fsize=8192",
    "--rlimit-nofile=100",
    "--rlimit-nproc=2000",
    "--rlimit-cpu=20",
43
44
45
    "--rlimit-as=1610612736",  # 1.5GB. correction_helper will cap
    # student code at 1GB, leaving some space for check.py to report
    # errors.
46
47
48
49
50
51
52
    "--blacklist=/var",
    "--blacklist=/sys",
    "--blacklist=/boot",
]


@shared_task
53
54
55
56
57
58
59
60
61
62
def run_snippet_task(source_code: str, pre: Optional[str] = None) -> str:
    """Just run the *source_code* by putting it in a `snippet.py` file and
    running it in a sandbox.

    `pre` can be used to run some preparation code
    **unsanboxed**. It's usefull to setup the test environment by
    pre-creating files, installing dependencies, placing i18n files,
    ...

    """
63
    with tempfile.TemporaryDirectory(prefix="hkis_snippets") as tmpdir:
64
65
        if pre:
            run_pre_check(tmpdir, pre)
66
67
68
69
70
71
        with open(os.path.join(tmpdir, "snippet.py"), "w") as snippet_file:
            snippet_file.write(source_code)
        firejail_env = os.environ.copy()
        prof_proc = Popen(
            ["firejail"]
            + FIREJAIL_OPTIONS
72
            + ["--private=" + tmpdir, "python3", "./snippet.py"],
73
74
75
            stdin=DEVNULL,
            stdout=PIPE,
            stderr=STDOUT,
76
            cwd=tmpdir,
77
78
79
80
            env=firejail_env,
        )
        try:
            stdout = prof_proc.communicate(timeout=40)[0]
81
82
            if prof_proc.returncode == 255:
                return "Checker timed out, look for infinite loops maybe?"
83
84
85
            return stdout.decode("UTF-8", "backslashreplace").replace(
                "\u0000", r"\x00"
            )[:65_536]
86
87
88
89
        except TimeoutExpired:
            prof_proc.kill()
            prof_proc.wait()
            return "Timed out after 20 seconds."
90
91
        except OSError as err:
            return str(err)
Julien Palard's avatar
Julien Palard committed
92
93
        except MemoryError:
            return "Not enough memory to run your code."
94
95


96
def congrats(language):
97
    """Generates a congratulation sentence."""
98
99
    return (
        choice(
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
            {
                "en": [
                    "Congrats",
                    "Nice job",
                    "Well done",
                    "Spot on",
                    "Bravo",
                    "Nice",
                    "Good",
                ],
                "fr": [
                    "Bravo",
                    "Bien joué",
                    "Super",
                    "Excellent",
                    "Joli",
                ],
            }[language]
118
119
        )
        + choice(
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
            {
                "en": ["! ", "!! ", "!!! ", "! ! "],
                "fr": [" ! ", " !! ", " !!! ", " ! ! "],
            }[language]
        )
        + choice(
            {
                "en": [
                    "Your exercise is OK.",
                    "Right answer.",
                    "Good answer.",
                    "Correct answer.",
                    "Looks good to me!",
                    "Your answer is right.",
                    "Your answer is correct.",
                ],
                "fr": [
                    "C'est juste.",
                    "Bonne réponse.",
                    "Correct.",
                    "Ça me semble bon.",
                    "C'est la bonne réponse.",
                    "Excellente réponse.",
                ],
            }[language]
145
146
147
148
        )
    )


149
def run_pre_check(tmpdir, pre_check: str, env=None) -> dict:
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
    """Run a pre-check script outside the sandbox before the actual check."""
    with open(os.path.join(tmpdir, "pre_check.py"), "w") as pre_check_file:
        pre_check_file.write(pre_check)
    logger.info("Running pre-check")
    pre_check_result = run(
        ["python3", os.path.join(tmpdir, "pre_check.py")],
        cwd=tmpdir,
        stdin=DEVNULL,
        stdout=PIPE,
        stderr=PIPE,
        env=env,
    )
    if pre_check_result.returncode != 0 or pre_check_result.stderr:
        logger.warning(
            "pre_check failed with code %d, stdout: %s, stderr: %s",
            pre_check_result.returncode,
            pre_check_result.stdout,
            pre_check_result.stderr,
        )
169
170
171
172
    try:
        return json.loads(pre_check_result.stdout)
    except json.JSONDecodeError:
        return {}
173
174


175
176
@shared_task
def check_answer_task(answer: dict):
177
178
179
    """Executed on Celery workers.
    answer should contain: check, source_code, and language.
    """
180
181
182
183
184
185
    with tempfile.TemporaryDirectory(prefix="hkis") as tmpdir:
        with open(os.path.join(tmpdir, "check.py"), "w") as check_file:
            check_file.write(answer["check"])
        with open(os.path.join(tmpdir, "solution.py"), "w") as answer_file:
            answer_file.write(answer["source_code"])
        firejail_env = os.environ.copy()
186
187
        if "language" in answer:
            firejail_env["LANGUAGE"] = answer["language"]
188
        if "pre_check" in answer and answer["pre_check"]:
189
190
191
            firejail_env.update(
                run_pre_check(tmpdir, answer["pre_check"], env=firejail_env)
            )
192
193
194
        prof_proc = Popen(
            ["firejail"]
            + FIREJAIL_OPTIONS
195
            + ["--private=" + tmpdir, "python3", "-u", "./check.py"],
196
197
198
            stdin=DEVNULL,
            stdout=PIPE,
            stderr=STDOUT,
199
            cwd=tmpdir,
200
201
202
            env=firejail_env,
        )
        try:
203
204
205
            stdout = (
                prof_proc.communicate(timeout=40)[0]
                .decode("UTF-8", "backslashreplace")
206
                .replace("\u0000", r"\x00")
207
208
209
                .replace(  # Simplify tracebacks by hiding the temporary directory
                    'File "' + os.path.expanduser("~/"), 'File "'
                )
210
            )[:65_536]
211
212
            if prof_proc.returncode == 0:
                return True, stdout or congrats(answer.get("language", "en"))
213
214
            if prof_proc.returncode == 255:
                return False, "Checker timed out, look for infinite loops maybe?"
215
            return False, stdout
216
217
218
219
        except TimeoutExpired:
            prof_proc.kill()
            prof_proc.wait()
            return False, "Checker timed out."
Julien Palard's avatar
Julien Palard committed
220
221
        except MemoryError:
            return False, "Not enough memory to run your code."
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237


async def check_answer(answer: dict):
    """Executed Django side.

    TODO with Celery 5: should no longer need run_in_executor.
    """

    def sync_celery_check_answer(answer: dict):
        return check_answer_task.apply_async((answer,), expires=60).get()

    return await asyncio.get_running_loop().run_in_executor(
        None, partial(sync_celery_check_answer, answer=answer)
    )


238
async def run_snippet(source_code: str, pre: Optional[str] = None) -> str:
239
240
241
242
243
244
    """Executed Django side.

    TODO with Celery 5: should no longer need run_in_executor.
    """

    def sync_celery_check_answer(source_code: str):
245
        return run_snippet_task.apply_async((source_code, pre), expires=60).get()
246
247
248
249

    return await asyncio.get_running_loop().run_in_executor(
        None, partial(sync_celery_check_answer, source_code=source_code)
    )