Skip to content

Commit

Permalink
#21 Grader now use most recent env
Browse files Browse the repository at this point in the history
  • Loading branch information
Coumes Quentin committed Feb 11, 2019
1 parent de32162 commit 6a97d29
Show file tree
Hide file tree
Showing 7 changed files with 169 additions and 196 deletions.
2 changes: 1 addition & 1 deletion default_file/builder.sh
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
#!/usr/bin/env bash
python3 builder.py pl.json built_pl.json 2> stderr.log
python3 builder.py pl.json processed.json 2> stderr.log
2 changes: 1 addition & 1 deletion default_file/grader.sh
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
#!/usr/bin/env bash
python3 grader.py built_pl.json answers.json evaluated_pl.json feedback.html 2> stderr.log
python3 grader.py pl.json answers.json processed.json feedback.html 2> stderr.log
23 changes: 23 additions & 0 deletions sandbox/container.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
import os
import shutil
import tarfile
import threading
import time

Expand Down Expand Up @@ -69,6 +70,28 @@ def _get_default_file(self):
shutil.copy2(s, d)


def extract_env(self, envid, suffix, prefix="", test=False):
"""Retrieve the environment from the docker and write it to:
[settings.MEDIA_ROOT]/[prefix][env_id][suffix][ext]
"test_" is added before [prefix] if test is True
An integer (up to 100) can be added before [ext] if the path already exists."""
base = os.path.join(settings.MEDIA_ROOT,
("test_" if test else "") + prefix + envid + suffix)
path = base + ".tgz"

print(test)
print(base)

for i in range(1, 100):
if os.path.exists(path):
path = base + str(i) + ".tgz"

with tarfile.open(path, "w|gz") as tar:
for name in os.listdir(self.envpath):
tar.add(os.path.join(self.envpath, name), arcname=name)


@staticmethod
def acquire():
"""Return the first available container, None if none were available."""
Expand Down
197 changes: 90 additions & 107 deletions sandbox/executor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,10 @@
EVAL_TIMEOUT = 8

CONTEXT_FILE = "pl.json"
BUILT_CONTEXT_FILE = "built_pl.json"
PROCESSED_CONTEXT_FILE = "processed.json"
STDOUT_FILE = "stdout.log"
STDERR_FILE = "stderr.log"
FEEDBACK_FILE = "feedback.html"
EVALUATED_CONTEXT_FILE = "evaluated_pl.json"
ANSWERS_FILE = "answers.json"

TIMEOUT_FEEDBACK = """
Expand All @@ -46,9 +45,8 @@ class Executor:
"""This class provide an interface to execute PL scripts."""


def __init__(self, cw, envpath, sandbox_url, timeout=0):
def __init__(self, cw, envpath, timeout=0):
self.envpath = envpath
self.sandbox_url = sandbox_url
self.envid = os.path.splitext(os.path.basename(envpath))[0]
self.cw = cw
self.docker = cw.container
Expand All @@ -61,6 +59,13 @@ def move_env_to_docker(self):
with tarfile.open(self.envpath, "r:gz") as tar:
tar.extractall(self.cw.envpath)
tar.close()

processed = os.path.join(self.cw.envpath, PROCESSED_CONTEXT_FILE)
old = os.path.join(self.cw.envpath, CONTEXT_FILE)
if os.path.isfile(processed):
os.remove(old)
os.rename(processed, old)

logger.debug("move_env_to_docker() took " + str(time.time() - start))


Expand Down Expand Up @@ -89,34 +94,28 @@ def get_feedback(self):


def get_context(self):
raise NotImplementedError


def execute(self):
raise NotImplementedError



class Builder(Executor):

def __init__(self, cw, envpath, sandbox_url, timeout=BUILD_TIMEOUT):
super().__init__(cw, envpath, sandbox_url, timeout)


def get_context(self):
"""Return content of BUILT_CONTEXT_FILE as a dictionnary (file must be a valid json).
"""Return content of PROCESSED_CONTEXT_FILE as a dictionnary (file must be a valid json).
Raises ContextNotFoundError if the file could not be found."""
start = time.time()

try:
with open(os.path.join(self.cw.envpath, BUILT_CONTEXT_FILE)) as f:
with open(os.path.join(self.cw.envpath, PROCESSED_CONTEXT_FILE)) as f:
j = json.load(f)
except FileNotFoundError:
logger.debug("get_context() took " + str(time.time() - start))
raise ContextNotFoundError

logger.debug("get_context() took " + str(time.time() - start))
return j



class Builder(Executor):
"""Used to build an exercise."""


def __init__(self, cw, envpath, timeout=BUILD_TIMEOUT):
super().__init__(cw, envpath, timeout)


@timeout_decorator.timeout(BUILD_TIMEOUT, use_signals=False)
Expand All @@ -139,42 +138,39 @@ def execute(self):
self.move_env_to_docker()
exit_code, _ = self.build()
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : exit_code,
"stderr" : self.get_stderr(),
"context" : self.get_context() if not exit_code else {},
"sandboxerr" : ""
"id": self.envid,
"status": exit_code,
"stderr": self.get_stderr(),
"context": self.get_context() if not exit_code else {},
"sandboxerr": ""
}
except timeout_decorator.TimeoutError:
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : SandboxErrCode.TIMEOUT,
"stderr" : self.get_stderr(),
"context" : {},
"sandboxerr" : ("Execution of the script build/before timed out after "
+ str(self.timeout) + " seconds.")
"id": self.envid,
"status": SandboxErrCode.TIMEOUT,
"stderr": self.get_stderr(),
"context": {},
"sandboxerr": ("Execution of the script build/before timed out after "
+ str(self.timeout) + " seconds.")
}
except ContextNotFoundError:
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : SandboxErrCode.CONTEXT_NOT_FOUND,
"stderr" : self.get_stderr(),
"context" : {},
"sandboxerr" : ("File '" + BUILT_CONTEXT_FILE + "' and '" + CONTEXT_FILE + "' were "
+ "not found in the environment after the execution of the "
+ "build/before script.")
"id": self.envid,
"status": SandboxErrCode.CONTEXT_NOT_FOUND,
"stderr": self.get_stderr(),
"context": {},
"sandboxerr": (
"File '" + PROCESSED_CONTEXT_FILE + "' and '" + CONTEXT_FILE + "' were "
+ "not found in the environment after the execution of the "
+ "build/before script.")
}
except Exception: # Unknown error
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : SandboxErrCode.UNKNOWN,
"stderr" : self.get_stderr(),
"context" : {},
"sandboxerr" : "An unknown error occured:\n" + traceback.format_exc()
"id": self.envid,
"status": SandboxErrCode.UNKNOWN,
"stderr": self.get_stderr(),
"context": {},
"sandboxerr": "An unknown error occured:\n" + traceback.format_exc()
}
logger.exception("An unknown exception occured during build of env %s:" % self.envid)

Expand All @@ -183,34 +179,22 @@ def execute(self):


class Evaluator(Executor):

def __init__(self, cw, envpath, sandbox_url, answers, timeout=EVAL_TIMEOUT):
super().__init__(cw, envpath, sandbox_url, timeout)
self.answers = answers
"""Use to grade an exercise."""


def get_context(self):
"""Return content of EVALUATED_CONTEXT_FILE as a dictionnary (file must be a valid json)."""
start = time.time()

try:
with open(os.path.join(self.cw.envpath, EVALUATED_CONTEXT_FILE)) as f:
j = json.load(f)
except FileNotFoundError:
logger.debug("get_context() took " + str(time.time() - start))
raise ContextNotFoundError

logger.debug("get_context() took " + str(time.time() - start))
return j
def __init__(self, cw, envpath, answers, timeout=EVAL_TIMEOUT):
super().__init__(cw, envpath, timeout)
self.answers = answers


def add_answer_to_env(self):
"""Add the answers in self.answers tp the environment."""
start = time.time()
with open(os.path.join(self.cw.envpath, ANSWERS_FILE), "w+") as f:
json.dump(self.answers, f)
logger.debug("add_answer_to_env() took " + str(time.time() - start))


@timeout_decorator.timeout(EVAL_TIMEOUT, use_signals=False)
def evaluate(self):
"""Execute grader.py, returning the result. """
Expand All @@ -228,6 +212,7 @@ def execute(self):
"""
Send the environnement to the docker and evaluate the student's code.
"""
stdout = None
try:
self.move_env_to_docker()
self.add_answer_to_env()
Expand All @@ -242,57 +227,55 @@ def execute(self):
if feedback == '\n':
feedback = ""
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : exit_code,
"grade" : stdout if not exit_code else -1,
"stderr" : self.get_stderr(),
"feedback" : feedback if feedback else str(stdout if not exit_code else -1),
"context" : self.get_context() if not exit_code else {},
"sandboxerr" : "",
"id": self.envid,
"status": exit_code,
"grade": stdout if not exit_code else (-1),
"stderr": self.get_stderr(),
"feedback": feedback if feedback else str(stdout if not exit_code else -1),
"context": self.get_context() if not exit_code else {},
"sandboxerr": "",
}
except timeout_decorator.TimeoutError:
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : SandboxErrCode.TIMEOUT,
"grade" : -1,
"stderr" : self.get_stderr(),
"feedback" : TIMEOUT_FEEDBACK % self.timeout,
"context" : {},
"sandboxerr" : ("Execution of the grader timed out after "
+ str(
self.timeout) + " seconds.\nThe RAM of the sandbox is currently"
+ " limited to " + settings.DOCKER_MEM_LIMIT + ", using more will "
+ "considerably slow the execution of your grader.\n"
+ "Do not forget to close every open file or to use 'with' "
+ "statement.")
"id": self.envid,
"status": SandboxErrCode.TIMEOUT,
"grade": (-1),
"stderr": self.get_stderr(),
"feedback": TIMEOUT_FEEDBACK % self.timeout,
"context": {},
"sandboxerr": ("Execution of the grader timed out after "
+ str(self.timeout)
+ " seconds.\nThe RAM of the sandbox is currently"
+ " limited to "
+ settings.DOCKER_MEM_LIMIT
+ ", using more will "
+ "considerably slow the execution of your grader.\n"
+ "Do not forget to close every open file or to use 'with' "
+ "statement.")
}
except GraderError:
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : SandboxErrCode.GRADER_NOT_INT,
"grade" : -1,
"stderr" : self.get_stderr(),
"feedback" : ("Execution of the evaluating script returned an invalid value."
+ " Please contact your teacher."),
"context" : {},
"sandboxerr" : (
"id": self.envid,
"status": SandboxErrCode.GRADER_NOT_INT,
"grade": (-1),
"stderr": self.get_stderr(),
"feedback": ("Execution of the evaluating script returned an invalid value."
+ " Please contact your teacher."),
"context": {},
"sandboxerr": (
"Grader script did not return a valid integer on stdout, received:\n"
+ ("'" + stdout + "'" if stdout else "[NOTHING]"))
+ ("'" + str(stdout) + "'" if str(stdout) else "[NOTHING]"))
}
except Exception: # Unknown error
response = {
"id" : self.envid,
"sandbox_url": self.sandbox_url,
"status" : SandboxErrCode.UNKNOWN,
"grade" : -1,
"stderr" : self.get_stderr(),
"feedback" : ("Execution of the evaluating script failed due to an unkwown error."
+ " Please contact your teacher."),
"context" : {},
"sandboxerr" : "An unknown error occured:\n" + traceback.format_exc()
"id": self.envid,
"status": SandboxErrCode.UNKNOWN,
"grade": (-1),
"stderr": self.get_stderr(),
"feedback": ("Execution of the evaluating script failed due to an unkwown error."
+ " Please contact your teacher."),
"context": {},
"sandboxerr": "An unknown error occured:\n" + traceback.format_exc()
}
logger.exception("An unknown exception occured during eval of env %s:" % self.envid)

Expand Down
1 change: 0 additions & 1 deletion sandbox/urls.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
app_name = "sandbox"

urlpatterns = [
path(r'', views.IndexView.as_view(), name="index"),
path(r'version/', views.VersionView.as_view(), name="version"),
path(r'env/<str:env>/', views.EnvView.as_view(), name="env"),
path(r'build/', views.BuildView.as_view(), name="build"),
Expand Down
26 changes: 12 additions & 14 deletions sandbox/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,16 @@ def remove_outdated_env():



def get_env_from_docker(cw, envpath, suffix):
"""Retrieve the environment from the docker and write it to envpath."""
path, ext = os.path.splitext(os.path.basename(envpath))
path = os.path.join(settings.MEDIA_ROOT, path + suffix + ext)
def get_most_recent_env(envid):
"""Returns the path to the most recent environment's tar whose name contains <envid> and any
suffix, return None if no such tar was found."""

with tarfile.open(path, "w|gz") as tar:
for name in os.listdir(cw.envpath):
tar.add(os.path.join(cw.envpath, name), arcname=name)



def get_env_and_reset(cw, envpath, suffix):
if cw is not None:
get_env_from_docker(cw, envpath, suffix)
cw.release()

def mtime(entry):
"""Return the modified time of <entry> in se ttings.MEDIA_ROOT."""
return os.stat(entry).st_mtime

entries = [os.path.join(settings.MEDIA_ROOT, e) for e in os.listdir(settings.MEDIA_ROOT)]
envs = [e for e in entries if envid in e and not os.path.splitext(e)[0].endswith(envid)]

return max(envs, key=mtime) if envs else None
Loading

0 comments on commit 6a97d29

Please sign in to comment.