diff --git a/codejail.profile b/codejail.profile new file mode 100644 index 0000000000..9a47655d70 --- /dev/null +++ b/codejail.profile @@ -0,0 +1,75 @@ +# AppArmor profile for running codejail-service in devstack. +# +# #=========# +# # WARNING # +# #=========# +# +# This is not a complete and secure apparmor profile! Do not use this +# in any deployed environment (even a staging environment) without +# careful inspection and modification to fit your needs. +# +# Failure to apply a secure apparmor profile *will* likely result in a +# compromise of your environment by an attacker. +# +# We may at some point make this file good enough for confinement in +# production, but for now it is only intended to be used in devstack. + + +#include + +# Declare ABI version explicitly to ensure that confinement is +# actually applied appropriately on newer Ubuntu. +abi , + +# This outer profile applies to the entire container, and isn't as +# important. If the sandbox profile doesn't work, it's not likely that +# the outer one is going to help. But there may be some small value in +# defense-in-depth, as it's possible that a bug in the child (sandbox) +# profile isn't present in the outer one. +profile codejail_service flags=(attach_disconnected,mediate_deleted) { + #include + + # Filesystem access -- self-explanatory + file, + + # `network` is required for sudo + # TODO: Restrict this so that general network access is not permitted + network, + + # Various capabilities required for sudoing to sandbox (setuid, + # setgid, audit_write) and for sending a kill signal (kill). + capability setuid setgid audit_write kill, + + # Allow sending a kill signal to the sandbox when the execution + # runs beyond time limits. + signal (send) set=(kill) peer=codejail_service//child, + + # Allow executing this binary, but force a transition to the specified + # profile (and scrub the environment). + /sandbox/venv/bin/python Cx -> child, + + # This is the important apparmor profile -- the one that actually + # constrains the sandbox Python process. + profile child flags=(attach_disconnected,mediate_deleted) { + #include + + # Read and run binaries and libraries in the virtualenv. This + # includes the sandbox's copy of Python as well as any + # dependencies that have been installed for inclusion in + # sandboxes. + /sandbox/venv/** rm, + + # Codejail has a hardcoded reference to this file path, although the + # use of /tmp specifically may be controllable with environment variables: + # https://github.com/openedx/codejail/blob/0165d9ca351/codejail/util.py#L15 + /tmp/codejail-*/ r, + /tmp/codejail-*/** rw, + + # Allow interactive terminal during development + /dev/pts/* rw, + + # Allow receiving a kill signal from the webapp when the execution + # runs beyond time limits. + signal (receive) set=(kill) peer=codejail_service, + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 4dd1598d5f..7ba7bca713 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -726,11 +726,13 @@ services: hostname: codejail.devstack.edx stdin_open: true tty: true - image: edxops/codejail-dev:latest + image: edxops/codejail-service-dev:latest environment: DJANGO_SETTINGS_MODULE: codejail_service.settings.devstack ports: - "18030:8080" + security_opt: + - apparmor=codejail_service xqueue: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" diff --git a/docs/codejail.rst b/docs/codejail.rst new file mode 100644 index 0000000000..2e3089b4d3 --- /dev/null +++ b/docs/codejail.rst @@ -0,0 +1,43 @@ +Codejail service +################ + +The ``codejail`` devstack component (codejail-service) requires some additional configuration before it can be enabled. This page describes how to set it up and debug it. + +Background +********** + +Both LMS and CMS can run Python code submitted by instructors and learners in order to implement custom Python-graded problems. By default this involves running the code on the same host as edxapp itself. Ordinarily this would be quite dangerous, but we use a sandboxing library called `codejail `__ in order to confine the code execution in terms of disk and network access as well as memory, CPU, and other resource limits. Part of these restrictions are implemented via AppArmor, a utility available in some Linux distributions (including Debian and Ubuntu). + +While AppArmor provides good protection, a sandbox escape could still be possible due to misconfiguration or bugs in AppArmor. For defense in depth, we're setting up a dedicated `codejail service `__ that will perform code execution for edxapp and which will allow further isolation. + +The default edxapp codejail defaults to unsafe, direct execution of Python code, and this remains true in devstack. We don't even have a way to run on-host codejail securely in devstack. In constrast, the codejail service refuses to run if codejail has not been configured properly, and we've included a way to run it in devstack. + +Configuration +************* + +In order to run the codejail devstack component: + +1. Install AppArmor: ``sudo apt install apparmor`` +2. Add the codejail AppArmor profile to your OS, or update it: ``sudo apparmor_parser --replace -W codejail.profile`` +3. Configure LMS and CMS to use the codejail-service by changing ``ENABLE_CODEJAIL_REST_SERVICE`` to ``True`` in ``py_configuration_files/{lms,cms}.py`` +4. Run ``make codejail-up`` + +The service does not need any provisioning, and does not have dependencies. + +Development +*********** + +Changes to the AppArmor profile must be coordinated with changes to the Dockerfile, as they need to agree on filesystem paths. + +Any time you update the profile, you'll need to re-run the command to apply the profile. + +The profile file contains the directive ``profile codejail_service``. That defines the name of the profile when it is installed into the kernel. In order to change that name, you must first remove the profile **under the old name**, then install a new profile under the new name. To remove a profile, use the ``--remove`` action instead of the ``-replace`` action: : ``sudo apparmor_parser --remove -W codejail.profile`` + +The profile name must also agree with the relevant ``security_opt`` line in devstack's ``docker-compose.yml``. + +Debugging +********* + +To check whether the profile has been applied, run ``sudo aa-status | grep codejail``. This won't tell you if the profile is out of date, but it will tell you if you have *some* version of it installed. + +If you need to debug the confinement, either because it is restricting too much or too little, a good strategy is to run ``tail -F /var/log/kern.log | grep codejail`` and watch for ``DENIED`` lines. You should expect to see several appear during service startup, as the service is designed to probe the confinement as part of its initial healthcheck. diff --git a/docs/index.rst b/docs/index.rst index a822ef5f61..2d2eb45cea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,3 +27,4 @@ Contents troubleshoot_general_tips manual_upgrades advanced_configuration + codejail diff --git a/py_configuration_files/cms.py b/py_configuration_files/cms.py index 1200a61b06..504af61d62 100644 --- a/py_configuration_files/cms.py +++ b/py_configuration_files/cms.py @@ -312,6 +312,13 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing xblock_duplicated_event_setting = EVENT_BUS_PRODUCER_CONFIG['org.openedx.content_authoring.xblock.duplicated.v1'] xblock_duplicated_event_setting['course-authoring-xblock-lifecycle']['enabled'] = True +############################ Codejail ############################ + +# Disabled by default since codejail service needs to be configured +# and started separately. See docs/codejail.rst for details. +ENABLE_CODEJAIL_REST_SERVICE = False +CODE_JAIL_REST_SERVICE_HOST = "http://edx.devstack.codejail:8080" + ################# New settings must go ABOVE this line ################# ######################################################################## diff --git a/py_configuration_files/codejail.py b/py_configuration_files/codejail.py index 7e466d1bff..5eb0576340 100644 --- a/py_configuration_files/codejail.py +++ b/py_configuration_files/codejail.py @@ -1,3 +1,28 @@ """Settings for devstack use.""" from codejail_service.settings.local import * # pylint: disable=wildcard-import + +ALLOWED_HOSTS = [ + # When called from outside of docker's network (dev's terminal) + 'localhost', + # When called from another container (lms, cms) + 'edx.devstack.codejail', +] + +CODEJAIL_ENABLED = True + +CODE_JAIL = { + # These values are coordinated with the Dockerfile and AppArmor profile + 'python_bin': '/sandbox/venv/bin/python', + 'user': 'sandbox', + + # Configurable limits. + 'limits': { + # CPU-seconds + 'CPU': 3, + # 100 MiB memory + 'VMEM': 100 * 1024 * 1024, + # Clock seconds + 'REALTIME': 3, + }, +} diff --git a/py_configuration_files/lms.py b/py_configuration_files/lms.py index 9aacd6d266..2d4105057d 100644 --- a/py_configuration_files/lms.py +++ b/py_configuration_files/lms.py @@ -554,6 +554,12 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing 'http://localhost:1996', # frontend-app-learner-dashboard ] +############################ Codejail ############################ + +# Disabled by default since codejail service needs to be configured +# and started separately. See docs/codejail.rst for details. +ENABLE_CODEJAIL_REST_SERVICE = False +CODE_JAIL_REST_SERVICE_HOST = "http://edx.devstack.codejail:8080" ################# New settings must go ABOVE this line ################# ########################################################################