From f39588238526293db433a4da05ca48222578cc5e Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 11 Feb 2025 22:28:42 +0000 Subject: [PATCH] feat: AppArmor confinement for codejail-service Allow codejail-service to actually run code (and run it securely) by giving it the needed confinement. Previously it would run but refuse to execute code since it would detect the insecure environment; now, the startup safety checks pass and the code-exec endpoint works as expected. - Add an AppArmor profile with fairly strict rules. It needs to be thoroughly vetted and to have exceptions added before it can be used in production, but it's be fine for devstack. Some parts are based on the existing edxapp apparmor config without careful review. - Apply the profile to the codejail service in docker-compose. - Add Django configs for codejail service. - Add documentation for installing the profile so that it is available for use on the dev's machine. Also: - Add configuration and documentation for edxapp to actually call the codejail service, disabled by default. (Will later want to make this default to true, once the service is working properly.) - Update image name in docker-compose to follow rename in https://github.com/edx/public-dockerfiles/pull/102 Currently edxapp gets an error back from codejail-service, and then isn't able to read that error; separate work in the app repo will be needed to fix those. (The first issue relates to python_path, and the other to not returning globals_dict when there's an emsg.) But the integration is working otherwise. --- codejail.profile | 75 ++++++++++++++++++++++++++++++ docker-compose.yml | 4 +- docs/codejail.rst | 43 +++++++++++++++++ docs/index.rst | 1 + py_configuration_files/cms.py | 7 +++ py_configuration_files/codejail.py | 25 ++++++++++ py_configuration_files/lms.py | 6 +++ 7 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 codejail.profile create mode 100644 docs/codejail.rst diff --git a/codejail.profile b/codejail.profile new file mode 100644 index 0000000000..9a47655d70 --- /dev/null +++ b/codejail.profile @@ -0,0 +1,75 @@ +# AppArmor profile for running codejail-service in devstack. +# +# #=========# +# # WARNING # +# #=========# +# +# This is not a complete and secure apparmor profile! Do not use this +# in any deployed environment (even a staging environment) without +# careful inspection and modification to fit your needs. +# +# Failure to apply a secure apparmor profile *will* likely result in a +# compromise of your environment by an attacker. +# +# We may at some point make this file good enough for confinement in +# production, but for now it is only intended to be used in devstack. + + +#include + +# Declare ABI version explicitly to ensure that confinement is +# actually applied appropriately on newer Ubuntu. +abi , + +# This outer profile applies to the entire container, and isn't as +# important. If the sandbox profile doesn't work, it's not likely that +# the outer one is going to help. But there may be some small value in +# defense-in-depth, as it's possible that a bug in the child (sandbox) +# profile isn't present in the outer one. +profile codejail_service flags=(attach_disconnected,mediate_deleted) { + #include + + # Filesystem access -- self-explanatory + file, + + # `network` is required for sudo + # TODO: Restrict this so that general network access is not permitted + network, + + # Various capabilities required for sudoing to sandbox (setuid, + # setgid, audit_write) and for sending a kill signal (kill). + capability setuid setgid audit_write kill, + + # Allow sending a kill signal to the sandbox when the execution + # runs beyond time limits. + signal (send) set=(kill) peer=codejail_service//child, + + # Allow executing this binary, but force a transition to the specified + # profile (and scrub the environment). + /sandbox/venv/bin/python Cx -> child, + + # This is the important apparmor profile -- the one that actually + # constrains the sandbox Python process. + profile child flags=(attach_disconnected,mediate_deleted) { + #include + + # Read and run binaries and libraries in the virtualenv. This + # includes the sandbox's copy of Python as well as any + # dependencies that have been installed for inclusion in + # sandboxes. + /sandbox/venv/** rm, + + # Codejail has a hardcoded reference to this file path, although the + # use of /tmp specifically may be controllable with environment variables: + # https://github.com/openedx/codejail/blob/0165d9ca351/codejail/util.py#L15 + /tmp/codejail-*/ r, + /tmp/codejail-*/** rw, + + # Allow interactive terminal during development + /dev/pts/* rw, + + # Allow receiving a kill signal from the webapp when the execution + # runs beyond time limits. + signal (receive) set=(kill) peer=codejail_service, + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 4dd1598d5f..7ba7bca713 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -726,11 +726,13 @@ services: hostname: codejail.devstack.edx stdin_open: true tty: true - image: edxops/codejail-dev:latest + image: edxops/codejail-service-dev:latest environment: DJANGO_SETTINGS_MODULE: codejail_service.settings.devstack ports: - "18030:8080" + security_opt: + - apparmor=codejail_service xqueue: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue" diff --git a/docs/codejail.rst b/docs/codejail.rst new file mode 100644 index 0000000000..2e3089b4d3 --- /dev/null +++ b/docs/codejail.rst @@ -0,0 +1,43 @@ +Codejail service +################ + +The ``codejail`` devstack component (codejail-service) requires some additional configuration before it can be enabled. This page describes how to set it up and debug it. + +Background +********** + +Both LMS and CMS can run Python code submitted by instructors and learners in order to implement custom Python-graded problems. By default this involves running the code on the same host as edxapp itself. Ordinarily this would be quite dangerous, but we use a sandboxing library called `codejail `__ in order to confine the code execution in terms of disk and network access as well as memory, CPU, and other resource limits. Part of these restrictions are implemented via AppArmor, a utility available in some Linux distributions (including Debian and Ubuntu). + +While AppArmor provides good protection, a sandbox escape could still be possible due to misconfiguration or bugs in AppArmor. For defense in depth, we're setting up a dedicated `codejail service `__ that will perform code execution for edxapp and which will allow further isolation. + +The default edxapp codejail defaults to unsafe, direct execution of Python code, and this remains true in devstack. We don't even have a way to run on-host codejail securely in devstack. In constrast, the codejail service refuses to run if codejail has not been configured properly, and we've included a way to run it in devstack. + +Configuration +************* + +In order to run the codejail devstack component: + +1. Install AppArmor: ``sudo apt install apparmor`` +2. Add the codejail AppArmor profile to your OS, or update it: ``sudo apparmor_parser --replace -W codejail.profile`` +3. Configure LMS and CMS to use the codejail-service by changing ``ENABLE_CODEJAIL_REST_SERVICE`` to ``True`` in ``py_configuration_files/{lms,cms}.py`` +4. Run ``make codejail-up`` + +The service does not need any provisioning, and does not have dependencies. + +Development +*********** + +Changes to the AppArmor profile must be coordinated with changes to the Dockerfile, as they need to agree on filesystem paths. + +Any time you update the profile, you'll need to re-run the command to apply the profile. + +The profile file contains the directive ``profile codejail_service``. That defines the name of the profile when it is installed into the kernel. In order to change that name, you must first remove the profile **under the old name**, then install a new profile under the new name. To remove a profile, use the ``--remove`` action instead of the ``-replace`` action: : ``sudo apparmor_parser --remove -W codejail.profile`` + +The profile name must also agree with the relevant ``security_opt`` line in devstack's ``docker-compose.yml``. + +Debugging +********* + +To check whether the profile has been applied, run ``sudo aa-status | grep codejail``. This won't tell you if the profile is out of date, but it will tell you if you have *some* version of it installed. + +If you need to debug the confinement, either because it is restricting too much or too little, a good strategy is to run ``tail -F /var/log/kern.log | grep codejail`` and watch for ``DENIED`` lines. You should expect to see several appear during service startup, as the service is designed to probe the confinement as part of its initial healthcheck. diff --git a/docs/index.rst b/docs/index.rst index a822ef5f61..2d2eb45cea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,3 +27,4 @@ Contents troubleshoot_general_tips manual_upgrades advanced_configuration + codejail diff --git a/py_configuration_files/cms.py b/py_configuration_files/cms.py index 1200a61b06..504af61d62 100644 --- a/py_configuration_files/cms.py +++ b/py_configuration_files/cms.py @@ -312,6 +312,13 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing xblock_duplicated_event_setting = EVENT_BUS_PRODUCER_CONFIG['org.openedx.content_authoring.xblock.duplicated.v1'] xblock_duplicated_event_setting['course-authoring-xblock-lifecycle']['enabled'] = True +############################ Codejail ############################ + +# Disabled by default since codejail service needs to be configured +# and started separately. See docs/codejail.rst for details. +ENABLE_CODEJAIL_REST_SERVICE = False +CODE_JAIL_REST_SERVICE_HOST = "http://edx.devstack.codejail:8080" + ################# New settings must go ABOVE this line ################# ######################################################################## diff --git a/py_configuration_files/codejail.py b/py_configuration_files/codejail.py index 7e466d1bff..5eb0576340 100644 --- a/py_configuration_files/codejail.py +++ b/py_configuration_files/codejail.py @@ -1,3 +1,28 @@ """Settings for devstack use.""" from codejail_service.settings.local import * # pylint: disable=wildcard-import + +ALLOWED_HOSTS = [ + # When called from outside of docker's network (dev's terminal) + 'localhost', + # When called from another container (lms, cms) + 'edx.devstack.codejail', +] + +CODEJAIL_ENABLED = True + +CODE_JAIL = { + # These values are coordinated with the Dockerfile and AppArmor profile + 'python_bin': '/sandbox/venv/bin/python', + 'user': 'sandbox', + + # Configurable limits. + 'limits': { + # CPU-seconds + 'CPU': 3, + # 100 MiB memory + 'VMEM': 100 * 1024 * 1024, + # Clock seconds + 'REALTIME': 3, + }, +} diff --git a/py_configuration_files/lms.py b/py_configuration_files/lms.py index 9aacd6d266..2d4105057d 100644 --- a/py_configuration_files/lms.py +++ b/py_configuration_files/lms.py @@ -554,6 +554,12 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing 'http://localhost:1996', # frontend-app-learner-dashboard ] +############################ Codejail ############################ + +# Disabled by default since codejail service needs to be configured +# and started separately. See docs/codejail.rst for details. +ENABLE_CODEJAIL_REST_SERVICE = False +CODE_JAIL_REST_SERVICE_HOST = "http://edx.devstack.codejail:8080" ################# New settings must go ABOVE this line ################# ########################################################################