From fd74df38574a45e1d75f67f5fbb1a30aacb5942e Mon Sep 17 00:00:00 2001 From: andrey-canon Date: Wed, 18 Sep 2024 16:23:57 -0500 Subject: [PATCH 01/30] feat: filter by current site organizations --- lms/djangoapps/mobile_api/users/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lms/djangoapps/mobile_api/users/views.py b/lms/djangoapps/mobile_api/users/views.py index c86f3add9d36..324db83a374c 100644 --- a/lms/djangoapps/mobile_api/users/views.py +++ b/lms/djangoapps/mobile_api/users/views.py @@ -40,6 +40,7 @@ from lms.djangoapps.courseware.views.index import save_positions_recursively_up from lms.djangoapps.mobile_api.models import MobileConfig from lms.djangoapps.mobile_api.utils import API_V1, API_V05, API_V2, API_V3, API_V4 +from openedx.core.djangoapps.site_configuration.helpers import get_current_site_orgs from openedx.features.course_duration_limits.access import check_course_expired from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order @@ -350,6 +351,11 @@ def is_org(self, check_org, course_org): """ Check course org matches request org param or no param provided """ + current_orgs = get_current_site_orgs() + + if current_orgs and course_org not in current_orgs: + return False + return check_org is None or (check_org.lower() == course_org.lower()) def get_serializer_context(self): From 825931a9b4e591df6248f359d79c5452835ec21a Mon Sep 17 00:00:00 2001 From: Marcos Date: Tue, 7 Jan 2025 10:32:08 -0300 Subject: [PATCH 02/30] chore: Removed edx-token-utils dep and moved necessary logic to the repo --- lms/djangoapps/courseware/jwt.py | 91 +++++++++++++++ lms/djangoapps/courseware/tests/test_jwt.py | 120 ++++++++++++++++++++ lms/djangoapps/courseware/utils.py | 38 +++++++ lms/djangoapps/courseware/views/views.py | 5 +- lms/envs/common.py | 17 ++- 5 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 lms/djangoapps/courseware/jwt.py create mode 100644 lms/djangoapps/courseware/tests/test_jwt.py diff --git a/lms/djangoapps/courseware/jwt.py b/lms/djangoapps/courseware/jwt.py new file mode 100644 index 000000000000..47642b869560 --- /dev/null +++ b/lms/djangoapps/courseware/jwt.py @@ -0,0 +1,91 @@ +""" +JWT Token handling and signing functions. +""" + +import json +from time import time + +from django.conf import settings +from jwkest import Expired, Invalid, MissingKey, jwk +from jwkest.jws import JWS + + +def create_jwt(lms_user_id, expires_in_seconds, additional_token_claims, now=None): + """ + Produce an encoded JWT (string) indicating some temporary permission for the indicated user. + + What permission that is must be encoded in additional_claims. + Arguments: + lms_user_id (int): LMS user ID this token is being generated for + expires_in_seconds (int): Time to token expiry, specified in seconds. + additional_token_claims (dict): Additional claims to include in the token. + now(int): optional now value for testing + """ + now = now or int(time()) + + payload = { + 'lms_user_id': lms_user_id, + 'exp': now + expires_in_seconds, + 'iat': now, + 'iss': settings.TOKEN_SIGNING['JWT_ISSUER'], + 'version': settings.TOKEN_SIGNING['JWT_SUPPORTED_VERSION'], + } + payload.update(additional_token_claims) + return _encode_and_sign(payload) + + +def _encode_and_sign(payload): + """ + Encode and sign the provided payload. + + The signing key and algorithm are pulled from settings. + """ + keys = jwk.KEYS() + + serialized_keypair = json.loads(settings.TOKEN_SIGNING['JWT_PRIVATE_SIGNING_JWK']) + keys.add(serialized_keypair) + algorithm = settings.TOKEN_SIGNING['JWT_SIGNING_ALGORITHM'] + + data = json.dumps(payload) + jws = JWS(data, alg=algorithm) + return jws.sign_compact(keys=keys) + + +def unpack_jwt(token, lms_user_id, now=None): + """ + Unpack and verify an encoded JWT. + + Validate the user and expiration. + + Arguments: + token (string): The token to be unpacked and verified. + lms_user_id (int): LMS user ID this token should match with. + now (int): Optional now value for testing. + + Returns a valid, decoded json payload (string). + """ + now = now or int(time()) + payload = _unpack_and_verify(token) + + if "lms_user_id" not in payload: + raise MissingKey("LMS user id is missing") + if "exp" not in payload: + raise MissingKey("Expiration is missing") + if payload["lms_user_id"] != lms_user_id: + raise Invalid("User does not match") + if payload["exp"] < now: + raise Expired("Token is expired") + + return payload + + +def _unpack_and_verify(token): + """ + Unpack and verify the provided token. + + The signing key and algorithm are pulled from settings. + """ + keys = jwk.KEYS() + keys.load_jwks(settings.TOKEN_SIGNING['JWT_PUBLIC_SIGNING_JWK_SET']) + decoded = JWS().verify_compact(token.encode('utf-8'), keys) + return decoded diff --git a/lms/djangoapps/courseware/tests/test_jwt.py b/lms/djangoapps/courseware/tests/test_jwt.py new file mode 100644 index 000000000000..d21ac1778fb9 --- /dev/null +++ b/lms/djangoapps/courseware/tests/test_jwt.py @@ -0,0 +1,120 @@ +""" +Tests for token handling +""" +import unittest + +from django.conf import settings +from jwkest import BadSignature, Expired, Invalid, MissingKey, jwk +from jwkest.jws import JWS + +from lms.djangoapps.courseware.jwt import _encode_and_sign, create_jwt, unpack_jwt + +import unittest +from unittest.mock import patch + + +test_user_id = 121 +invalid_test_user_id = 120 +test_timeout = 60 +test_now = 1661432902 +test_claims = {"foo": "bar", "baz": "quux", "meaning": 42} +expected_full_token = { + "lms_user_id": test_user_id, + "iat": 1661432902, + "exp": 1661432902 + 60, + "iss": "token-test-issuer", # these lines from test_settings.py + "version": "1.2.0", # these lines from test_settings.py +} + + +class TestSign(unittest.TestCase): + def test_create_jwt(self): + token = create_jwt(test_user_id, test_timeout, {}, test_now) + + decoded = _verify_jwt(token) + self.assertEqual(expected_full_token, decoded) + + def test_create_jwt_with_claims(self): + token = create_jwt(test_user_id, test_timeout, test_claims, test_now) + + expected_token_with_claims = expected_full_token.copy() + expected_token_with_claims.update(test_claims) + + decoded = _verify_jwt(token) + self.assertEqual(expected_token_with_claims, decoded) + + def test_malformed_token(self): + token = create_jwt(test_user_id, test_timeout, test_claims, test_now) + token = token + "a" + + expected_token_with_claims = expected_full_token.copy() + expected_token_with_claims.update(test_claims) + + with self.assertRaises(BadSignature): + _verify_jwt(token) + +def _verify_jwt(jwt_token): + """ + Helper function which verifies the signature and decodes the token + from string back to claims form + """ + keys = jwk.KEYS() + keys.load_jwks(settings.TOKEN_SIGNING['JWT_PUBLIC_SIGNING_JWK_SET']) + decoded = JWS().verify_compact(jwt_token.encode('utf-8'), keys) + return decoded + + +class TestUnpack(unittest.TestCase): + def test_unpack_jwt(self): + token = create_jwt(test_user_id, test_timeout, {}, test_now) + decoded = unpack_jwt(token, test_user_id, test_now) + + self.assertEqual(expected_full_token, decoded) + + def test_unpack_jwt_with_claims(self): + token = create_jwt(test_user_id, test_timeout, test_claims, test_now) + + expected_token_with_claims = expected_full_token.copy() + expected_token_with_claims.update(test_claims) + + decoded = unpack_jwt(token, test_user_id, test_now) + + self.assertEqual(expected_token_with_claims, decoded) + + def test_malformed_token(self): + token = create_jwt(test_user_id, test_timeout, test_claims, test_now) + token = token + "a" + + expected_token_with_claims = expected_full_token.copy() + expected_token_with_claims.update(test_claims) + + with self.assertRaises(BadSignature): + unpack_jwt(token, test_user_id, test_now) + + def test_unpack_token_with_invalid_user(self): + token = create_jwt(invalid_test_user_id, test_timeout, {}, test_now) + + with self.assertRaises(Invalid): + unpack_jwt(token, test_user_id, test_now) + + def test_unpack_expired_token(self): + token = create_jwt(test_user_id, test_timeout, {}, test_now) + + with self.assertRaises(Expired): + unpack_jwt(token, test_user_id, test_now + test_timeout + 1) + + def test_missing_expired_lms_user_id(self): + payload = expected_full_token.copy() + del payload['lms_user_id'] + token = _encode_and_sign(payload) + + with self.assertRaises(MissingKey): + unpack_jwt(token, test_user_id, test_now) + + def test_missing_expired_key(self): + payload = expected_full_token.copy() + del payload['exp'] + token = _encode_and_sign(payload) + + with self.assertRaises(MissingKey): + unpack_jwt(token, test_user_id, test_now) diff --git a/lms/djangoapps/courseware/utils.py b/lms/djangoapps/courseware/utils.py index 5409c89f636b..413e024d281b 100644 --- a/lms/djangoapps/courseware/utils.py +++ b/lms/djangoapps/courseware/utils.py @@ -5,6 +5,8 @@ import hashlib import logging +from time import time + from django.conf import settings from django.http import HttpResponse, HttpResponseBadRequest from edx_rest_api_client.client import OAuthAPIClient @@ -15,6 +17,9 @@ ENROLLMENT_TRACK_PARTITION_ID # lint-amnesty, pylint: disable=wrong-import-order from xmodule.partitions.partitions_service import PartitionService # lint-amnesty, pylint: disable=wrong-import-order +from jwkest import Expired, Invalid, MissingKey, jwk +from jwkest.jws import JWS + from common.djangoapps.course_modes.models import CourseMode from lms.djangoapps.commerce.utils import EcommerceService from lms.djangoapps.courseware.config import ENABLE_NEW_FINANCIAL_ASSISTANCE_FLOW @@ -229,3 +234,36 @@ def _use_new_financial_assistance_flow(course_id): ): return True return False + + + +def unpack_jwt(token, lms_user_id, now=None): + """ + Unpack and verify an encoded JWT. + + Validate the user and expiration. + + Arguments: + token (string): The token to be unpacked and verified. + lms_user_id (int): LMS user ID this token should match with. + now (int): Optional now value for testing. + + Returns a valid, decoded json payload (string). + """ + now = now or int(time()) + + # Unpack and verify token + keys = jwk.KEYS() + keys.load_jwks(settings.TOKEN_SIGNING['JWT_PUBLIC_SIGNING_JWK_SET']) + payload = JWS().verify_compact(token.encode('utf-8'), keys) + + if "lms_user_id" not in payload: + raise MissingKey("LMS user id is missing") + if "exp" not in payload: + raise MissingKey("Expiration is missing") + if payload["lms_user_id"] != lms_user_id: + raise Invalid("User does not match") + if payload["exp"] < now: + raise Expired("Token is expired") + + return payload diff --git a/lms/djangoapps/courseware/views/views.py b/lms/djangoapps/courseware/views/views.py index 6e0804db8ca0..33bd4013f3c7 100644 --- a/lms/djangoapps/courseware/views/views.py +++ b/lms/djangoapps/courseware/views/views.py @@ -46,7 +46,6 @@ from rest_framework.decorators import api_view, throttle_classes from rest_framework.response import Response from rest_framework.throttling import UserRateThrottle -from token_utils.api import unpack_token_for from web_fragments.fragment import Fragment from xmodule.course_block import ( COURSE_VISIBILITY_PUBLIC, @@ -106,7 +105,7 @@ from lms.djangoapps.courseware.utils import ( _use_new_financial_assistance_flow, create_financial_assistance_application, - is_eligible_for_financial_aid + is_eligible_for_financial_aid, unpack_jwt ) from lms.djangoapps.edxnotes.helpers import is_feature_enabled from lms.djangoapps.experiments.utils import get_experiment_user_metadata_context @@ -1535,7 +1534,7 @@ def _check_sequence_exam_access(request, location): try: # unpack will validate both expiration and the requesting user matches the # token user - exam_access_unpacked = unpack_token_for(exam_access_token, request.user.id) + exam_access_unpacked = unpack_jwt(exam_access_token, request.user.id) except: # pylint: disable=bare-except log.exception(f"Failed to validate exam access token. user_id={request.user.id} location={location}") return False diff --git a/lms/envs/common.py b/lms/envs/common.py index cb7643c3668e..e17b88fe914d 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -4315,7 +4315,22 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring 'JWT_ISSUER': 'http://127.0.0.1:8740', 'JWT_SIGNING_ALGORITHM': 'RS512', 'JWT_SUPPORTED_VERSION': '1.2.0', - 'JWT_PUBLIC_SIGNING_JWK_SET': None, + 'JWT_PUBLIC_SIGNING_JWK_SET': '''{ + "keys": [ + { + "kid":"token-test-wrong-key", + "e": "AQAB", + "kty": "RSA", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dffgRQLD1qf5D6sprmYfWVokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" + }, + { + "kid":"token-test-sign", + "e": "AQAB", + "kty": "RSA", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" + } + ] + }''', } COURSE_CATALOG_URL_ROOT = 'http://localhost:8008' From 3bcbcaac2d5a7182ad347d541690dfd7f5bb9413 Mon Sep 17 00:00:00 2001 From: Marcos Date: Tue, 7 Jan 2025 10:46:54 -0300 Subject: [PATCH 03/30] chore: Removed unused dependency --- .../workflows/check_python_dependencies.yml | 11 +++---- lms/djangoapps/courseware/tests/test_jwt.py | 12 +++++-- lms/djangoapps/courseware/utils.py | 33 ------------------- lms/djangoapps/courseware/views/views.py | 3 +- requirements/edx/base.txt | 6 ++-- requirements/edx/development.txt | 6 ++-- requirements/edx/doc.txt | 6 ++-- requirements/edx/kernel.in | 1 - requirements/edx/semgrep.txt | 2 +- requirements/edx/testing.txt | 6 ++-- scripts/xblock/requirements.txt | 6 ++-- 11 files changed, 27 insertions(+), 65 deletions(-) diff --git a/.github/workflows/check_python_dependencies.yml b/.github/workflows/check_python_dependencies.yml index 85a4e796ce78..b691e68d4be9 100644 --- a/.github/workflows/check_python_dependencies.yml +++ b/.github/workflows/check_python_dependencies.yml @@ -14,18 +14,18 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 - + - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - + - name: Install repo-tools run: pip install edx-repo-tools[find_dependencies] - name: Install setuptool - run: pip install setuptools - + run: pip install setuptools + - name: Run Python script run: | find_python_dependencies \ @@ -35,6 +35,5 @@ jobs: --ignore https://github.com/edx/braze-client \ --ignore https://github.com/edx/edx-name-affirmation \ --ignore https://github.com/mitodl/edx-sga \ - --ignore https://github.com/edx/token-utils \ --ignore https://github.com/open-craft/xblock-poll - + diff --git a/lms/djangoapps/courseware/tests/test_jwt.py b/lms/djangoapps/courseware/tests/test_jwt.py index d21ac1778fb9..a513b1fec634 100644 --- a/lms/djangoapps/courseware/tests/test_jwt.py +++ b/lms/djangoapps/courseware/tests/test_jwt.py @@ -9,9 +9,6 @@ from lms.djangoapps.courseware.jwt import _encode_and_sign, create_jwt, unpack_jwt -import unittest -from unittest.mock import patch - test_user_id = 121 invalid_test_user_id = 120 @@ -28,6 +25,10 @@ class TestSign(unittest.TestCase): + """ + Tests for JWT creation and signing. + """ + def test_create_jwt(self): token = create_jwt(test_user_id, test_timeout, {}, test_now) @@ -53,6 +54,7 @@ def test_malformed_token(self): with self.assertRaises(BadSignature): _verify_jwt(token) + def _verify_jwt(jwt_token): """ Helper function which verifies the signature and decodes the token @@ -65,6 +67,10 @@ def _verify_jwt(jwt_token): class TestUnpack(unittest.TestCase): + """ + Tests for JWT unpacking. + """ + def test_unpack_jwt(self): token = create_jwt(test_user_id, test_timeout, {}, test_now) decoded = unpack_jwt(token, test_user_id, test_now) diff --git a/lms/djangoapps/courseware/utils.py b/lms/djangoapps/courseware/utils.py index 413e024d281b..6755c5e7b5d8 100644 --- a/lms/djangoapps/courseware/utils.py +++ b/lms/djangoapps/courseware/utils.py @@ -234,36 +234,3 @@ def _use_new_financial_assistance_flow(course_id): ): return True return False - - - -def unpack_jwt(token, lms_user_id, now=None): - """ - Unpack and verify an encoded JWT. - - Validate the user and expiration. - - Arguments: - token (string): The token to be unpacked and verified. - lms_user_id (int): LMS user ID this token should match with. - now (int): Optional now value for testing. - - Returns a valid, decoded json payload (string). - """ - now = now or int(time()) - - # Unpack and verify token - keys = jwk.KEYS() - keys.load_jwks(settings.TOKEN_SIGNING['JWT_PUBLIC_SIGNING_JWK_SET']) - payload = JWS().verify_compact(token.encode('utf-8'), keys) - - if "lms_user_id" not in payload: - raise MissingKey("LMS user id is missing") - if "exp" not in payload: - raise MissingKey("Expiration is missing") - if payload["lms_user_id"] != lms_user_id: - raise Invalid("User does not match") - if payload["exp"] < now: - raise Expired("Token is expired") - - return payload diff --git a/lms/djangoapps/courseware/views/views.py b/lms/djangoapps/courseware/views/views.py index 33bd4013f3c7..65c417279875 100644 --- a/lms/djangoapps/courseware/views/views.py +++ b/lms/djangoapps/courseware/views/views.py @@ -91,6 +91,7 @@ ) from lms.djangoapps.courseware.date_summary import verified_upgrade_deadline_link from lms.djangoapps.courseware.exceptions import CourseAccessRedirect, Redirect +from lms.djangoapps.courseware.jwt import unpack_jwt from lms.djangoapps.courseware.masquerade import is_masquerading_as_specific_student, setup_masquerade from lms.djangoapps.courseware.model_data import FieldDataCache from lms.djangoapps.courseware.models import BaseStudentModuleHistory, StudentModule @@ -105,7 +106,7 @@ from lms.djangoapps.courseware.utils import ( _use_new_financial_assistance_flow, create_financial_assistance_application, - is_eligible_for_financial_aid, unpack_jwt + is_eligible_for_financial_aid ) from lms.djangoapps.edxnotes.helpers import is_feature_enabled from lms.djangoapps.experiments.utils import get_experiment_user_metadata_context diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 4861b7be1f63..4dc66c138a50 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -221,7 +221,6 @@ django==4.2.18 # edx-search # edx-submissions # edx-toggles - # edx-token-utils # edx-when # edxval # enmerkar @@ -546,7 +545,7 @@ edx-when==2.5.1 # edx-proctoring edxval==2.9.0 # via -r requirements/edx/kernel.in -elasticsearch==7.9.1 +elasticsearch==7.13.4 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -931,7 +930,6 @@ pygments==2.19.1 pyjwkest==1.4.2 # via # -r requirements/edx/kernel.in - # edx-token-utils # lti-consumer-xblock pyjwt[crypto]==2.10.1 # via @@ -1215,7 +1213,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==2.2.3 +urllib3==1.26.20 # via # -c requirements/edx/../common_constraints.txt # botocore diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index ae377bd5cc7c..84e2a26239b3 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -395,7 +395,6 @@ django==4.2.18 # edx-search # edx-submissions # edx-toggles - # edx-token-utils # edx-when # edxval # enmerkar @@ -858,7 +857,7 @@ edxval==2.9.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -elasticsearch==7.9.1 +elasticsearch==7.13.4 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -1598,7 +1597,6 @@ pyjwkest==1.4.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt - # edx-token-utils # lti-consumer-xblock pyjwt[crypto]==2.10.1 # via @@ -2172,7 +2170,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==2.2.3 +urllib3==1.26.20 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 354e85c98f55..2b7072e447f0 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -280,7 +280,6 @@ django==4.2.18 # edx-search # edx-submissions # edx-toggles - # edx-token-utils # edx-when # edxval # enmerkar @@ -637,7 +636,7 @@ edx-when==2.5.1 # edx-proctoring edxval==2.9.0 # via -r requirements/edx/base.txt -elasticsearch==7.9.1 +elasticsearch==7.13.4 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -1147,7 +1146,6 @@ pygments==2.19.1 pyjwkest==1.4.2 # via # -r requirements/edx/base.txt - # edx-token-utils # lti-consumer-xblock pyjwt[crypto]==2.10.1 # via @@ -1529,7 +1527,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==2.2.3 +urllib3==1.26.20 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index d1a132778133..a17b9db4c868 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -84,7 +84,6 @@ edx-rest-api-client edx-search edx-submissions edx-toggles # Feature toggles management -edx-token-utils # Validate exam access tokens edx-when edxval event-tracking diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index 65a0b794b9e6..0caf869df716 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -125,7 +125,7 @@ typing-extensions==4.12.2 # opentelemetry-sdk # referencing # semgrep -urllib3==2.2.3 +urllib3==2.3.0 # via # -c requirements/edx/../common_constraints.txt # requests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 799f8b16ed26..29bc1e960b16 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -306,7 +306,6 @@ django==4.2.18 # edx-search # edx-submissions # edx-toggles - # edx-token-utils # edx-when # edxval # enmerkar @@ -660,7 +659,7 @@ edx-when==2.5.1 # edx-proctoring edxval==2.9.0 # via -r requirements/edx/base.txt -elasticsearch==7.9.1 +elasticsearch==7.13.4 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -1211,7 +1210,6 @@ pygments==2.19.1 pyjwkest==1.4.2 # via # -r requirements/edx/base.txt - # edx-token-utils # lti-consumer-xblock pyjwt[crypto]==2.10.1 # via @@ -1615,7 +1613,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==2.2.3 +urllib3==1.26.20 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt diff --git a/scripts/xblock/requirements.txt b/scripts/xblock/requirements.txt index 9af137853dc5..aabed41ba0a9 100644 --- a/scripts/xblock/requirements.txt +++ b/scripts/xblock/requirements.txt @@ -14,7 +14,5 @@ idna==3.10 # via requests requests==2.32.3 # via -r scripts/xblock/requirements.in -urllib3==2.2.3 - # via - # -c scripts/xblock/../../requirements/common_constraints.txt - # requests +urllib3==2.3.0 + # via requests From 3bbf8aca7f7365811a53bd9f1cf4f9aa087f22c9 Mon Sep 17 00:00:00 2001 From: Marcos Date: Tue, 7 Jan 2025 10:46:54 -0300 Subject: [PATCH 04/30] chore: Run make compile-requirements to update dependencies --- lms/djangoapps/courseware/utils.py | 5 ----- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/semgrep.txt | 2 +- requirements/edx/testing.txt | 2 +- scripts/xblock/requirements.txt | 6 ++++-- 7 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lms/djangoapps/courseware/utils.py b/lms/djangoapps/courseware/utils.py index 6755c5e7b5d8..5409c89f636b 100644 --- a/lms/djangoapps/courseware/utils.py +++ b/lms/djangoapps/courseware/utils.py @@ -5,8 +5,6 @@ import hashlib import logging -from time import time - from django.conf import settings from django.http import HttpResponse, HttpResponseBadRequest from edx_rest_api_client.client import OAuthAPIClient @@ -17,9 +15,6 @@ ENROLLMENT_TRACK_PARTITION_ID # lint-amnesty, pylint: disable=wrong-import-order from xmodule.partitions.partitions_service import PartitionService # lint-amnesty, pylint: disable=wrong-import-order -from jwkest import Expired, Invalid, MissingKey, jwk -from jwkest.jws import JWS - from common.djangoapps.course_modes.models import CourseMode from lms.djangoapps.commerce.utils import EcommerceService from lms.djangoapps.courseware.config import ENABLE_NEW_FINANCIAL_ASSISTANCE_FLOW diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 4dc66c138a50..b9179d11ffab 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -545,7 +545,7 @@ edx-when==2.5.1 # edx-proctoring edxval==2.9.0 # via -r requirements/edx/kernel.in -elasticsearch==7.13.4 +elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 84e2a26239b3..8ec4d878f0d4 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -857,7 +857,7 @@ edxval==2.9.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -elasticsearch==7.13.4 +elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 2b7072e447f0..1619e3972557 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -636,7 +636,7 @@ edx-when==2.5.1 # edx-proctoring edxval==2.9.0 # via -r requirements/edx/base.txt -elasticsearch==7.13.4 +elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index 0caf869df716..65a0b794b9e6 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -125,7 +125,7 @@ typing-extensions==4.12.2 # opentelemetry-sdk # referencing # semgrep -urllib3==2.3.0 +urllib3==2.2.3 # via # -c requirements/edx/../common_constraints.txt # requests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 29bc1e960b16..8f007851e635 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -659,7 +659,7 @@ edx-when==2.5.1 # edx-proctoring edxval==2.9.0 # via -r requirements/edx/base.txt -elasticsearch==7.13.4 +elasticsearch==7.9.1 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/scripts/xblock/requirements.txt b/scripts/xblock/requirements.txt index aabed41ba0a9..9af137853dc5 100644 --- a/scripts/xblock/requirements.txt +++ b/scripts/xblock/requirements.txt @@ -14,5 +14,7 @@ idna==3.10 # via requests requests==2.32.3 # via -r scripts/xblock/requirements.in -urllib3==2.3.0 - # via requests +urllib3==2.2.3 + # via + # -c scripts/xblock/../../requirements/common_constraints.txt + # requests From ca3cd45a7954bb5509c88ce6b0a8fd6883a5ce8b Mon Sep 17 00:00:00 2001 From: Marcos Date: Thu, 9 Jan 2025 09:38:53 -0300 Subject: [PATCH 05/30] chore: fix tests --- lms/djangoapps/courseware/tests/test_views.py | 8 ++--- lms/envs/common.py | 18 ++--------- lms/envs/test.py | 32 +++++++++++++++++++ 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/lms/djangoapps/courseware/tests/test_views.py b/lms/djangoapps/courseware/tests/test_views.py index 2bedcd0d8e33..da31ebccce45 100644 --- a/lms/djangoapps/courseware/tests/test_views.py +++ b/lms/djangoapps/courseware/tests/test_views.py @@ -2933,9 +2933,9 @@ def test_render_xblock_with_course_duration_limits_in_mobile_browser(self, mock_ ) @ddt.unpack @patch.dict('django.conf.settings.FEATURES', {'ENABLE_PROCTORED_EXAMS': True}) - @patch('lms.djangoapps.courseware.views.views.unpack_token_for') + @patch('lms.djangoapps.courseware.views.views.unpack_jwt') def test_render_descendant_of_exam_gated_by_access_token(self, exam_access_token, - expected_response, _mock_token_unpack): + expected_response, _mock_unpack_jwt): """ Verify blocks inside an exam that requires token access are gated by a valid exam access JWT issued for that exam sequence. @@ -2968,7 +2968,7 @@ def test_render_descendant_of_exam_gated_by_access_token(self, exam_access_token CourseOverview.load_from_module_store(self.course.id) self.setup_user(admin=False, enroll=True, login=True) - def _mock_token_unpack_fn(token, user_id): + def _mock_unpack_jwt_fn(token, user_id): if token == 'valid-jwt-for-exam-sequence': return {'content_id': str(self.sequence.location)} elif token == 'valid-jwt-for-incorrect-sequence': @@ -2976,7 +2976,7 @@ def _mock_token_unpack_fn(token, user_id): else: raise Exception('invalid JWT') - _mock_token_unpack.side_effect = _mock_token_unpack_fn + _mock_unpack_jwt.side_effect = _mock_unpack_jwt_fn # Problem and Vertical response should be gated on access token for block in [self.problem_block, self.vertical_block]: diff --git a/lms/envs/common.py b/lms/envs/common.py index e17b88fe914d..5ca5bcac9a4a 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -4315,22 +4315,8 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring 'JWT_ISSUER': 'http://127.0.0.1:8740', 'JWT_SIGNING_ALGORITHM': 'RS512', 'JWT_SUPPORTED_VERSION': '1.2.0', - 'JWT_PUBLIC_SIGNING_JWK_SET': '''{ - "keys": [ - { - "kid":"token-test-wrong-key", - "e": "AQAB", - "kty": "RSA", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dffgRQLD1qf5D6sprmYfWVokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" - }, - { - "kid":"token-test-sign", - "e": "AQAB", - "kty": "RSA", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" - } - ] - }''', + 'JWT_PRIVATE_SIGNING_JWK': None, + 'JWT_PUBLIC_SIGNING_JWK_SET': None, } COURSE_CATALOG_URL_ROOT = 'http://localhost:8008' diff --git a/lms/envs/test.py b/lms/envs/test.py index a9e8aaf9f2e2..38c12370f1c7 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -657,3 +657,35 @@ # case of new django version these values will override. if django.VERSION[0] >= 4: # for greater than django 3.2 use with schemes. CSRF_TRUSTED_ORIGINS = CSRF_TRUSTED_ORIGINS_WITH_SCHEME + + +############## Settings for JWT token handling ############## +TOKEN_SIGNING = { + 'JWT_ISSUER': 'token-test-issuer', + 'JWT_SIGNING_ALGORITHM': 'RS512', + 'JWT_SUPPORTED_VERSION': '1.2.0', + 'JWT_PRIVATE_SIGNING_JWK': '''{ + "e": "AQAB", + "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", + "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", + "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", + "kid": "token-test-sign", "kty": "RSA" + }''', + 'JWT_PUBLIC_SIGNING_JWK_SET': '''{ + "keys": [ + { + "kid":"token-test-wrong-key", + "e": "AQAB", + "kty": "RSA", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dffgRQLD1qf5D6sprmYfWVokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" + }, + { + "kid":"token-test-sign", + "e": "AQAB", + "kty": "RSA", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" + } + ] + }''', +} From 67a56c270fa51cce4fec7ca9f3b1f9d5059686b4 Mon Sep 17 00:00:00 2001 From: Marcos Date: Wed, 15 Jan 2025 15:04:38 -0300 Subject: [PATCH 06/30] chore: Moved jwt file to openedx.core.lib --- lms/djangoapps/courseware/views/views.py | 2 +- {lms/djangoapps/courseware => openedx/core/lib}/jwt.py | 0 .../courseware => openedx/core/lib}/tests/test_jwt.py | 2 +- requirements/edx/base.txt | 5 +---- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 7 files changed, 6 insertions(+), 9 deletions(-) rename {lms/djangoapps/courseware => openedx/core/lib}/jwt.py (100%) rename {lms/djangoapps/courseware => openedx/core/lib}/tests/test_jwt.py (97%) diff --git a/lms/djangoapps/courseware/views/views.py b/lms/djangoapps/courseware/views/views.py index 65c417279875..19eabe692997 100644 --- a/lms/djangoapps/courseware/views/views.py +++ b/lms/djangoapps/courseware/views/views.py @@ -91,7 +91,6 @@ ) from lms.djangoapps.courseware.date_summary import verified_upgrade_deadline_link from lms.djangoapps.courseware.exceptions import CourseAccessRedirect, Redirect -from lms.djangoapps.courseware.jwt import unpack_jwt from lms.djangoapps.courseware.masquerade import is_masquerading_as_specific_student, setup_masquerade from lms.djangoapps.courseware.model_data import FieldDataCache from lms.djangoapps.courseware.models import BaseStudentModuleHistory, StudentModule @@ -138,6 +137,7 @@ from openedx.core.djangoapps.zendesk_proxy.utils import create_zendesk_ticket from openedx.core.djangolib.markup import HTML, Text from openedx.core.lib.courses import get_course_by_id +from openedx.core.lib.jwt import unpack_jwt from openedx.core.lib.mobile_utils import is_request_from_mobile_app from openedx.features.course_duration_limits.access import generate_course_expired_fragment from openedx.features.course_experience import course_home_url diff --git a/lms/djangoapps/courseware/jwt.py b/openedx/core/lib/jwt.py similarity index 100% rename from lms/djangoapps/courseware/jwt.py rename to openedx/core/lib/jwt.py diff --git a/lms/djangoapps/courseware/tests/test_jwt.py b/openedx/core/lib/tests/test_jwt.py similarity index 97% rename from lms/djangoapps/courseware/tests/test_jwt.py rename to openedx/core/lib/tests/test_jwt.py index a513b1fec634..79caf0207fa1 100644 --- a/lms/djangoapps/courseware/tests/test_jwt.py +++ b/openedx/core/lib/tests/test_jwt.py @@ -7,7 +7,7 @@ from jwkest import BadSignature, Expired, Invalid, MissingKey, jwk from jwkest.jws import JWS -from lms.djangoapps.courseware.jwt import _encode_and_sign, create_jwt, unpack_jwt +from openedx.core.lib.jwt import _encode_and_sign, create_jwt, unpack_jwt test_user_id = 121 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index b9179d11ffab..c8f2af138c64 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -80,9 +80,6 @@ boto3==1.36.3 # ora2 botocore==1.36.3 # via - # -r requirements/edx/kernel.in - # boto3 - # s3transfer bridgekeeper==0.9 # via -r requirements/edx/kernel.in cachecontrol==0.14.2 @@ -1213,7 +1210,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==1.26.20 +urllib3==2.2.3 # via # -c requirements/edx/../common_constraints.txt # botocore diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 8ec4d878f0d4..61305f1cbc5b 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -2170,7 +2170,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==1.26.20 +urllib3==2.2.3 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 1619e3972557..c08b5b6fb396 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1527,7 +1527,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==1.26.20 +urllib3==2.2.3 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 8f007851e635..ae5aed4234f4 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1613,7 +1613,7 @@ uritemplate==4.1.1 # drf-spectacular # drf-yasg # google-api-python-client -urllib3==1.26.20 +urllib3==2.2.3 # via # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt From 6e9888974836b1622f8546cdc5f00594c247a950 Mon Sep 17 00:00:00 2001 From: Marcos Date: Wed, 15 Jan 2025 15:50:50 -0300 Subject: [PATCH 07/30] chore: Updated TOKEN_SIGNING on cms --- cms/envs/test.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cms/envs/test.py b/cms/envs/test.py index 49db50608858..d391ccba5e98 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -343,3 +343,34 @@ } } } + +############## Settings for JWT token handling ############## +TOKEN_SIGNING = { + 'JWT_ISSUER': 'token-test-issuer', + 'JWT_SIGNING_ALGORITHM': 'RS512', + 'JWT_SUPPORTED_VERSION': '1.2.0', + 'JWT_PRIVATE_SIGNING_JWK': '''{ + "e": "AQAB", + "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", + "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", + "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", + "kid": "token-test-sign", "kty": "RSA" + }''', + 'JWT_PUBLIC_SIGNING_JWK_SET': '''{ + "keys": [ + { + "kid":"token-test-wrong-key", + "e": "AQAB", + "kty": "RSA", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dffgRQLD1qf5D6sprmYfWVokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" + }, + { + "kid":"token-test-sign", + "e": "AQAB", + "kty": "RSA", + "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" + } + ] + }''', +} From 5d494feb0197f08827b13ca49e272e7c6b4eab82 Mon Sep 17 00:00:00 2001 From: Marcos Date: Wed, 15 Jan 2025 16:10:04 -0300 Subject: [PATCH 08/30] chore: Updated defaults for token handling on CMS --- cms/envs/common.py | 9 +++++++++ lms/envs/common.py | 1 + 2 files changed, 10 insertions(+) diff --git a/cms/envs/common.py b/cms/envs/common.py index 591247388a9d..8f068a5c0072 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2530,6 +2530,15 @@ EXAMS_SERVICE_URL = 'http://localhost:18740/api/v1' EXAMS_SERVICE_USERNAME = 'edx_exams_worker' +############## Settings for JWT token handling ############## +TOKEN_SIGNING = { + 'JWT_ISSUER': 'http://127.0.0.1:8740', + 'JWT_SIGNING_ALGORITHM': 'RS512', + 'JWT_SUPPORTED_VERSION': '1.2.0', + 'JWT_PRIVATE_SIGNING_JWK': None, + 'JWT_PUBLIC_SIGNING_JWK_SET': None, +} + FINANCIAL_REPORTS = { 'STORAGE_TYPE': 'localfs', 'BUCKET': None, diff --git a/lms/envs/common.py b/lms/envs/common.py index 5ca5bcac9a4a..8c966e67f6d0 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -4311,6 +4311,7 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # Exam Service EXAMS_SERVICE_URL = 'http://localhost:18740/api/v1' +############## Settings for JWT token handling ############## TOKEN_SIGNING = { 'JWT_ISSUER': 'http://127.0.0.1:8740', 'JWT_SIGNING_ALGORITHM': 'RS512', From 747289bc1a748573889be4b7f777ecc19bfd7b07 Mon Sep 17 00:00:00 2001 From: Marcos Date: Thu, 23 Jan 2025 12:41:37 -0300 Subject: [PATCH 09/30] fix: Removed JWT constants from CMS and added comments on how to generate them --- cms/envs/common.py | 9 --------- cms/envs/test.py | 31 ------------------------------ lms/envs/common.py | 6 ++++++ openedx/core/lib/tests/test_jwt.py | 3 +++ requirements/edx/base.txt | 5 +++-- requirements/edx/development.txt | 4 ---- requirements/edx/doc.txt | 2 -- requirements/edx/testing.txt | 2 -- 8 files changed, 12 insertions(+), 50 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 8f068a5c0072..591247388a9d 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2530,15 +2530,6 @@ EXAMS_SERVICE_URL = 'http://localhost:18740/api/v1' EXAMS_SERVICE_USERNAME = 'edx_exams_worker' -############## Settings for JWT token handling ############## -TOKEN_SIGNING = { - 'JWT_ISSUER': 'http://127.0.0.1:8740', - 'JWT_SIGNING_ALGORITHM': 'RS512', - 'JWT_SUPPORTED_VERSION': '1.2.0', - 'JWT_PRIVATE_SIGNING_JWK': None, - 'JWT_PUBLIC_SIGNING_JWK_SET': None, -} - FINANCIAL_REPORTS = { 'STORAGE_TYPE': 'localfs', 'BUCKET': None, diff --git a/cms/envs/test.py b/cms/envs/test.py index d391ccba5e98..49db50608858 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -343,34 +343,3 @@ } } } - -############## Settings for JWT token handling ############## -TOKEN_SIGNING = { - 'JWT_ISSUER': 'token-test-issuer', - 'JWT_SIGNING_ALGORITHM': 'RS512', - 'JWT_SUPPORTED_VERSION': '1.2.0', - 'JWT_PRIVATE_SIGNING_JWK': '''{ - "e": "AQAB", - "d": "HIiV7KNjcdhVbpn3KT-I9n3JPf5YbGXsCIedmPqDH1d4QhBofuAqZ9zebQuxkRUpmqtYMv0Zi6ECSUqH387GYQF_XvFUFcjQRPycISd8TH0DAKaDpGr-AYNshnKiEtQpINhcP44I1AYNPCwyoxXA1fGTtmkKChsuWea7o8kytwU5xSejvh5-jiqu2SF4GEl0BEXIAPZsgbzoPIWNxgO4_RzNnWs6nJZeszcaDD0CyezVSuH9QcI6g5QFzAC_YuykSsaaFJhZ05DocBsLczShJ9Omf6PnK9xlm26I84xrEh_7x4fVmNBg3xWTLh8qOnHqGko93A1diLRCrKHOvnpvgQ", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ", - "q": "3T3DEtBUka7hLGdIsDlC96Uadx_q_E4Vb1cxx_4Ss_wGp1Loz3N3ZngGyInsKlmbBgLo1Ykd6T9TRvRNEWEtFSOcm2INIBoVoXk7W5RuPa8Cgq2tjQj9ziGQ08JMejrPlj3Q1wmALJr5VTfvSYBu0WkljhKNCy1KB6fCby0C9WE", - "p": "vUqzWPZnDG4IXyo-k5F0bHV0BNL_pVhQoLW7eyFHnw74IOEfSbdsMspNcPSFIrtgPsn7981qv3lN_staZ6JflKfHayjB_lvltHyZxfl0dvruShZOx1N6ykEo7YrAskC_qxUyrIvqmJ64zPW3jkuOYrFs7Ykj3zFx3Zq1H5568G0", - "kid": "token-test-sign", "kty": "RSA" - }''', - 'JWT_PUBLIC_SIGNING_JWK_SET': '''{ - "keys": [ - { - "kid":"token-test-wrong-key", - "e": "AQAB", - "kty": "RSA", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dffgRQLD1qf5D6sprmYfWVokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" - }, - { - "kid":"token-test-sign", - "e": "AQAB", - "kty": "RSA", - "n": "o5cn3ljSRi6FaDEKTn0PS-oL9EFyv1pI7dRgffQLD1qf5D6sprmYfWWokSsrWig8u2y0HChSygR6Jn5KXBqQn6FpM0dDJLnWQDRXHLl3Ey1iPYgDSmOIsIGrV9ZyNCQwk03wAgWbfdBTig3QSDYD-sTNOs3pc4UD_PqAvU2nz_1SS2ZiOwOn5F6gulE1L0iE3KEUEvOIagfHNVhz0oxa_VRZILkzV-zr6R_TW1m97h4H8jXl_VJyQGyhMGGypuDrQ9_vaY_RLEulLCyY0INglHWQ7pckxBtI5q55-Vio2wgewe2_qYcGsnBGaDNbySAsvYcWRrqDiFyzrJYivodqTQ" - } - ] - }''', -} diff --git a/lms/envs/common.py b/lms/envs/common.py index 8c966e67f6d0..b89f20e3fe85 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -4320,6 +4320,12 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring 'JWT_PUBLIC_SIGNING_JWK_SET': None, } +# NOTE: In order to create both JWT_PRIVATE_SIGNING_JWK and JWT_PUBLIC_SIGNING_JWK_SET, +# start devstack on an lms shell and then run the command: +# > python manage.py lms generate_jwt_signing_key +# This will output asymmetric JWTs to use here. Read more on this on: +# https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst + COURSE_CATALOG_URL_ROOT = 'http://localhost:8008' COURSE_CATALOG_API_URL = f'{COURSE_CATALOG_URL_ROOT}/api/v1' diff --git a/openedx/core/lib/tests/test_jwt.py b/openedx/core/lib/tests/test_jwt.py index 79caf0207fa1..7a678dd3c09b 100644 --- a/openedx/core/lib/tests/test_jwt.py +++ b/openedx/core/lib/tests/test_jwt.py @@ -7,6 +7,7 @@ from jwkest import BadSignature, Expired, Invalid, MissingKey, jwk from jwkest.jws import JWS +from openedx.core.djangolib.testing.utils import skip_unless_lms from openedx.core.lib.jwt import _encode_and_sign, create_jwt, unpack_jwt @@ -24,6 +25,7 @@ } +@skip_unless_lms class TestSign(unittest.TestCase): """ Tests for JWT creation and signing. @@ -66,6 +68,7 @@ def _verify_jwt(jwt_token): return decoded +@skip_unless_lms class TestUnpack(unittest.TestCase): """ Tests for JWT unpacking. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c8f2af138c64..bd412f92132a 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -80,6 +80,9 @@ boto3==1.36.3 # ora2 botocore==1.36.3 # via + # -r requirements/edx/kernel.in + # boto3 + # s3transfer bridgekeeper==0.9 # via -r requirements/edx/kernel.in cachecontrol==0.14.2 @@ -534,8 +537,6 @@ edx-toggles==5.2.0 # edxval # event-tracking # ora2 -edx-token-utils==0.2.1 - # via -r requirements/edx/kernel.in edx-when==2.5.1 # via # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 61305f1cbc5b..6a8c617d41be 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -844,10 +844,6 @@ edx-toggles==5.2.0 # edxval # event-tracking # ora2 -edx-token-utils==0.2.1 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt edx-when==2.5.1 # via # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index c08b5b6fb396..66bd24f8fcfc 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -628,8 +628,6 @@ edx-toggles==5.2.0 # edxval # event-tracking # ora2 -edx-token-utils==0.2.1 - # via -r requirements/edx/base.txt edx-when==2.5.1 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index ae5aed4234f4..27fe32c152b2 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -651,8 +651,6 @@ edx-toggles==5.2.0 # edxval # event-tracking # ora2 -edx-token-utils==0.2.1 - # via -r requirements/edx/base.txt edx-when==2.5.1 # via # -r requirements/edx/base.txt From dd86710d9d41d92506881ba3009d1be2b6ebad74 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 23 Jan 2025 16:16:19 -0500 Subject: [PATCH 10/30] docs: Update lms/envs/common.py --- lms/envs/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index b89f20e3fe85..76127d062d81 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -4321,7 +4321,7 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring } # NOTE: In order to create both JWT_PRIVATE_SIGNING_JWK and JWT_PUBLIC_SIGNING_JWK_SET, -# start devstack on an lms shell and then run the command: +# in an lms shell run the following command: # > python manage.py lms generate_jwt_signing_key # This will output asymmetric JWTs to use here. Read more on this on: # https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst From 3eab7ea27f92f4dc09401450fc840c77a26019b0 Mon Sep 17 00:00:00 2001 From: andrey-canon Date: Thu, 23 Jan 2025 18:07:05 -0500 Subject: [PATCH 11/30] feat: add site-aware unit test --- lms/djangoapps/mobile_api/users/tests.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lms/djangoapps/mobile_api/users/tests.py b/lms/djangoapps/mobile_api/users/tests.py index 2cfefaa058d9..7c4b3e437de5 100644 --- a/lms/djangoapps/mobile_api/users/tests.py +++ b/lms/djangoapps/mobile_api/users/tests.py @@ -285,6 +285,35 @@ def test_org_query(self, api_version): for entry in courses: assert entry['course']['org'] == 'edX' + @ddt.data(API_V05, API_V1, API_V2) + @patch('lms.djangoapps.mobile_api.users.views.get_current_site_orgs', return_value=['edX']) + def test_filter_by_current_site_orgs(self, api_version, get_current_site_orgs_mock): + self.login() + + # Create list of courses with various organizations + courses = [ + CourseFactory.create(org='edX', mobile_available=True), + CourseFactory.create(org='edX', mobile_available=True), + CourseFactory.create(org='edX', mobile_available=True, visible_to_staff_only=True), + CourseFactory.create(org='Proversity.org', mobile_available=True), + CourseFactory.create(org='MITx', mobile_available=True), + CourseFactory.create(org='HarvardX', mobile_available=True), + ] + + # Enroll in all the courses + for course in courses: + self.enroll(course.id) + + response = self.api_response(api_version=api_version) + courses = response.data['enrollments'] if api_version == API_V2 else response.data + + # Test for 3 expected courses + self.assertEqual(len(courses), 3) + + # Verify only edX courses are returned + for entry in courses: + self.assertEqual(entry['course']['org'], 'edX') + def create_enrollment(self, expired): """ Create an enrollment From 5f13b5ed3050cf99e989115f812edf378e9d38df Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Fri, 24 Jan 2025 19:58:01 +0500 Subject: [PATCH 12/30] fix: remove publish-ci-docker-image workflow file (#36164) --- .github/workflows/publish-ci-docker-image.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .github/workflows/publish-ci-docker-image.yml diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml deleted file mode 100644 index e69de29bb2d1..000000000000 From da348fa45633269b188d97c3d38c962680d5127d Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Fri, 24 Jan 2025 07:27:55 -0800 Subject: [PATCH 13/30] feat!: Removing the long-deprecated legacy course_modes chooser (#36156) * feat!: Removing the long-deprecated legacy course_modes chooser `course_modes/choose.html` (and its corresponding `_upgrade_button.html`) were specifically only used for the edge case where an enterprise user found themselves in the non-enterprise learner dashboard, and attempted to enroll in a course outside of the enterprise flow. Once upon a time, in a 2U-only workflow, the commerce system would apply specific discounts for users within the said case. That's no longer true, and it has never been true outside of this one company. Removing this template cleans up a legacy version of a legacy page that was, realistically, exclusively seen by employees of 2U, and nobody else. Removes: * The corresponding testsfor behavior only seen in the legacy page. * A waffle flag since all cases route as if the flag is set: `VALUE_PROP_TRACK_SELECTION_FLAG`: `course_modes.use_new_track_selection` * Some variables set in `CourseModeView` which were only ever rendered in the legacy template (`title_content`, `has_credit_upsell`) have been removed from the class. * There is a high likelihood that the class is still a target for re-factoring now that the legacy view is gone, but I'm hesitant to touch something which is not covered by previously existing tests, because the logic around what template gets rendered when is complex. FIXES: APER-3779 FIXES: #36090 --- .../course_modes/tests/test_views.py | 140 +++-------- common/djangoapps/course_modes/views.py | 52 +--- .../course_modes/_upgrade_button.html | 36 --- lms/templates/course_modes/choose.html | 233 ------------------ 4 files changed, 41 insertions(+), 420 deletions(-) delete mode 100644 lms/templates/course_modes/_upgrade_button.html delete mode 100644 lms/templates/course_modes/choose.html diff --git a/common/djangoapps/course_modes/tests/test_views.py b/common/djangoapps/course_modes/tests/test_views.py index d7b069865317..b4a777d2d677 100644 --- a/common/djangoapps/course_modes/tests/test_views.py +++ b/common/djangoapps/course_modes/tests/test_views.py @@ -21,7 +21,6 @@ from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory from common.djangoapps.util.testing import UrlResetMixin from common.djangoapps.util.tests.mixins.discovery import CourseCatalogServiceMockMixin -from edx_toggles.toggles.testutils import override_waffle_flag # lint-amnesty, pylint: disable=wrong-import-order from lms.djangoapps.commerce.tests import test_utils as ecomm_test_utils from lms.djangoapps.commerce.tests.mocks import mock_payment_processors from lms.djangoapps.verify_student.services import IDVerificationService @@ -33,8 +32,6 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order -from ..views import VALUE_PROP_TRACK_SELECTION_FLAG - # Name of the method to mock for Content Type Gating. GATING_METHOD_NAME = 'openedx.features.content_type_gating.models.ContentTypeGatingConfig.enabled_for_enrollment' @@ -186,27 +183,6 @@ def test_suggested_prices(self, price_list): # TODO: Fix it so that response.templates works w/ mako templates, and then assert # that the right template rendered - @httpretty.activate - @ddt.data( - (['honor', 'verified', 'credit'], True), - (['honor', 'verified'], False), - ) - @ddt.unpack - def test_credit_upsell_message(self, available_modes, show_upsell): - # Create the course modes - for mode in available_modes: - CourseModeFactory.create(mode_slug=mode, course_id=self.course.id) - - # Check whether credit upsell is shown on the page - # This should *only* be shown when a credit mode is available - url = reverse('course_modes_choose', args=[str(self.course.id)]) - response = self.client.get(url) - - if show_upsell: - self.assertContains(response, "Credit") - else: - self.assertNotContains(response, "Credit") - @httpretty.activate @patch('common.djangoapps.course_modes.views.enterprise_customer_for_request') @patch('common.djangoapps.course_modes.views.get_course_final_price') @@ -240,29 +216,6 @@ def test_display_after_discounted_price( self.assertContains(response, discounted_price) self.assertContains(response, verified_mode.min_price) - @httpretty.activate - @ddt.data(True, False) - def test_congrats_on_enrollment_message(self, create_enrollment): - # Create the course mode - CourseModeFactory.create(mode_slug='verified', course_id=self.course.id) - - if create_enrollment: - CourseEnrollmentFactory( - is_active=True, - course_id=self.course.id, - user=self.user - ) - - # Check whether congratulations message is shown on the page - # This should *only* be shown when an enrollment exists - url = reverse('course_modes_choose', args=[str(self.course.id)]) - response = self.client.get(url) - - if create_enrollment: - self.assertContains(response, "Congratulations! You are now enrolled in") - else: - self.assertNotContains(response, "Congratulations! You are now enrolled in") - @ddt.data('professional', 'no-id-professional') def test_professional_enrollment(self, mode): # The only course mode is professional ed @@ -529,26 +482,24 @@ def test_errors(self, has_perm, post_params, error_msg, status_code, mock_has_pe for mode in ('audit', 'honor', 'verified'): CourseModeFactory.create(mode_slug=mode, course_id=self.course.id) - # Value Prop TODO (REV-2378): remove waffle flag from tests once flag is removed. - with override_waffle_flag(VALUE_PROP_TRACK_SELECTION_FLAG, active=True): - mock_has_perm.return_value = has_perm - url = reverse('course_modes_choose', args=[str(self.course.id)]) + mock_has_perm.return_value = has_perm + url = reverse('course_modes_choose', args=[str(self.course.id)]) - # Choose mode (POST request) - response = self.client.post(url, post_params) - self.assertEqual(response.status_code, status_code) + # Choose mode (POST request) + response = self.client.post(url, post_params) + self.assertEqual(response.status_code, status_code) - if has_perm: - self.assertContains(response, error_msg) - self.assertContains(response, 'Sorry, we were unable to enroll you') + if has_perm: + self.assertContains(response, error_msg) + self.assertContains(response, 'Sorry, we were unable to enroll you') - # Check for CTA button on error page - marketing_root = settings.MKTG_URLS.get('ROOT') - search_courses_url = urljoin(marketing_root, '/search?tab=course') - self.assertContains(response, search_courses_url) - self.assertContains(response, 'Explore all courses') - else: - self.assertTrue(CourseEnrollment.is_enrollment_closed(self.user, self.course)) + # Check for CTA button on error page + marketing_root = settings.MKTG_URLS.get('ROOT') + search_courses_url = urljoin(marketing_root, '/search?tab=course') + self.assertContains(response, search_courses_url) + self.assertContains(response, 'Explore all courses') + else: + self.assertTrue(CourseEnrollment.is_enrollment_closed(self.user, self.course)) def _assert_fbe_page(self, response, min_price=None, **_): """ @@ -607,33 +558,19 @@ def _assert_unfbe_page(self, response, min_price=None, **_): # Check for the HTML element for courses with more than one mode self.assertContains(response, '
') - def _assert_legacy_page(self, response, **_): - """ - Assert choose.html was rendered. - """ - # Check for string unique to the legacy choose.html. - self.assertContains(response, "Choose Your Track") - # This string only occurs in lms/templates/course_modes/choose.html - # and related theme and translation files. - @override_settings(MKTG_URLS={'ROOT': 'https://www.example.edx.org'}) @ddt.data( - # gated_content_on, course_duration_limits_on, waffle_flag_on, expected_page_assertion_function - (True, True, True, _assert_fbe_page), - (True, False, True, _assert_unfbe_page), - (False, True, True, _assert_unfbe_page), - (False, False, True, _assert_unfbe_page), - (True, True, False, _assert_legacy_page), - (True, False, False, _assert_legacy_page), - (False, True, False, _assert_legacy_page), - (False, False, False, _assert_legacy_page), + # gated_content_on, course_duration_limits_on, expected_page_assertion_function + (True, True, _assert_fbe_page), + (True, False, _assert_unfbe_page), + (False, True, _assert_unfbe_page), + (False, False, _assert_unfbe_page), ) @ddt.unpack def test_track_selection_types( self, gated_content_on, course_duration_limits_on, - waffle_flag_on, expected_page_assertion_function ): """ @@ -644,7 +581,6 @@ def test_track_selection_types( verified course modes), the learner may view 3 different pages: 1. fbe.html - full FBE 2. unfbe.html - partial or no FBE - 3. choose.html - legacy track selection page This test checks that the right template is rendered. @@ -667,15 +603,11 @@ def test_track_selection_types( user=self.user ) - # Value Prop TODO (REV-2378): remove waffle flag from tests once the new Track Selection template is rolled out. - # Check whether new track selection template is rendered. - # This should *only* be shown when the waffle flag is on. - with override_waffle_flag(VALUE_PROP_TRACK_SELECTION_FLAG, active=waffle_flag_on): - with patch(GATING_METHOD_NAME, return_value=gated_content_on): - with patch(CDL_METHOD_NAME, return_value=course_duration_limits_on): - url = reverse('course_modes_choose', args=[str(self.course_that_started.id)]) - response = self.client.get(url) - expected_page_assertion_function(self, response, min_price=verified_mode.min_price) + with patch(GATING_METHOD_NAME, return_value=gated_content_on): + with patch(CDL_METHOD_NAME, return_value=course_duration_limits_on): + url = reverse('course_modes_choose', args=[str(self.course_that_started.id)]) + response = self.client.get(url) + expected_page_assertion_function(self, response, min_price=verified_mode.min_price) def test_verified_mode_only(self): # Create only the verified mode and enroll the user @@ -690,18 +622,16 @@ def test_verified_mode_only(self): user=self.user ) - # Value Prop TODO (REV-2378): remove waffle flag from tests once the new Track Selection template is rolled out. - with override_waffle_flag(VALUE_PROP_TRACK_SELECTION_FLAG, active=True): - with patch(GATING_METHOD_NAME, return_value=True): - with patch(CDL_METHOD_NAME, return_value=True): - url = reverse('course_modes_choose', args=[str(self.course_that_started.id)]) - response = self.client.get(url) - # Check that only the verified option is rendered - self.assertNotContains(response, "Choose a path for your course in") - self.assertContains(response, "Earn a certificate") - self.assertNotContains(response, "Access this course") - self.assertContains(response, '
') - self.assertNotContains(response, '
') + with patch(GATING_METHOD_NAME, return_value=True): + with patch(CDL_METHOD_NAME, return_value=True): + url = reverse('course_modes_choose', args=[str(self.course_that_started.id)]) + response = self.client.get(url) + # Check that only the verified option is rendered + self.assertNotContains(response, "Choose a path for your course in") + self.assertContains(response, "Earn a certificate") + self.assertNotContains(response, "Access this course") + self.assertContains(response, '
') + self.assertNotContains(response, '
') @skip_unless_lms diff --git a/common/djangoapps/course_modes/views.py b/common/djangoapps/course_modes/views.py index 759073a13583..09164dc40e7c 100644 --- a/common/djangoapps/course_modes/views.py +++ b/common/djangoapps/course_modes/views.py @@ -29,7 +29,6 @@ from common.djangoapps.course_modes.helpers import get_course_final_price, get_verified_track_links from common.djangoapps.edxmako.shortcuts import render_to_response from common.djangoapps.util.date_utils import strftime_localized_html -from edx_toggles.toggles import WaffleFlag # lint-amnesty, pylint: disable=wrong-import-order from lms.djangoapps.commerce.utils import EcommerceService from lms.djangoapps.experiments.utils import get_experiment_user_metadata_context from lms.djangoapps.verify_student.services import IDVerificationService @@ -47,17 +46,6 @@ LOG = logging.getLogger(__name__) -# .. toggle_name: course_modes.use_new_track_selection -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: This flag enables the use of the new track selection template for testing purposes before full rollout -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2021-8-23 -# .. toggle_target_removal_date: None -# .. toggle_tickets: REV-2133 -# .. toggle_warning: This temporary feature toggle does not have a target removal date. -VALUE_PROP_TRACK_SELECTION_FLAG = WaffleFlag('course_modes.use_new_track_selection', __name__) - class ChooseModeView(View): """View used when the user is asked to pick a mode. @@ -158,18 +146,6 @@ def get(self, request, course_id, error=None): # lint-amnesty, pylint: disable= ) return redirect('{}?{}'.format(reverse('dashboard'), params)) - # When a credit mode is available, students will be given the option - # to upgrade from a verified mode to a credit mode at the end of the course. - # This allows students who have completed photo verification to be eligible - # for university credit. - # Since credit isn't one of the selectable options on the track selection page, - # we need to check *all* available course modes in order to determine whether - # a credit mode is available. If so, then we show slightly different messaging - # for the verified track. - has_credit_upsell = any( - CourseMode.is_credit_mode(mode) for mode - in CourseMode.modes_for_course(course_key, only_selectable=False) - ) course_id = str(course_key) gated_content = ContentTypeGatingConfig.enabled_for_enrollment( user=request.user, @@ -184,7 +160,6 @@ def get(self, request, course_id, error=None): # lint-amnesty, pylint: disable= ), "modes": modes, "is_single_mode": is_single_mode, - "has_credit_upsell": has_credit_upsell, "course_name": course.display_name_with_default, "course_org": course.display_org_with_default, "course_num": course.display_number_with_default, @@ -204,14 +179,6 @@ def get(self, request, course_id, error=None): # lint-amnesty, pylint: disable= ) ) - title_content = '' - if enrollment_mode: - title_content = _("Congratulations! You are now enrolled in {course_name}").format( - course_name=course.display_name_with_default - ) - - context["title_content"] = title_content - if "verified" in modes: verified_mode = modes["verified"] context["suggested_prices"] = [ @@ -266,19 +233,12 @@ def get(self, request, course_id, error=None): # lint-amnesty, pylint: disable= context['audit_access_deadline'] = formatted_audit_access_date fbe_is_on = deadline and gated_content - # Route to correct Track Selection page. - # REV-2378 TODO Value Prop: remove waffle flag after all edge cases for track selection are completed. - if VALUE_PROP_TRACK_SELECTION_FLAG.is_enabled(): - if not enterprise_customer_for_request(request): # TODO: Remove by executing REV-2342 - if error: - return render_to_response("course_modes/error.html", context) - if fbe_is_on: - return render_to_response("course_modes/fbe.html", context) - else: - return render_to_response("course_modes/unfbe.html", context) - - # If enterprise_customer, failover to old choose.html page - return render_to_response("course_modes/choose.html", context) + if error: + return render_to_response("course_modes/error.html", context) + if fbe_is_on: + return render_to_response("course_modes/fbe.html", context) + else: + return render_to_response("course_modes/unfbe.html", context) @method_decorator(transaction.non_atomic_requests) @method_decorator(login_required) diff --git a/lms/templates/course_modes/_upgrade_button.html b/lms/templates/course_modes/_upgrade_button.html deleted file mode 100644 index 02ff82c101ba..000000000000 --- a/lms/templates/course_modes/_upgrade_button.html +++ /dev/null @@ -1,36 +0,0 @@ -<%page args="content_gating_enabled, course_duration_limit_enabled, min_price, price_before_discount" expression_filter="h"/> - -<%! -from django.utils.translation import gettext as _ -from openedx.core.djangolib.markup import HTML, Text -%> - -<%namespace name='static' file='../static_content.html'/> - -
  • - - % if content_gating_enabled or course_duration_limit_enabled: - -
    • - -<%static:require_module_async module_name="js/commerce/track_ecommerce_events" class_name="TrackECommerceEvents"> -var upgradeLink = $("#track_selection_upgrade"); - -TrackECommerceEvents.trackUpsellClick(upgradeLink, 'track_selection', { - pageName: "track_selection", - linkType: "button", - linkCategory: "(none)" -}); - - \ No newline at end of file diff --git a/lms/templates/course_modes/choose.html b/lms/templates/course_modes/choose.html deleted file mode 100644 index 78b6dcc26ebc..000000000000 --- a/lms/templates/course_modes/choose.html +++ /dev/null @@ -1,233 +0,0 @@ -<%page expression_filter="h"/> -<%inherit file="../main.html" /> -<%! -from django.utils.translation import gettext as _ -from django.urls import reverse -from openedx.core.djangolib.js_utils import js_escaped_string -from openedx.core.djangolib.markup import HTML, Text -%> - -<%block name="bodyclass">register verification-process step-select-track -<%block name="pagetitle"> - ${_("Enroll In {course_name} | Choose Your Track").format(course_name=course_name)} - - -<%block name="js_extra"> - - - -<%block name="content"> - % if error: -
    -
    - -
    -

    ${_("Sorry, there was an error when trying to enroll you")}

    -
    -

    ${error}

    -
    -
    -
    -
    - %endif - -
    -
    -
    -
    -
    -

    - ${title_content} -

    -
    - -
    - <% - b_tag_kwargs = {'b_start': HTML(''), 'b_end': HTML('')} - %> - % if "verified" in modes: -
    -
    - - % if has_credit_upsell: - % if content_gating_enabled or course_duration_limit_enabled: -

    ${_("Pursue Academic Credit with the Verified Track")}

    - % else: -

    ${_("Pursue Academic Credit with a Verified Certificate")}

    - % endif - -
    -

    ${_("Become eligible for academic credit and highlight your new skills and knowledge with a verified certificate. Use this valuable credential to qualify for academic credit, advance your career, or strengthen your school applications.")}

    -

    -

    -
    - % if content_gating_enabled or course_duration_limit_enabled: -

    ${_("Benefits of the Verified Track")}

    -
      -
    • ${Text(_("{b_start}Eligible for credit:{b_end} Receive academic credit after successfully completing the course")).format(**b_tag_kwargs)}
      • - % if course_duration_limit_enabled: -
    • ${Text(_("{b_start}Unlimited Course Access: {b_end}Learn at your own pace, and access materials anytime to brush up on what you've learned.")).format(**b_tag_kwargs)}
      • - % endif - % if content_gating_enabled: -
    • ${Text(_("{b_start}Graded Assignments: {b_end}Build your skills through graded assignments and projects.")).format(**b_tag_kwargs)}
      • - % endif -
    • ${Text(_("{b_start}Easily Sharable: {b_end}Add the certificate to your CV or resumé, or post it directly on LinkedIn.")).format(**b_tag_kwargs)}
      • -
    - % else: -

    ${_("Benefits of a Verified Certificate")}

    -
      -
    • ${Text(_("{b_start}Eligible for credit:{b_end} Receive academic credit after successfully completing the course")).format(**b_tag_kwargs)}
      • -
    • ${Text(_("{b_start}Official:{b_end} Receive an instructor-signed certificate with the institution's logo")).format(**b_tag_kwargs)}
      • -
    • ${Text(_("{b_start}Easily shareable:{b_end} Add the certificate to your CV or resumé, or post it directly on LinkedIn")).format(**b_tag_kwargs)}
      • -
    - % endif -
    -
    -
      - <%include file='_upgrade_button.html' args='content_gating_enabled=content_gating_enabled, course_duration_limit_enabled=course_duration_limit_enabled, currency=currency, currency_symbol=currency_symbol, min_price=min_price, price_before_discount=price_before_discount' /> -
    -
    -
    -

    -
    - % else: - % if content_gating_enabled or course_duration_limit_enabled: -

    ${_("Pursue the Verified Track")}

    - % else: -

    ${_("Pursue a Verified Certificate")}

    - % endif - - -
    -

    ${_("Highlight your new knowledge and skills with a verified certificate. Use this valuable credential to improve your job prospects and advance your career, or highlight your certificate in school applications.")}

    -

    -

    -
    - % if content_gating_enabled or course_duration_limit_enabled: -

    ${_("Benefits of the Verified Track")}

    -
      - % if course_duration_limit_enabled: -
    • ${Text(_("{b_start}Unlimited Course Access: {b_end}Learn at your own pace, and access materials anytime to brush up on what you've learned.")).format(**b_tag_kwargs)}
      • - % endif - % if content_gating_enabled: -
    • ${Text(_("{b_start}Graded Assignments: {b_end}Build your skills through graded assignments and projects.")).format(**b_tag_kwargs)}
      • - % endif -
    • ${Text(_("{b_start}Easily Sharable: {b_end}Add the certificate to your CV or resumé, or post it directly on LinkedIn.")).format(**b_tag_kwargs)}
      • -
    - % else: -

    ${_("Benefits of a Verified Certificate")}

    -
      -
    • ${Text(_("{b_start}Official: {b_end}Receive an instructor-signed certificate with the institution's logo")).format(**b_tag_kwargs)}
      • -
    • ${Text(_("{b_start}Easily shareable: {b_end}Add the certificate to your CV or resumé, or post it directly on LinkedIn")).format(**b_tag_kwargs)}
      • -
    • ${Text(_("{b_start}Motivating: {b_end}Give yourself an additional incentive to complete the course")).format(**b_tag_kwargs)}
      • -
    - % endif -
    -
    -
      - <%include file='_upgrade_button.html' args='content_gating_enabled=content_gating_enabled, course_duration_limit_enabled=course_duration_limit_enabled, currency=currency, currency_symbol=currency_symbol, min_price=min_price, price_before_discount=price_before_discount' /> -
    -
    -
    -

    -
    - % endif -
    -
    - % endif - - % if "honor" in modes: - - ${_("or")} - - -
    -
    - -

    ${_("Audit This Course")}

    -
    -

    ${_("Audit this course for free and have complete access to all the course material, activities, tests, and forums.")}

    -
    -
    - -
      -
    • - -
      • -
    -
    - % elif "audit" in modes: - - ${_("or")} - - -
    -
    - -

    ${_("Audit This Course (No Certificate)")}

    -
    - ## Translators: b_start notes the beginning of a section of text bolded for emphasis, and b_end marks the end of the bolded text. - % if content_gating_enabled and course_duration_limit_enabled: -

    ${Text(_("Audit this course for free and have access to course materials and discussions forums. {b_start}This track does not include graded assignments, or unlimited course access.{b_end}")).format(**b_tag_kwargs)}

    - % elif content_gating_enabled and not course_duration_limit_enabled: -

    ${Text(_("Audit this course for free and have access to course materials and discussions forums. {b_start}This track does not include graded assignments.{b_end}")).format(**b_tag_kwargs)}

    - % elif not content_gating_enabled and course_duration_limit_enabled: -

    ${Text(_("Audit this course for free and have access to course materials and discussions forums. {b_start}This track does not include unlimited course access.{b_end}")).format(**b_tag_kwargs)}

    - % else: -

    ${Text(_("Audit this course for free and have complete access to all the course material, activities, tests, and forums. {b_start}Please note that this track does not offer a certificate for learners who earn a passing grade.{b_end}")).format(**b_tag_kwargs)}

    - % endif -
    -
    - -
      -
    • - -
      • -
    -
    - % endif - - -
    -
    -
    -
    -
    - From cb55799bab1b0a8f549dc1a40f702e96a84a1222 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Fri, 24 Jan 2025 10:45:35 -0500 Subject: [PATCH 14/30] chore: Upgrade Python requirements (#36166) --- requirements/edx-sandbox/base.txt | 2 +- requirements/edx/base.txt | 22 ++++++------- requirements/edx/development.txt | 32 +++++++++---------- requirements/edx/doc.txt | 22 ++++++------- requirements/edx/semgrep.txt | 4 +-- requirements/edx/testing.txt | 32 +++++++++---------- scripts/user_retirement/requirements/base.txt | 10 +++--- .../user_retirement/requirements/testing.txt | 10 +++--- 8 files changed, 67 insertions(+), 67 deletions(-) diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index da781eb84587..f474fe6bef29 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -20,7 +20,7 @@ cryptography==44.0.0 # via -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib -fonttools==4.55.4 +fonttools==4.55.5 # via matplotlib joblib==1.4.2 # via nltk diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index bd412f92132a..b6713ec58bdc 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -72,13 +72,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.36.3 +boto3==1.36.5 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 -botocore==1.36.3 +botocore==1.36.5 # via # -r requirements/edx/kernel.in # boto3 @@ -87,7 +87,7 @@ bridgekeeper==0.9 # via -r requirements/edx/kernel.in cachecontrol==0.14.2 # via firebase-admin -cachetools==5.5.0 +cachetools==5.5.1 # via google-auth camel-converter[pydantic]==4.0.1 # via meilisearch @@ -513,7 +513,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/kernel.in # edx-enterprise # edx-proctoring -edx-search==4.1.1 +edx-search==4.1.2 # via # -r requirements/edx/kernel.in # openedx-forum @@ -595,7 +595,7 @@ google-api-core[grpc]==2.24.0 # google-cloud-storage google-api-python-client==2.159.0 # via firebase-admin -google-auth==2.37.0 +google-auth==2.38.0 # via # google-api-core # google-api-python-client @@ -623,11 +623,11 @@ googleapis-common-protos==1.66.0 # via # google-api-core # grpcio-status -grpcio==1.69.0 +grpcio==1.70.0 # via # google-api-core # grpcio-status -grpcio-status==1.69.0 +grpcio-status==1.70.0 # via google-api-core gunicorn==23.0.0 # via -r requirements/edx/kernel.in @@ -702,7 +702,7 @@ lazy==1.6 # xblock loremipsum==1.0.5 # via ora2 -lti-consumer-xblock==9.13.1 +lti-consumer-xblock==9.13.2 # via -r requirements/edx/kernel.in lxml[html-clean,html_clean]==5.3.0 # via @@ -919,7 +919,7 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.5 +pydantic==2.10.6 # via camel-converter pydantic-core==2.27.2 # via pydantic @@ -1089,7 +1089,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.11.1 +s3transfer==0.11.2 # via boto3 sailthru-client==2.2.3 # via edx-ace @@ -1135,7 +1135,7 @@ slumber==0.7.1 # -r requirements/edx/kernel.in # edx-bulk-grades # edx-enterprise -snowflake-connector-python==3.12.4 +snowflake-connector-python==3.13.0 # via edx-enterprise social-auth-app-django==5.4.1 # via diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 6a8c617d41be..a92f149228e3 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -145,14 +145,14 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.36.3 +boto3==1.36.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ses # fs-s3fs # ora2 -botocore==1.36.3 +botocore==1.36.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -171,7 +171,7 @@ cachecontrol==0.14.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # firebase-admin -cachetools==5.5.0 +cachetools==5.5.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -766,7 +766,7 @@ edx-i18n-tools==1.5.0 # -r requirements/edx/testing.txt # ora2 # xblocks-contrib -edx-lint==5.4.1 +edx-lint==5.5.0 # via -r requirements/edx/testing.txt edx-milestones==0.6.0 # via @@ -812,7 +812,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/testing.txt # edx-enterprise # edx-proctoring -edx-search==4.1.1 +edx-search==4.1.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -884,11 +884,11 @@ execnet==2.1.1 # pytest-xdist factory-boy==3.3.1 # via -r requirements/edx/testing.txt -faker==33.3.1 +faker==35.0.0 # via # -r requirements/edx/testing.txt # factory-boy -fastapi==0.115.6 +fastapi==0.115.7 # via # -r requirements/edx/testing.txt # pact-python @@ -962,7 +962,7 @@ google-api-python-client==2.159.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # firebase-admin -google-auth==2.37.0 +google-auth==2.38.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1014,13 +1014,13 @@ grimp==3.5 # via # -r requirements/edx/testing.txt # import-linter -grpcio==1.69.0 +grpcio==1.70.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # google-api-core # grpcio-status -grpcio-status==1.69.0 +grpcio-status==1.70.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1184,7 +1184,7 @@ loremipsum==1.0.5 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 -lti-consumer-xblock==9.13.1 +lti-consumer-xblock==9.13.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1419,7 +1419,7 @@ packaging==24.2 # snowflake-connector-python # sphinx # tox -pact-python==2.3.0 +pact-python==2.3.1 # via -r requirements/edx/testing.txt pansi==2024.11.0 # via @@ -1564,7 +1564,7 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.5 +pydantic==2.10.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1882,7 +1882,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.11.1 +s3transfer==0.11.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1963,7 +1963,7 @@ snowballstemmer==2.2.0 # via # -r requirements/edx/doc.txt # sphinx -snowflake-connector-python==3.12.4 +snowflake-connector-python==3.13.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2061,7 +2061,7 @@ staff-graded-xblock==3.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -starlette==0.41.3 +starlette==0.45.3 # via # -r requirements/edx/testing.txt # fastapi diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 66bd24f8fcfc..02e631fbaf1a 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -107,13 +107,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.36.3 +boto3==1.36.5 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.36.3 +botocore==1.36.5 # via # -r requirements/edx/base.txt # boto3 @@ -124,7 +124,7 @@ cachecontrol==0.14.2 # via # -r requirements/edx/base.txt # firebase-admin -cachetools==5.5.0 +cachetools==5.5.1 # via # -r requirements/edx/base.txt # google-auth @@ -602,7 +602,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/base.txt # edx-enterprise # edx-proctoring -edx-search==4.1.1 +edx-search==4.1.2 # via # -r requirements/edx/base.txt # openedx-forum @@ -705,7 +705,7 @@ google-api-python-client==2.159.0 # via # -r requirements/edx/base.txt # firebase-admin -google-auth==2.37.0 +google-auth==2.38.0 # via # -r requirements/edx/base.txt # google-api-core @@ -745,12 +745,12 @@ googleapis-common-protos==1.66.0 # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio==1.69.0 +grpcio==1.70.0 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.69.0 +grpcio-status==1.70.0 # via # -r requirements/edx/base.txt # google-api-core @@ -857,7 +857,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.13.1 +lti-consumer-xblock==9.13.2 # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via @@ -1123,7 +1123,7 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.5 +pydantic==2.10.6 # via # -r requirements/edx/base.txt # camel-converter @@ -1328,7 +1328,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.11.1 +s3transfer==0.11.2 # via # -r requirements/edx/base.txt # boto3 @@ -1386,7 +1386,7 @@ smmap==5.0.2 # via gitdb snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python==3.12.4 +snowflake-connector-python==3.13.0 # via # -r requirements/edx/base.txt # edx-enterprise diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index 65a0b794b9e6..e5022308aaed 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -34,7 +34,7 @@ colorama==0.4.6 # via semgrep defusedxml==0.7.1 # via semgrep -deprecated==1.2.15 +deprecated==1.2.16 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-http @@ -116,7 +116,7 @@ ruamel-yaml==0.18.10 # via semgrep ruamel-yaml-clib==0.2.12 # via ruamel-yaml -semgrep==1.103.0 +semgrep==1.104.0 # via -r requirements/edx/semgrep.in tomli==2.0.2 # via semgrep diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 27fe32c152b2..ad8432ca9073 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -104,13 +104,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.36.3 +boto3==1.36.5 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.36.3 +botocore==1.36.5 # via # -r requirements/edx/base.txt # boto3 @@ -121,7 +121,7 @@ cachecontrol==0.14.2 # via # -r requirements/edx/base.txt # firebase-admin -cachetools==5.5.0 +cachetools==5.5.1 # via # -r requirements/edx/base.txt # google-auth @@ -589,7 +589,7 @@ edx-i18n-tools==1.5.0 # -r requirements/edx/base.txt # ora2 # xblocks-contrib -edx-lint==5.4.1 +edx-lint==5.5.0 # via -r requirements/edx/testing.in edx-milestones==0.6.0 # via -r requirements/edx/base.txt @@ -625,7 +625,7 @@ edx-rest-api-client==6.0.0 # -r requirements/edx/base.txt # edx-enterprise # edx-proctoring -edx-search==4.1.1 +edx-search==4.1.2 # via # -r requirements/edx/base.txt # openedx-forum @@ -681,9 +681,9 @@ execnet==2.1.1 # via pytest-xdist factory-boy==3.3.1 # via -r requirements/edx/testing.in -faker==33.3.1 +faker==35.0.0 # via factory-boy -fastapi==0.115.6 +fastapi==0.115.7 # via pact-python fastavro==1.10.0 # via @@ -736,7 +736,7 @@ google-api-python-client==2.159.0 # via # -r requirements/edx/base.txt # firebase-admin -google-auth==2.37.0 +google-auth==2.38.0 # via # -r requirements/edx/base.txt # google-api-core @@ -778,12 +778,12 @@ googleapis-common-protos==1.66.0 # grpcio-status grimp==3.5 # via import-linter -grpcio==1.69.0 +grpcio==1.70.0 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.69.0 +grpcio-status==1.70.0 # via # -r requirements/edx/base.txt # google-api-core @@ -900,7 +900,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.13.1 +lti-consumer-xblock==9.13.2 # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via @@ -1073,7 +1073,7 @@ packaging==24.2 # pytest # snowflake-connector-python # tox -pact-python==2.3.0 +pact-python==2.3.1 # via -r requirements/edx/testing.in pansi==2024.11.0 # via @@ -1190,7 +1190,7 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.10.5 +pydantic==2.10.6 # via # -r requirements/edx/base.txt # camel-converter @@ -1441,7 +1441,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.11.1 +s3transfer==0.11.2 # via # -r requirements/edx/base.txt # boto3 @@ -1500,7 +1500,7 @@ slumber==0.7.1 # edx-enterprise sniffio==1.3.1 # via anyio -snowflake-connector-python==3.12.4 +snowflake-connector-python==3.13.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1532,7 +1532,7 @@ sqlparse==0.5.3 # django staff-graded-xblock==3.0.0 # via -r requirements/edx/base.txt -starlette==0.41.3 +starlette==0.45.3 # via fastapi stevedore==5.4.0 # via diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 5629aee43eea..8f1b96239bc4 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -10,13 +10,13 @@ attrs==24.3.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -boto3==1.36.3 +boto3==1.36.5 # via -r scripts/user_retirement/requirements/base.in -botocore==1.36.3 +botocore==1.36.5 # via # boto3 # s3transfer -cachetools==5.5.0 +cachetools==5.5.1 # via google-auth certifi==2024.12.14 # via requests @@ -54,7 +54,7 @@ google-api-core==2.24.0 # via google-api-python-client google-api-python-client==2.159.0 # via -r scripts/user_retirement/requirements/base.in -google-auth==2.37.0 +google-auth==2.38.0 # via # google-api-core # google-api-python-client @@ -136,7 +136,7 @@ requests-toolbelt==1.0.0 # via zeep rsa==4.9 # via google-auth -s3transfer==0.11.1 +s3transfer==0.11.2 # via boto3 simple-salesforce==1.12.6 # via -r scripts/user_retirement/requirements/base.in diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index ad69c332b702..19d54d5adb2f 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -14,17 +14,17 @@ attrs==24.3.0 # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -boto3==1.36.3 +boto3==1.36.5 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.36.3 +botocore==1.36.5 # via # -r scripts/user_retirement/requirements/base.txt # boto3 # moto # s3transfer -cachetools==5.5.0 +cachetools==5.5.1 # via # -r scripts/user_retirement/requirements/base.txt # google-auth @@ -78,7 +78,7 @@ google-api-core==2.24.0 # google-api-python-client google-api-python-client==2.159.0 # via -r scripts/user_retirement/requirements/base.txt -google-auth==2.37.0 +google-auth==2.38.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -235,7 +235,7 @@ rsa==4.9 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -s3transfer==0.11.1 +s3transfer==0.11.2 # via # -r scripts/user_retirement/requirements/base.txt # boto3 From 19def9266a7b9aa50506ad8dd7b20d95b5e6524d Mon Sep 17 00:00:00 2001 From: pwnage101 <85151+pwnage101@users.noreply.github.com> Date: Fri, 24 Jan 2025 19:03:55 +0000 Subject: [PATCH 15/30] feat: Upgrade Python dependency edx-enterprise Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 283ab625f42b..f771f12ebe62 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -80,7 +80,7 @@ django-storages<1.14.4 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==5.6.4 +edx-enterprise==5.6.5 # Date: 2024-05-09 # This has to be constrained as well because newer versions of edx-i18n-tools need the diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index b6713ec58bdc..c075f9f2d40a 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -467,7 +467,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.4 +edx-enterprise==5.6.5 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index a92f149228e3..412b8c6147bf 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -746,7 +746,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.4 +edx-enterprise==5.6.5 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 02e631fbaf1a..06436873c4cd 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -554,7 +554,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.4 +edx-enterprise==5.6.5 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index ad8432ca9073..4fd65de9cfa6 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -575,7 +575,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.4 +edx-enterprise==5.6.5 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 46c7f6db5225ae80b681a93fc2dc76904bcaac4e Mon Sep 17 00:00:00 2001 From: Ahtisham Shahid Date: Mon, 27 Jan 2025 12:07:29 +0500 Subject: [PATCH 16/30] fix: bypass edx ace for sending goal reminder email using ses (#36148) --- .../commands/goal_reminder_email.py | 57 ++++++++++++++++++- .../tests/test_goal_reminder_email.py | 8 +-- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py b/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py index af48472886bb..a9ab150221ea 100644 --- a/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py +++ b/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py @@ -3,6 +3,10 @@ """ import time from datetime import date, datetime, timedelta + +import boto3 +from edx_ace.channel.django_email import DjangoEmailChannel +from edx_ace.channel.mixins import EmailChannelMixin from eventtracking import tracker import logging import uuid @@ -10,10 +14,10 @@ from django.conf import settings from django.contrib.sites.models import Site from django.core.management.base import BaseCommand -from edx_ace import ace +from edx_ace import ace, presentation from edx_ace.message import Message from edx_ace.recipient import Recipient - +from edx_ace.utils.signals import send_ace_message_sent_signal from common.djangoapps.student.models import CourseEnrollment from lms.djangoapps.certificates.api import get_certificate_for_user_id from lms.djangoapps.certificates.data import CertificateStatuses @@ -121,7 +125,11 @@ def send_ace_message(goal, session_id): with emulate_http_request(site, user): try: start_time = time.perf_counter() - ace.send(msg) + if is_ses_enabled: + # experimental implementation to log errors with ses + send_email_using_ses(user, msg) + else: + ace.send(msg) end_time = time.perf_counter() log.info(f"Goal Reminder for {user.id} for course {goal.course_key} sent in {end_time - start_time} " f"using {'SES' if is_ses_enabled else 'others'}") @@ -297,3 +305,46 @@ def handle_goal(goal, today, sunday_date, monday_date, session_id): return True return False + + +def send_email_using_ses(user, msg): + """ + Send email using AWS SES + """ + msg = presentation.render(DjangoEmailChannel, msg) + # send rendered email using SES + sender = EmailChannelMixin.get_from_address(msg) + recipient = user.email + subject = EmailChannelMixin.get_subject(msg) + body_text = msg.body + body_html = msg.body_html + + try: + # Send email + response = boto3.client('ses', settings.AWS_SES_REGION_NAME).send_email( + Source=sender, + Destination={ + 'ToAddresses': [recipient], + }, + Message={ + 'Subject': { + 'Data': subject, + 'Charset': 'UTF-8' + }, + 'Body': { + 'Text': { + 'Data': body_text, + 'Charset': 'UTF-8' + }, + 'Html': { + 'Data': body_html, + 'Charset': 'UTF-8' + } + } + } + ) + + log.info(f"Goal Reminder Email: email sent using SES with message ID {response['MessageId']}") + send_ace_message_sent_signal(DjangoEmailChannel, msg) + except Exception as e: + log.error(f"Goal Reminder Email: Error sending email using SES: {e}") diff --git a/lms/djangoapps/course_goals/management/commands/tests/test_goal_reminder_email.py b/lms/djangoapps/course_goals/management/commands/tests/test_goal_reminder_email.py index 5b98b202d41f..31804037919c 100644 --- a/lms/djangoapps/course_goals/management/commands/tests/test_goal_reminder_email.py +++ b/lms/djangoapps/course_goals/management/commands/tests/test_goal_reminder_email.py @@ -182,8 +182,8 @@ def test_old_course(self, end): self.make_valid_goal(overview__end=end) self.call_command(expect_sent=False) - @mock.patch('lms.djangoapps.course_goals.management.commands.goal_reminder_email.ace.send') - def test_params_with_ses(self, mock_ace): + @mock.patch('lms.djangoapps.course_goals.management.commands.goal_reminder_email.send_email_using_ses') + def test_params_with_ses(self, mock_send_email_using_ses): """Test that the parameters of the msg passed to ace.send() are set correctly when SES is enabled""" with override_waffle_flag(ENABLE_SES_FOR_GOALREMINDER, active=None): goal = self.make_valid_goal() @@ -193,8 +193,8 @@ def test_params_with_ses(self, mock_ace): with freeze_time('2021-03-02 10:00:00'): call_command('goal_reminder_email') - assert mock_ace.call_count == 1 - msg = mock_ace.call_args[0][0] + assert mock_send_email_using_ses.call_count == 1 + msg = mock_send_email_using_ses.call_args[0][1] assert msg.options['override_default_channel'] == 'django_email' assert msg.options['from_address'] == settings.LMS_COMM_DEFAULT_FROM_EMAIL From 3847cec503df0ca4c3c452d6daa6cd66474f92f1 Mon Sep 17 00:00:00 2001 From: Jillian Date: Tue, 28 Jan 2025 02:07:53 +1030 Subject: [PATCH 17/30] feat: add and switch to downstream-only fields [FC-0076] (#36158) Adds the concept of "downstream-only" fields to the XBlock upstream sync logic. Downstream-only fields are customizable fields set only on the downstream XBlock -- we don't keep track of the upstream field value anywhere on the downstream XBlock. Changes made to these fields in the upstream block are ignored, and if the link to the upstream block is severed, the downstream changes are preserved (not reset back to defaults, like the upstream-tracked customizable fields are). The fields chosen as "downstream-only" are those related to scoring and grading. The `max_attempts` field was previously a customizable field that tracked the upstream value. However, because it is scoring-related, it has been converted to a "downstream-only" field. This change impacts course authors' use of library content in their courses. --- .../views/tests/test_clipboard_paste.py | 1 - cms/lib/xblock/test/test_upstream_sync.py | 118 +++++++++++++++++- cms/lib/xblock/upstream_sync.py | 36 ++++-- 3 files changed, 143 insertions(+), 12 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py b/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py index 9244ffa989b6..3b39f2918a36 100644 --- a/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py +++ b/cms/djangoapps/contentstore/views/tests/test_clipboard_paste.py @@ -555,7 +555,6 @@ def _copy_paste_and_assert_link(key_to_copy): assert new_block.upstream == str(self.lib_block_key) assert new_block.upstream_version == 3 assert new_block.upstream_display_name == "MCQ-draft" - assert new_block.upstream_max_attempts == 5 return new_block_key # first verify link for copied block from library diff --git a/cms/lib/xblock/test/test_upstream_sync.py b/cms/lib/xblock/test/test_upstream_sync.py index cc3d661ca6e9..2f8f77ab6560 100644 --- a/cms/lib/xblock/test/test_upstream_sync.py +++ b/cms/lib/xblock/test/test_upstream_sync.py @@ -1,7 +1,9 @@ """ Test CMS's upstream->downstream syncing system """ +import datetime import ddt +from pytz import utc from organizations.api import ensure_organization from organizations.models import Organization @@ -42,13 +44,33 @@ def setUp(self): title="Test Upstream Library", ) self.upstream_key = libs.create_library_block(self.library.key, "html", "test-upstream").usage_key - libs.create_library_block(self.library.key, "video", "video-upstream") upstream = xblock.load_block(self.upstream_key, self.user) upstream.display_name = "Upstream Title V2" upstream.data = "Upstream content V2" upstream.save() + self.upstream_problem_key = libs.create_library_block(self.library.key, "problem", "problem-upstream").usage_key + libs.set_library_block_olx(self.upstream_problem_key, ( + '\n' + )) + libs.publish_changes(self.library.key, self.user.id) self.taxonomy_all_org = tagging_api.create_taxonomy( @@ -179,6 +201,100 @@ def test_sync_updates_happy_path(self): for object_tag in object_tags: assert object_tag.value in new_upstream_tags + # pylint: disable=too-many-statements + def test_sync_updates_to_downstream_only_fields(self): + """ + If we sync to modified content, will it preserve downstream-only fields, and overwrite the rest? + """ + downstream = BlockFactory.create(category='problem', parent=self.unit, upstream=str(self.upstream_problem_key)) + + # Initial sync + sync_from_upstream(downstream, self.user) + + # These fields are copied from upstream + assert downstream.upstream_display_name == "Upstream Problem Title V2" + assert downstream.display_name == "Upstream Problem Title V2" + assert downstream.rerandomize == '"always"' + assert downstream.matlab_api_key == 'abc' + assert not downstream.use_latex_compiler + + # These fields are "downstream only", so field defaults are preserved, and values are NOT copied from upstream + assert downstream.attempts_before_showanswer_button == 0 + assert downstream.due is None + assert not downstream.force_save_button + assert downstream.graceperiod is None + assert downstream.grading_method == 'last_score' + assert downstream.max_attempts is None + assert downstream.show_correctness == 'always' + assert not downstream.show_reset_button + assert downstream.showanswer == 'finished' + assert downstream.submission_wait_seconds == 0 + assert downstream.weight is None + + # Upstream updates + libs.set_library_block_olx(self.upstream_problem_key, ( + '\n' + )) + libs.publish_changes(self.library.key, self.user.id) + + # Modifing downstream-only fields are "safe" customizations + downstream.display_name = "Downstream Title Override" + downstream.attempts_before_showanswer_button = 2 + downstream.due = datetime.datetime(2025, 2, 2, tzinfo=utc) + downstream.force_save_button = True + downstream.graceperiod = '2d' + downstream.grading_method = 'last_score' + downstream.max_attempts = 100 + downstream.show_correctness = 'always' + downstream.show_reset_button = True + downstream.showanswer = 'on_expired' + downstream.submission_wait_seconds = 100 + downstream.weight = 3 + + # Modifying synchronized fields are "unsafe" customizations + downstream.rerandomize = '"onreset"' + downstream.matlab_api_key = 'hij' + downstream.save() + + # Follow-up sync. + sync_from_upstream(downstream, self.user) + + # "unsafe" customizations are overridden by upstream + assert downstream.upstream_display_name == "Upstream Problem Title V3" + assert downstream.rerandomize == '"per_student"' + assert downstream.matlab_api_key == 'def' + assert downstream.use_latex_compiler + + # but "safe" customizations survive + assert downstream.display_name == "Downstream Title Override" + assert downstream.attempts_before_showanswer_button == 2 + assert downstream.due == datetime.datetime(2025, 2, 2, tzinfo=utc) + assert downstream.force_save_button + assert downstream.graceperiod == '2d' + assert downstream.grading_method == 'last_score' + assert downstream.max_attempts == 100 + assert downstream.show_correctness == 'always' + assert downstream.show_reset_button + assert downstream.showanswer == 'on_expired' + assert downstream.submission_wait_seconds == 100 + assert downstream.weight == 3 + def test_sync_updates_to_modified_content(self): """ If we sync to modified content, will it preserve customizable fields, but overwrite the rest? diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 0d95931ce29d..4723e566ffc3 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -252,10 +252,6 @@ def _update_customizable_fields(*, upstream: XBlock, downstream: XBlock, only_fe * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: * Set `course_problem.display_name = lib_problem.display_name` ("sync"). - - * Set `course_problem.upstream_max_attempts = lib_problem.max_attempts` ("fetch"). - * If `not only_fetch`, and `course_problem.max_attempts` wasn't customized, then: - * Set `course_problem.max_attempts = lib_problem.max_attempts` ("sync"). """ syncable_field_names = _get_synchronizable_fields(upstream, downstream) @@ -264,6 +260,10 @@ def _update_customizable_fields(*, upstream: XBlock, downstream: XBlock, only_fe if field_name not in syncable_field_names: continue + # Downstream-only fields don't have an upstream fetch field + if fetch_field_name is None: + continue + # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). old_upstream_value = getattr(downstream, fetch_field_name) new_upstream_value = getattr(upstream, field_name) @@ -361,6 +361,9 @@ def sever_upstream_link(downstream: XBlock) -> None: downstream.upstream = None downstream.upstream_version = None for _, fetched_upstream_field in downstream.get_customizable_fields().items(): + # Downstream-only fields don't have an upstream fetch field + if fetched_upstream_field is None: + continue setattr(downstream, fetched_upstream_field, None) # Null out upstream_display_name, et al. @@ -414,21 +417,30 @@ class UpstreamSyncMixin(XBlockMixin): help=("The value of display_name on the linked upstream block."), default=None, scope=Scope.settings, hidden=True, enforce_type=True, ) - upstream_max_attempts = Integer( - help=("The value of max_attempts on the linked upstream block."), - default=None, scope=Scope.settings, hidden=True, enforce_type=True, - ) @classmethod - def get_customizable_fields(cls) -> dict[str, str]: + def get_customizable_fields(cls) -> dict[str, str | None]: """ Mapping from each customizable field to the field which can be used to restore its upstream value. + If the customizable field is mapped to None, then it is considered "downstream only", and cannot be restored + from the upstream value. + XBlocks outside of edx-platform can override this in order to set up their own customizable fields. """ return { "display_name": "upstream_display_name", - "max_attempts": "upstream_max_attempts", + "attempts_before_showanswer_button": None, + "due": None, + "force_save_button": None, + "graceperiod": None, + "grading_method": None, + "max_attempts": None, + "show_correctness": None, + "show_reset_button": None, + "showanswer": None, + "submission_wait_seconds": None, + "weight": None, } # PRESERVING DOWNSTREAM CUSTOMIZATIONS and RESTORING UPSTREAM VALUES @@ -485,6 +497,10 @@ def get_customizable_fields(cls) -> dict[str, str]: # if field_name in self.downstream_customized: # continue # + # # If there is no restore_field name, it's a downstream-only field + # if restore_field_name is None: + # continue + # # # If this field's value doesn't match the synced upstream value, then mark the field # # as customized so that we don't clobber it later when syncing. # # NOTE: Need to consider the performance impact of all these field lookups. From 2d7a3d3fbdd63257d8cf58382a174409ca448999 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Mon, 27 Jan 2025 21:25:42 +0500 Subject: [PATCH 18/30] fix: remove pylint constraint (#36169) --- .../content_staging/tests/test_clipboard.py | 1 + pylint_django_settings.py | 8 +-- pylintrc | 56 +++++++++++-------- pylintrc_tweaks | 13 +++++ requirements/constraints.txt | 5 -- requirements/edx-sandbox/base.txt | 2 +- requirements/edx/base.txt | 8 +-- requirements/edx/development.txt | 30 ++++------ requirements/edx/doc.txt | 20 +++---- requirements/edx/semgrep.txt | 8 +-- requirements/edx/testing.in | 4 +- requirements/edx/testing.txt | 30 ++++------ scripts/user_retirement/requirements/base.txt | 6 +- .../user_retirement/requirements/testing.txt | 6 +- 14 files changed, 98 insertions(+), 99 deletions(-) diff --git a/openedx/core/djangoapps/content_staging/tests/test_clipboard.py b/openedx/core/djangoapps/content_staging/tests/test_clipboard.py index 551f94e90e1a..ab65d444ed6f 100644 --- a/openedx/core/djangoapps/content_staging/tests/test_clipboard.py +++ b/openedx/core/djangoapps/content_staging/tests/test_clipboard.py @@ -1,3 +1,4 @@ +# pylint: skip-file """ Tests for the clipboard functionality """ diff --git a/pylint_django_settings.py b/pylint_django_settings.py index 46abfd81f883..6051d9ab4b56 100644 --- a/pylint_django_settings.py +++ b/pylint_django_settings.py @@ -1,5 +1,5 @@ -from pylint_django.checkers import ForeignKeyStringsChecker -from pylint_plugin_utils import get_checker +import os +import sys class ArgumentCompatibilityError(Exception): @@ -47,6 +47,4 @@ def load_configuration(linter): """ Configures the Django settings module based on the command-line arguments passed to pylint. """ - name_checker = get_checker(linter, ForeignKeyStringsChecker) - arguments = linter.cmdline_parser.parse_args()[1] - name_checker.config.django_settings_module = _get_django_settings_module(arguments) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", _get_django_settings_module(sys.argv[1:])) diff --git a/pylintrc b/pylintrc index 55a9bbab3b9c..81b984a23cf8 100644 --- a/pylintrc +++ b/pylintrc @@ -64,7 +64,7 @@ # SERIOUSLY. # # ------------------------------ -# Generated by edx-lint version: 5.3.7 +# Generated by edx-lint version: 5.4.1 # ------------------------------ [MASTER] ignore = ,.git,.tox,migrations,node_modules,.pycharm_helpers @@ -72,10 +72,10 @@ persistent = yes load-plugins = edx_lint.pylint,pylint_django_settings,pylint_django,pylint_celery,pylint_pytest [MESSAGES CONTROL] -enable = +enable = blacklisted-name, line-too-long, - + abstract-class-instantiated, abstract-method, access-member-before-definition, @@ -184,26 +184,26 @@ enable = used-before-assignment, using-constant-test, yield-outside-function, - + astroid-error, fatal, method-check-failed, parse-error, raw-checker-failed, - + empty-docstring, invalid-characters-in-docstring, missing-docstring, wrong-spelling-in-comment, wrong-spelling-in-docstring, - + unused-argument, unused-import, unused-variable, - + eval-used, exec-used, - + bad-classmethod-argument, bad-mcs-classmethod-argument, bad-mcs-method-argument, @@ -234,30 +234,30 @@ enable = unneeded-not, useless-else-on-loop, wrong-assert-type, - + deprecated-method, deprecated-module, - + too-many-boolean-expressions, too-many-nested-blocks, too-many-statements, - + wildcard-import, wrong-import-order, wrong-import-position, - + missing-final-newline, mixed-line-endings, trailing-newlines, trailing-whitespace, unexpected-line-ending-format, - + bad-inline-option, bad-option-value, deprecated-pragma, unrecognized-inline-option, useless-suppression, -disable = +disable = bad-indentation, broad-exception-raised, consider-using-f-string, @@ -282,10 +282,10 @@ disable = unspecified-encoding, unused-wildcard-import, use-maxsplit-arg, - + feature-toggle-needs-doc, illegal-waffle-usage, - + logging-fstring-interpolation, import-outside-toplevel, inconsistent-return-statements, @@ -314,6 +314,18 @@ disable = c-extension-no-member, no-name-in-module, unnecessary-lambda-assignment, + too-many-positional-arguments, + possibly-used-before-assignment, + use-dict-literal, + useless-return, + superfluous-parens, + logging-not-lazy, + broad-exception-caught, + no-else-raise, + pointless-exception-statement, + consider-using-join, + use-yield-from, + used-before-assignment, [REPORTS] output-format = text @@ -356,7 +368,7 @@ ignore-imports = no ignore-mixin-members = yes ignored-classes = SQLObject unsafe-load-any-extension = yes -generated-members = +generated-members = REQUEST, acl_users, aq_parent, @@ -382,7 +394,7 @@ generated-members = [VARIABLES] init-import = no dummy-variables-rgx = _|dummy|unused|.*_unused -additional-builtins = +additional-builtins = [CLASSES] defining-attr-methods = __init__,__new__,setUp @@ -403,11 +415,11 @@ max-public-methods = 20 [IMPORTS] deprecated-modules = regsub,TERMIOS,Bastion,rexec -import-graph = -ext-import-graph = -int-import-graph = +import-graph = +ext-import-graph = +int-import-graph = [EXCEPTIONS] overgeneral-exceptions = builtins.Exception -# e624ea03d8124aa9cf2e577f830632344a0a07d9 +# 86de3cc2b03ecf7e6f6f9f2d5f790bb9e7c3eb4c diff --git a/pylintrc_tweaks b/pylintrc_tweaks index 1633da5c10a4..bfd6127880a8 100644 --- a/pylintrc_tweaks +++ b/pylintrc_tweaks @@ -33,6 +33,19 @@ disable+ = c-extension-no-member, no-name-in-module, unnecessary-lambda-assignment, + too-many-positional-arguments, + possibly-used-before-assignment, + use-dict-literal, + useless-return, + superfluous-parens, + logging-not-lazy, + broad-exception-caught, + no-else-raise, + pointless-exception-statement, + consider-using-join, + use-yield-from, + used-before-assignment, + [BASIC] attr-rgx = [a-z_][a-z0-9_]{2,40}$ diff --git a/requirements/constraints.txt b/requirements/constraints.txt index f771f12ebe62..ec078202c16a 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -149,11 +149,6 @@ path<16.12.0 # Constraint can be removed once the issue https://github.com/PyCQA/pycodestyle/issues/1090 is fixed. pycodestyle<2.9.0 -# Date: 2021-07-12 -# Issue for unpinning: https://github.com/openedx/edx-platform/issues/33560 -pylint<2.16.0 # greater version failing quality test. Fix them in seperate ticket. -astroid<2.14.0 - # Date: 2021-08-25 # At the time of writing this comment, we do not know whether py2neo>=2022 # will support our currently-deployed Neo4j version (3.5). diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index f474fe6bef29..821664fe9fc3 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -20,7 +20,7 @@ cryptography==44.0.0 # via -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib -fonttools==4.55.5 +fonttools==4.55.6 # via matplotlib joblib==1.4.2 # via nltk diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c075f9f2d40a..034f03afb0de 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -37,7 +37,7 @@ asgiref==3.8.1 # django-countries asn1crypto==1.5.1 # via snowflake-connector-python -attrs==24.3.0 +attrs==25.1.0 # via # -r requirements/edx/kernel.in # aiohttp @@ -72,13 +72,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.36.5 +boto3==1.36.6 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 -botocore==1.36.5 +botocore==1.36.6 # via # -r requirements/edx/kernel.in # boto3 @@ -1040,7 +1040,7 @@ redis==5.2.1 # via # -r requirements/edx/kernel.in # walrus -referencing==0.36.1 +referencing==0.36.2 # via # jsonschema # jsonschema-specifications diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 412b8c6147bf..5deaa2bcce64 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -82,15 +82,14 @@ asn1crypto==1.5.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # snowflake-connector-python -astroid==2.13.5 +astroid==3.3.8 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pylint # pylint-celery # sphinx-autoapi -attrs==24.3.0 +attrs==25.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -145,14 +144,14 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.36.5 +boto3==1.36.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ses # fs-s3fs # ora2 -botocore==1.36.5 +botocore==1.36.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -766,7 +765,7 @@ edx-i18n-tools==1.5.0 # -r requirements/edx/testing.txt # ora2 # xblocks-contrib -edx-lint==5.5.0 +edx-lint==5.6.0 # via -r requirements/edx/testing.txt edx-milestones==0.6.0 # via @@ -1170,11 +1169,6 @@ lazy==1.6 # lti-consumer-xblock # ora2 # xblock -lazy-object-proxy==1.10.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # astroid libsass==0.10.0 # via # -c requirements/edx/../constraints.txt @@ -1613,9 +1607,8 @@ pylatexenc==2.10 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # olxcleaner -pylint==2.15.10 +pylint==3.3.3 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/testing.txt # edx-lint # pylint-celery @@ -1626,7 +1619,7 @@ pylint-celery==0.3 # via # -r requirements/edx/testing.txt # edx-lint -pylint-django==2.5.5 +pylint-django==2.6.1 # via # -r requirements/edx/testing.txt # edx-lint @@ -1635,7 +1628,7 @@ pylint-plugin-utils==0.8.2 # -r requirements/edx/testing.txt # pylint-celery # pylint-django -pylint-pytest==0.3.0 +pylint-pytest==1.1.8 # via -r requirements/edx/testing.txt pylti1p3==2.0.0 # via @@ -1699,7 +1692,7 @@ pysrt==1.1.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval -pytest==8.3.4 +pytest==8.2.0 # via # -r requirements/edx/testing.txt # pylint-pytest @@ -1718,7 +1711,7 @@ pytest-django==4.9.0 # via -r requirements/edx/testing.txt pytest-json-report==1.5.0 # via -r requirements/edx/testing.txt -pytest-metadata==1.8.0 +pytest-metadata==3.1.1 # via # -r requirements/edx/testing.txt # pytest-json-report @@ -1816,7 +1809,7 @@ redis==5.2.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # walrus -referencing==0.36.1 +referencing==0.36.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2246,7 +2239,6 @@ wrapt==1.17.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt - # astroid xblock[django]==5.1.1 # via # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 06436873c4cd..b54a871071b4 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -57,11 +57,9 @@ asn1crypto==1.5.1 # via # -r requirements/edx/base.txt # snowflake-connector-python -astroid==2.13.5 - # via - # -c requirements/edx/../constraints.txt - # sphinx-autoapi -attrs==24.3.0 +astroid==3.3.8 + # via sphinx-autoapi +attrs==25.1.0 # via # -r requirements/edx/base.txt # aiohttp @@ -107,13 +105,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.36.5 +boto3==1.36.6 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.36.5 +botocore==1.36.6 # via # -r requirements/edx/base.txt # boto3 @@ -851,8 +849,6 @@ lazy==1.6 # lti-consumer-xblock # ora2 # xblock -lazy-object-proxy==1.10.0 - # via astroid loremipsum==1.0.5 # via # -r requirements/edx/base.txt @@ -1271,7 +1267,7 @@ redis==5.2.1 # via # -r requirements/edx/base.txt # walrus -referencing==0.36.1 +referencing==0.36.2 # via # -r requirements/edx/base.txt # jsonschema @@ -1576,9 +1572,7 @@ wheel==0.45.1 # -r requirements/edx/base.txt # django-pipeline wrapt==1.17.2 - # via - # -r requirements/edx/base.txt - # astroid + # via -r requirements/edx/base.txt xblock[django]==5.1.1 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index e5022308aaed..9d25000148f3 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -4,7 +4,7 @@ # # make upgrade # -attrs==24.3.0 +attrs==25.1.0 # via # glom # jsonschema @@ -34,7 +34,7 @@ colorama==0.4.6 # via semgrep defusedxml==0.7.1 # via semgrep -deprecated==1.2.16 +deprecated==1.2.18 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-http @@ -92,13 +92,13 @@ packaging==24.2 # via semgrep peewee==3.17.8 # via semgrep -protobuf==4.25.5 +protobuf==4.25.6 # via # googleapis-common-protos # opentelemetry-proto pygments==2.19.1 # via rich -referencing==0.36.1 +referencing==0.36.2 # via # jsonschema # jsonschema-specifications diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index cf57aeb0fc46..14a0c781da82 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -37,13 +37,13 @@ pytest-attrib # Select tests based on attributes pytest-cov # pytest plugin for measuring code coverage pytest-django # Django support for pytest pytest-json-report # Output json formatted warnings after running pytest -pytest-metadata==1.8.0 # To prevent 'make upgrade' failure, dependency of pytest-json-report +pytest-metadata # To prevent 'make upgrade' failure, dependency of pytest-json-report pytest-randomly # pytest plugin to randomly order tests pytest-xdist[psutil] # Parallel execution of tests on multiple CPU cores or hosts singledispatch # Backport of functools.singledispatch from Python 3.4+, used in tests of XBlock rendering testfixtures # Provides a LogCapture utility used by several tests tox # virtualenv management for tests unidiff # Required by coverage_pytest_plugin -pylint-pytest==0.3.0 # A Pylint plugin to suppress pytest-related false positives. +pylint-pytest # A Pylint plugin to suppress pytest-related false positives. pact-python # Library for contract testing py # Needed for pytest configurations, was previously been fetched through tox diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 4fd65de9cfa6..15c73b94b0fe 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -55,12 +55,11 @@ asn1crypto==1.5.1 # via # -r requirements/edx/base.txt # snowflake-connector-python -astroid==2.13.5 +astroid==3.3.8 # via - # -c requirements/edx/../constraints.txt # pylint # pylint-celery -attrs==24.3.0 +attrs==25.1.0 # via # -r requirements/edx/base.txt # aiohttp @@ -104,13 +103,13 @@ bleach[css]==6.2.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.36.5 +boto3==1.36.6 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.36.5 +botocore==1.36.6 # via # -r requirements/edx/base.txt # boto3 @@ -589,7 +588,7 @@ edx-i18n-tools==1.5.0 # -r requirements/edx/base.txt # ora2 # xblocks-contrib -edx-lint==5.5.0 +edx-lint==5.6.0 # via -r requirements/edx/testing.in edx-milestones==0.6.0 # via -r requirements/edx/base.txt @@ -894,8 +893,6 @@ lazy==1.6 # lti-consumer-xblock # ora2 # xblock -lazy-object-proxy==1.10.0 - # via astroid loremipsum==1.0.5 # via # -r requirements/edx/base.txt @@ -1226,9 +1223,8 @@ pylatexenc==2.10 # via # -r requirements/edx/base.txt # olxcleaner -pylint==2.15.10 +pylint==3.3.3 # via - # -c requirements/edx/../constraints.txt # edx-lint # pylint-celery # pylint-django @@ -1236,13 +1232,13 @@ pylint==2.15.10 # pylint-pytest pylint-celery==0.3 # via edx-lint -pylint-django==2.5.5 +pylint-django==2.6.1 # via edx-lint pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pylint-pytest==0.3.0 +pylint-pytest==1.1.8 # via -r requirements/edx/testing.in pylti1p3==2.0.0 # via -r requirements/edx/base.txt @@ -1287,7 +1283,7 @@ pysrt==1.1.2 # via # -r requirements/edx/base.txt # edxval -pytest==8.3.4 +pytest==8.2.0 # via # -r requirements/edx/testing.in # pylint-pytest @@ -1306,7 +1302,7 @@ pytest-django==4.9.0 # via -r requirements/edx/testing.in pytest-json-report==1.5.0 # via -r requirements/edx/testing.in -pytest-metadata==1.8.0 +pytest-metadata==3.1.1 # via # -r requirements/edx/testing.in # pytest-json-report @@ -1384,7 +1380,7 @@ redis==5.2.1 # via # -r requirements/edx/base.txt # walrus -referencing==0.36.1 +referencing==0.36.2 # via # -r requirements/edx/base.txt # jsonschema @@ -1666,9 +1662,7 @@ wheel==0.45.1 # -r requirements/edx/base.txt # django-pipeline wrapt==1.17.2 - # via - # -r requirements/edx/base.txt - # astroid + # via -r requirements/edx/base.txt xblock[django]==5.1.1 # via # -r requirements/edx/base.txt diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 8f1b96239bc4..35cf225d6b1b 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -6,13 +6,13 @@ # asgiref==3.8.1 # via django -attrs==24.3.0 +attrs==25.1.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -boto3==1.36.5 +boto3==1.36.6 # via -r scripts/user_retirement/requirements/base.in -botocore==1.36.5 +botocore==1.36.6 # via # boto3 # s3transfer diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index 19d54d5adb2f..4f13157aee07 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -8,17 +8,17 @@ asgiref==3.8.1 # via # -r scripts/user_retirement/requirements/base.txt # django -attrs==24.3.0 +attrs==25.1.0 # via # -r scripts/user_retirement/requirements/base.txt # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -boto3==1.36.5 +boto3==1.36.6 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.36.5 +botocore==1.36.6 # via # -r scripts/user_retirement/requirements/base.txt # boto3 From e7771d65264d71dd21feac89e86c32d1a3c11685 Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Mon, 27 Jan 2025 10:59:02 -0800 Subject: [PATCH 19/30] fix: catch a possible exception in beta course configuration (#36172) * fix: catch a possible exception in beta course configuration when the learner is a beta tester, and the beta test has been set up with a large duration, the courseware djangoapp can return an overflow error on a timedelta call. In those circumstances, this defaults to returning an unmodified date. FIXES: APER-3848 --- lms/djangoapps/courseware/access_utils.py | 45 ++++---- lms/djangoapps/courseware/tests/test_utils.py | 103 +++++++++++------- 2 files changed, 88 insertions(+), 60 deletions(-) diff --git a/lms/djangoapps/courseware/access_utils.py b/lms/djangoapps/courseware/access_utils.py index d53699e5e1e4..99aa1e605bcc 100644 --- a/lms/djangoapps/courseware/access_utils.py +++ b/lms/djangoapps/courseware/access_utils.py @@ -3,7 +3,6 @@ It allows us to share code between access.py and block transformers. """ - from datetime import datetime, timedelta from logging import getLogger @@ -21,7 +20,7 @@ EnrollmentRequiredAccessError, IncorrectActiveEnterpriseAccessError, StartDateEnterpriseLearnerError, - StartDateError + StartDateError, ) from lms.djangoapps.courseware.masquerade import get_course_masquerade, is_masquerading_as_student from openedx.features.course_experience import COURSE_ENABLE_UNENROLLED_ACCESS_FLAG, COURSE_PRE_START_ACCESS_FLAG @@ -58,9 +57,12 @@ def adjust_start_date(user, days_early_for_beta, start, course_key): if CourseBetaTesterRole(course_key).has_user(user): debug("Adjust start time: user in beta role for %s", course_key) - delta = timedelta(days_early_for_beta) - effective = start - delta - return effective + # timedelta.max days from now is in the year 2739931, so that's probably pretty safe + delta = timedelta(min(days_early_for_beta, timedelta.max.days)) + try: + return start - delta + except OverflowError: + return start return start @@ -93,7 +95,7 @@ def enterprise_learner_enrolled(request, user, course_key): # enterprise_customer_data is either None (if learner is not linked to any customer) or a serialized # EnterpriseCustomer representing the learner's active linked customer. enterprise_customer_data = enterprise_customer_from_session_or_learner_data(request) - learner_portal_enabled = enterprise_customer_data and enterprise_customer_data['enable_learner_portal'] + learner_portal_enabled = enterprise_customer_data and enterprise_customer_data["enable_learner_portal"] if not learner_portal_enabled: return False @@ -102,18 +104,18 @@ def enterprise_learner_enrolled(request, user, course_key): enterprise_enrollments = EnterpriseCourseEnrollment.objects.filter( course_id=course_key, enterprise_customer_user__user_id=user.id, - enterprise_customer_user__enterprise_customer__uuid=enterprise_customer_data['uuid'], + enterprise_customer_user__enterprise_customer__uuid=enterprise_customer_data["uuid"], ) enterprise_enrollment_exists = enterprise_enrollments.exists() log.info( ( - '[enterprise_learner_enrolled] Checking for an enterprise enrollment for ' - 'lms_user_id=%s in course_key=%s via enterprise_customer_uuid=%s. ' - 'Exists: %s' + "[enterprise_learner_enrolled] Checking for an enterprise enrollment for " + "lms_user_id=%s in course_key=%s via enterprise_customer_uuid=%s. " + "Exists: %s" ), user.id, course_key, - enterprise_customer_data['uuid'], + enterprise_customer_data["uuid"], enterprise_enrollment_exists, ) return enterprise_enrollment_exists @@ -130,7 +132,7 @@ def check_start_date(user, days_early_for_beta, start, course_key, display_error Returns: AccessResponse: Either ACCESS_GRANTED or StartDateError. """ - start_dates_disabled = settings.FEATURES['DISABLE_START_DATES'] + start_dates_disabled = settings.FEATURES["DISABLE_START_DATES"] masquerading_as_student = is_masquerading_as_student(user, course_key) if start_dates_disabled and not masquerading_as_student: @@ -161,8 +163,8 @@ def in_preview_mode(): Returns whether the user is in preview mode or not. """ hostname = get_current_request_hostname() - preview_lms_base = settings.FEATURES.get('PREVIEW_LMS_BASE', None) - return bool(preview_lms_base and hostname and hostname.split(':')[0] == preview_lms_base.split(':')[0]) + preview_lms_base = settings.FEATURES.get("PREVIEW_LMS_BASE", None) + return bool(preview_lms_base and hostname and hostname.split(":")[0] == preview_lms_base.split(":")[0]) def check_course_open_for_learner(user, course): @@ -233,18 +235,19 @@ def check_public_access(course, visibilities): def check_data_sharing_consent(course_id): """ - Grants access if the user is do not need DataSharing consent, otherwise returns data sharing link. + Grants access if the user is do not need DataSharing consent, otherwise returns data sharing link. - Returns: - AccessResponse: Either ACCESS_GRANTED or DataSharingConsentRequiredAccessError - """ + Returns: + AccessResponse: Either ACCESS_GRANTED or DataSharingConsentRequiredAccessError + """ from openedx.features.enterprise_support.api import get_enterprise_consent_url + consent_url = get_enterprise_consent_url( request=get_current_request(), course_id=str(course_id), - return_to='courseware', + return_to="courseware", enrollment_exists=True, - source='CoursewareAccess' + source="CoursewareAccess", ) if consent_url: return DataSharingConsentRequiredAccessError(consent_url=consent_url) @@ -274,7 +277,7 @@ def check_correct_active_enterprise_customer(user, course_id): except (EnterpriseCustomerUser.DoesNotExist, EnterpriseCustomerUser.MultipleObjectsReturned): # Ideally this should not happen. As there should be only 1 active enterprise customer in our system log.error("Multiple or No Active Enterprise found for the user %s.", user.id) - active_enterprise_name = 'Incorrect' + active_enterprise_name = "Incorrect" enrollment_enterprise_name = enterprise_enrollments.first().enterprise_customer_user.enterprise_customer.name return IncorrectActiveEnterpriseAccessError(enrollment_enterprise_name, active_enterprise_name) diff --git a/lms/djangoapps/courseware/tests/test_utils.py b/lms/djangoapps/courseware/tests/test_utils.py index f8508f69c90c..0fedc273905b 100644 --- a/lms/djangoapps/courseware/tests/test_utils.py +++ b/lms/djangoapps/courseware/tests/test_utils.py @@ -1,7 +1,9 @@ """ Unit test for various Utility functions """ + import json +from datetime import date, timedelta from unittest.mock import patch import ddt @@ -14,13 +16,44 @@ from common.djangoapps.student.tests.factories import GlobalStaffFactory, UserFactory from lms.djangoapps.courseware.constants import UNEXPECTED_ERROR_IS_ELIGIBLE from lms.djangoapps.courseware.tests.factories import FinancialAssistanceConfigurationFactory +from lms.djangoapps.courseware.access_utils import adjust_start_date from lms.djangoapps.courseware.utils import ( create_financial_assistance_application, get_financial_assistance_application_status, - is_eligible_for_financial_aid + is_eligible_for_financial_aid, ) +@ddt.ddt +class TestAccessUtils(TestCase): + """Tests for the access_utils functions.""" + + @ddt.data( + # days_early_for_beta, is_beta_user, expected_date + (None, True, "2025-01-03"), + (2, True, "2025-01-01"), + (timedelta.max.days + 10, True, "2025-01-03"), + (None, False, "2025-01-03"), + (2, False, "2025-01-03"), + (timedelta.max.days + 10, False, "2025-01-03"), + ) + @ddt.unpack + def test_adjust_start_date(self, days_early_for_beta, is_beta_user, expected_date): + """Tests adjust_start_date + + Should only modify the date if the user is beta for the course, + and `days_early_for_beta` is sensible number.""" + start = date(2025, 1, 3) + expected = date.fromisoformat(expected_date) + user = "princessofpower" + course_key = "edx1+8675" + with patch("lms.djangoapps.courseware.access_utils.CourseBetaTesterRole") as role_mock: + instance = role_mock.return_value + instance.has_user.return_value = is_beta_user + adjusted_date = adjust_start_date(user, days_early_for_beta, start, course_key) + self.assertEqual(expected, adjusted_date) + + @ddt.ddt class TestFinancialAssistanceViews(TestCase): """ @@ -29,17 +62,17 @@ class TestFinancialAssistanceViews(TestCase): def setUp(self) -> None: super().setUp() - self.test_course_id = 'course-v1:edX+Test+1' + self.test_course_id = "course-v1:edX+Test+1" self.user = UserFactory() self.global_staff = GlobalStaffFactory.create() _ = FinancialAssistanceConfigurationFactory( - api_base_url='http://financial.assistance.test:1234', + api_base_url="http://financial.assistance.test:1234", service_username=self.global_staff.username, fa_backend_enabled_courses_percentage=100, - enabled=True + enabled=True, ) _ = Application.objects.create( - name='Test Application', + name="Test Application", user=self.global_staff, client_type=Application.CLIENT_PUBLIC, authorization_grant_type=Application.GRANT_CLIENT_CREDENTIALS, @@ -51,33 +84,31 @@ def _mock_response(self, status_code, content=None): """ mock_response = Response() mock_response.status_code = status_code - mock_response._content = json.dumps(content).encode('utf-8') # pylint: disable=protected-access + mock_response._content = json.dumps(content).encode("utf-8") # pylint: disable=protected-access return mock_response @ddt.data( - {'is_eligible': True, 'reason': None}, - {'is_eligible': False, 'reason': 'This course is not eligible for financial aid'} + {"is_eligible": True, "reason": None}, + {"is_eligible": False, "reason": "This course is not eligible for financial aid"}, ) def test_is_eligible_for_financial_aid(self, response_data): """ Tests the functionality of is_eligible_for_financial_aid which calls edx-financial-assistance backend to return eligibility status for financial assistance for a given course. """ - with patch.object(OAuthAPIClient, 'request') as oauth_mock: + with patch.object(OAuthAPIClient, "request") as oauth_mock: oauth_mock.return_value = self._mock_response(status.HTTP_200_OK, response_data) is_eligible, reason = is_eligible_for_financial_aid(self.test_course_id) - assert is_eligible is response_data.get('is_eligible') - assert reason == response_data.get('reason') + assert is_eligible is response_data.get("is_eligible") + assert reason == response_data.get("reason") def test_is_eligible_for_financial_aid_invalid_course_id(self): """ Tests the functionality of is_eligible_for_financial_aid for an invalid course id. """ error_message = f"Invalid course id {self.test_course_id} provided" - with patch.object(OAuthAPIClient, 'request') as oauth_mock: - oauth_mock.return_value = self._mock_response( - status.HTTP_400_BAD_REQUEST, {"message": error_message} - ) + with patch.object(OAuthAPIClient, "request") as oauth_mock: + oauth_mock.return_value = self._mock_response(status.HTTP_400_BAD_REQUEST, {"message": error_message}) is_eligible, reason = is_eligible_for_financial_aid(self.test_course_id) assert is_eligible is False assert reason == error_message @@ -86,9 +117,9 @@ def test_is_eligible_for_financial_aid_invalid_unexpected_error(self): """ Tests the functionality of is_eligible_for_financial_aid for an unexpected error """ - with patch.object(OAuthAPIClient, 'request') as oauth_mock: + with patch.object(OAuthAPIClient, "request") as oauth_mock: oauth_mock.return_value = self._mock_response( - status.HTTP_500_INTERNAL_SERVER_ERROR, {'error': 'unexpected error occurred'} + status.HTTP_500_INTERNAL_SERVER_ERROR, {"error": "unexpected error occurred"} ) is_eligible, reason = is_eligible_for_financial_aid(self.test_course_id) assert is_eligible is False @@ -99,33 +130,27 @@ def test_get_financial_assistance_application_status(self): Tests the functionality of get_financial_assistance_application_status against a user id and a course id edx-financial-assistance backend to return status of a financial assistance application. """ - test_response = {'id': 123, 'status': 'ACCEPTED', 'coupon_code': 'ABCD..'} - with patch.object(OAuthAPIClient, 'request') as oauth_mock: + test_response = {"id": 123, "status": "ACCEPTED", "coupon_code": "ABCD.."} + with patch.object(OAuthAPIClient, "request") as oauth_mock: oauth_mock.return_value = self._mock_response(status.HTTP_200_OK, test_response) has_application, reason = get_financial_assistance_application_status(self.user.id, self.test_course_id) assert has_application is True assert reason == test_response @ddt.data( - { - 'status': status.HTTP_400_BAD_REQUEST, - 'content': {'message': 'Invalid course id provided'} - }, - { - 'status': status.HTTP_404_NOT_FOUND, - 'content': {'message': 'Application details not found'} - } + {"status": status.HTTP_400_BAD_REQUEST, "content": {"message": "Invalid course id provided"}}, + {"status": status.HTTP_404_NOT_FOUND, "content": {"message": "Application details not found"}}, ) def test_get_financial_assistance_application_status_unsuccessful(self, response_data): """ Tests unsuccessful scenarios of get_financial_assistance_application_status against a user id and a course id edx-financial-assistance backend. """ - with patch.object(OAuthAPIClient, 'request') as oauth_mock: - oauth_mock.return_value = self._mock_response(response_data.get('status'), response_data.get('content')) + with patch.object(OAuthAPIClient, "request") as oauth_mock: + oauth_mock.return_value = self._mock_response(response_data.get("status"), response_data.get("content")) has_application, reason = get_financial_assistance_application_status(self.user.id, self.test_course_id) assert has_application is False - assert reason == response_data.get('content').get('message') + assert reason == response_data.get("content").get("message") def test_create_financial_assistance_application(self): """ @@ -133,11 +158,11 @@ def test_create_financial_assistance_application(self): to create a new financial assistance application given a form data. """ test_form_data = { - 'lms_user_id': self.user.id, - 'course_id': self.test_course_id, + "lms_user_id": self.user.id, + "course_id": self.test_course_id, } - with patch.object(OAuthAPIClient, 'request') as oauth_mock: - oauth_mock.return_value = self._mock_response(status.HTTP_200_OK, {'success': True}) + with patch.object(OAuthAPIClient, "request") as oauth_mock: + oauth_mock.return_value = self._mock_response(status.HTTP_200_OK, {"success": True}) response = create_financial_assistance_application(form_data=test_form_data) assert response.status_code == status.HTTP_204_NO_CONTENT @@ -147,12 +172,12 @@ def test_create_financial_assistance_application_bad_request(self): to create a new financial assistance application given a form data. """ test_form_data = { - 'lms_user_id': self.user.id, - 'course_id': 'invalid_course_id', + "lms_user_id": self.user.id, + "course_id": "invalid_course_id", } - error_response = {'message': 'Invalid course id provided'} - with patch.object(OAuthAPIClient, 'request') as oauth_mock: + error_response = {"message": "Invalid course id provided"} + with patch.object(OAuthAPIClient, "request") as oauth_mock: oauth_mock.return_value = self._mock_response(status.HTTP_400_BAD_REQUEST, error_response) response = create_financial_assistance_application(form_data=test_form_data) assert response.status_code == status.HTTP_400_BAD_REQUEST - assert json.loads(response.content.decode('utf-8')) == error_response + assert json.loads(response.content.decode("utf-8")) == error_response From dc2a38b1f4f4edae08e7a7f2bebfbe6a06f06c8d Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 27 Jan 2025 15:29:29 -0500 Subject: [PATCH 20/30] feat: dump_settings management command (#36162) This command dumps the current Django settings to JSON for debugging/diagnostics. The output of this command is for *humans*... it is NOT suitable for consumption by production systems. In particular, we are introducing this command as part of a series of refactorings to the Django settings files lms/envs/* and cms/envs/*. We want to ensure that these refactorings do not introduce any unexpected breaking changes, so the dump_settings command will both help us manually verify our refactorings and help operators verify that our refactorings behave expectedly when using their custom python/yaml settings files. Related to: https://github.com/openedx/edx-platform/pull/36131 --- .../util/management/commands/dump_settings.py | 93 +++++++++++++++++++ .../util/tests/test_dump_settings.py | 64 +++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 openedx/core/djangoapps/util/management/commands/dump_settings.py create mode 100644 openedx/core/djangoapps/util/tests/test_dump_settings.py diff --git a/openedx/core/djangoapps/util/management/commands/dump_settings.py b/openedx/core/djangoapps/util/management/commands/dump_settings.py new file mode 100644 index 000000000000..1f9949ffbdaf --- /dev/null +++ b/openedx/core/djangoapps/util/management/commands/dump_settings.py @@ -0,0 +1,93 @@ +""" +Defines the dump_settings management command. +""" +import inspect +import json +import re + +from django.conf import settings +from django.core.management.base import BaseCommand + + +SETTING_NAME_REGEX = re.compile(r'^[A-Z][A-Z0-9_]*$') + + +class Command(BaseCommand): + """ + Dump current Django settings to JSON for debugging/diagnostics. + + BEWARE: OUTPUT IS NOT SUITABLE FOR CONSUMPTION BY PRODUCTION SYSTEMS. + The purpose of this output is to be *helpful* for a *human* operator to understand how their settings are being + rendered and how they differ between different settings files. The serialization format is NOT perfect: there are + certain situations where two different settings will output identical JSON. For example, this command does NOT: + + disambiguate between lists and tuples: + * (1, 2, 3) # <-- this tuple will be printed out as [1, 2, 3] + * [1, 2, 3] + + disambiguate between sets and sorted lists: + * {2, 1, 3} # <-- this set will be printed out as [1, 2, 3] + * [1, 2, 3] + + disambiguate between internationalized and non-internationalized strings: + * _("hello") # <-- this will become just "hello" + * "hello" + + Furthermore, objects which are not easily JSON-ifiable will stringified using their `repr(...)`, e.g.: + * "Path('my/path')" # a Path object + * "" # some random class instance + * "<_io.TextIOWrapper name='' mode='w' encoding='utf-8'>" # sys.stderr + + and lambdas are printed by *roughly* printing out their source lines (it's impossible in Python to get the *exact* + source code, as it's been compiled into bytecode). + """ + + def handle(self, *args, **kwargs): + """ + Handle the command. + """ + settings_json = { + name: _to_json_friendly_repr(getattr(settings, name)) + for name in dir(settings) + if SETTING_NAME_REGEX.match(name) + } + print(json.dumps(settings_json, indent=4)) + + +def _to_json_friendly_repr(value: object) -> object: + """ + Turn the value into something that we can print to a JSON file (that is: str, bool, None, int, float, list, dict). + + See the docstring of `Command` for warnings about this function's behavior. + """ + if isinstance(value, (type(None), bool, int, float, str)): + # All these types can be printed directly + return value + if isinstance(value, (list, tuple, set)): + if isinstance(value, set): + # Print sets by sorting them (so that order doesn't matter) into a JSON array. + elements = sorted(value) + else: + # Print both lists and tuples as JSON arrays. + elements = value + return [_to_json_friendly_repr(element) for ix, element in enumerate(elements)] + if isinstance(value, dict): + # Print dicts as JSON objects + for subkey in value.keys(): + if not isinstance(subkey, (str, int)): + raise ValueError(f"Unexpected dict key {subkey} of type {type(subkey)}") + return {subkey: _to_json_friendly_repr(subval) for subkey, subval in value.items()} + if proxy_args := getattr(value, "_proxy____args", None): + if len(proxy_args) == 1 and isinstance(proxy_args[0], str): + # Print gettext_lazy as simply the wrapped string + return proxy_args[0] + try: + qualname = value.__qualname__ + except AttributeError: + pass + else: + if qualname == "": + # Handle lambdas by printing the source lines + return "lambda defined with line(s): " + inspect.getsource(value).strip() + # For all other objects, print the repr + return repr(value) diff --git a/openedx/core/djangoapps/util/tests/test_dump_settings.py b/openedx/core/djangoapps/util/tests/test_dump_settings.py new file mode 100644 index 000000000000..b8712e9aed1c --- /dev/null +++ b/openedx/core/djangoapps/util/tests/test_dump_settings.py @@ -0,0 +1,64 @@ +""" +Basic tests for dump_settings management command. + +These are moreso testing that dump_settings works, less-so testing anything about the Django +settings files themselves. Remember that tests only run with (lms,cms)/envs/test.py, +which are based on (lms,cms)/envs/common.py, so these tests will not execute any of the +YAML-loading or post-processing defined in (lms,cms)/envs/production.py. +""" +import json + +from django.core.management import call_command + +from openedx.core.djangolib.testing.utils import skip_unless_lms, skip_unless_cms + + +@skip_unless_lms +def test_for_lms_settings(capsys): + """ + Ensure LMS's test settings can be dumped, and sanity-check them for certain values. + """ + dump = _get_settings_dump(capsys) + + # Check: something LMS-specific + assert dump['MODULESTORE_BRANCH'] == "published-only" + + # Check: tuples are converted to lists + assert isinstance(dump['XBLOCK_MIXINS'], list) + + # Check: objects (like classes) are repr'd + assert "" in dump['XBLOCK_MIXINS'] + + # Check: nested dictionaries come through OK, and int'l strings are just strings + assert dump['COURSE_ENROLLMENT_MODES']['audit']['display_name'] == "Audit" + + +@skip_unless_cms +def test_for_cms_settings(capsys): + """ + Ensure CMS's test settings can be dumped, and sanity-check them for certain values. + """ + dump = _get_settings_dump(capsys) + + # Check: something CMS-specific + assert dump['MODULESTORE_BRANCH'] == "draft-preferred" + + # Check: tuples are converted to lists + assert isinstance(dump['XBLOCK_MIXINS'], list) + + # Check: objects (like classes) are repr'd + assert "" in dump['XBLOCK_MIXINS'] + + # Check: nested dictionaries come through OK, and int'l strings are just strings + assert dump['COURSE_ENROLLMENT_MODES']['audit']['display_name'] == "Audit" + + +def _get_settings_dump(captured_sys): + """ + Call dump_settings, ensure no error output, and return parsed JSON. + """ + call_command('dump_settings') + out, err = captured_sys.readouterr() + assert out + assert not err + return json.loads(out) From 8e1e55a0d7273207c78ee52e7ec03003c507a4c4 Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Mon, 27 Jan 2025 13:53:47 -0800 Subject: [PATCH 21/30] chore: linting as a separate commit (#36179) I'm letting autoformat hit this file to make it match our current standards before I actually make any code changes. --- .../management/commands/notify_credentials.py | 95 +++++++++---------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py index cbfdd8e7337f..1b7ff682baf6 100644 --- a/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py +++ b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py @@ -7,6 +7,7 @@ This management command will manually trigger the receivers we care about. (We don't want to trigger all receivers for these signals, since these are busy signals.) """ + import logging import shlex @@ -58,6 +59,7 @@ class Command(BaseCommand): course-v1:edX+RecordsSelfPaced+1 for user 17 course-v1:edX+RecordsSelfPaced+1 for user 18 """ + help = ( "Simulate certificate/grade changes without actually modifying database " "content. Specifically, trigger the handlers that send data to Credentials." @@ -65,98 +67,98 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--dry-run', - action='store_true', - help='Just show a preview of what would happen.', + "--dry-run", + action="store_true", + help="Just show a preview of what would happen.", ) parser.add_argument( - '--site', + "--site", default=None, help="Site domain to notify for (if not specified, all sites are notified). Uses course_org_filter.", ) parser.add_argument( - '--courses', - nargs='+', - help='Send information only for specific course runs.', + "--courses", + nargs="+", + help="Send information only for specific course runs.", ) parser.add_argument( - '--program_uuids', - nargs='+', - help='Send user data for course runs for courses within a program based on program uuids provided.', + "--program_uuids", + nargs="+", + help="Send user data for course runs for courses within a program based on program uuids provided.", ) parser.add_argument( - '--start-date', + "--start-date", type=parsetime, - help='Send information only for certificates or grades that have changed since this date.', + help="Send information only for certificates or grades that have changed since this date.", ) parser.add_argument( - '--end-date', + "--end-date", type=parsetime, - help='Send information only for certificates or grades that have changed before this date.', + help="Send information only for certificates or grades that have changed before this date.", ) parser.add_argument( - '--delay', + "--delay", type=float, default=0, help="Number of seconds to sleep between processing queries, so that we don't flood our queues.", ) parser.add_argument( - '--page-size', + "--page-size", type=int, default=100, help="Number of items to query at once.", ) parser.add_argument( - '--auto', - action='store_true', - help='Use to run the management command periodically', + "--auto", + action="store_true", + help="Use to run the management command periodically", ) parser.add_argument( - '--args-from-database', - action='store_true', - help='Use arguments from the NotifyCredentialsConfig model instead of the command line.', + "--args-from-database", + action="store_true", + help="Use arguments from the NotifyCredentialsConfig model instead of the command line.", ) parser.add_argument( - '--verbose', - action='store_true', - help='Run grade/cert change signal in verbose mode', + "--verbose", + action="store_true", + help="Run grade/cert change signal in verbose mode", ) parser.add_argument( - '--notify_programs', - action='store_true', - help='Send program award notifications with course notification tasks', + "--notify_programs", + action="store_true", + help="Send program award notifications with course notification tasks", ) parser.add_argument( - '--user_ids', + "--user_ids", default=None, - nargs='+', - help='Run the command for the given user or list of users', + nargs="+", + help="Run the command for the given user or list of users", ) parser.add_argument( - '--revoke_program_certs', - action='store_true', - help="If true, system will check if any program certificates need to be revoked from learners" + "--revoke_program_certs", + action="store_true", + help="If true, system will check if any program certificates need to be revoked from learners", ) def get_args_from_database(self): - """ Returns an options dictionary from the current NotifyCredentialsConfig model. """ + """Returns an options dictionary from the current NotifyCredentialsConfig model.""" config = NotifyCredentialsConfig.current() if not config.enabled: - raise CommandError('NotifyCredentialsConfig is disabled, but --args-from-database was requested.') + raise CommandError("NotifyCredentialsConfig is disabled, but --args-from-database was requested.") # This split will allow for quotes to wrap datetimes, like "2020-10-20 04:00:00" and other # arguments as if it were the command line argv = shlex.split(config.arguments) - parser = self.create_parser('manage.py', 'notify_credentials') - return parser.parse_args(argv).__dict__ # we want a dictionary, not a non-iterable Namespace object + parser = self.create_parser("manage.py", "notify_credentials") + return parser.parse_args(argv).__dict__ # we want a dictionary, not a non-iterable Namespace object def handle(self, *args, **options): - if options['args_from_database']: + if options["args_from_database"]: options = self.get_args_from_database() - if options['auto']: - options['end_date'] = datetime.now().replace(minute=0, second=0, microsecond=0) - options['start_date'] = options['end_date'] - timedelta(hours=4) + if options["auto"]: + options["end_date"] = datetime.now().replace(minute=0, second=0, microsecond=0) + options["start_date"] = options["end_date"] - timedelta(hours=4) log.info( f"notify_credentials starting, dry-run={options['dry_run']}, site={options['site']}, " @@ -176,14 +178,9 @@ def handle(self, *args, **options): course_runs.extend(program_course_run_keys) course_run_keys = self._get_validated_course_run_keys(course_runs) - if not ( - course_run_keys or - options['start_date'] or - options['end_date'] or - options['user_ids'] - ): + if not (course_run_keys or options["start_date"] or options["end_date"] or options["user_ids"]): raise CommandError( - 'You must specify a filter (e.g. --courses, --program_uuids, --start-date, or --user_ids)' + "You must specify a filter (e.g. --courses, --program_uuids, --start-date, or --user_ids)" ) handle_notify_credentials.delay(options, course_run_keys) From 1fe67d3f6b40233791d4599bae28df8c0ac91c4d Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Mon, 27 Jan 2025 14:26:00 -0800 Subject: [PATCH 22/30] feat: increase frequency of scheduled notify credentials (#36180) `notify_credentials` has 2 ways of running. 1. The manual, one-off method which uses `--args_from_database` to specify what should be sent. 2. [The automated method](https://github.com/openedx/edx-platform/blob/7316111b35c8db0b93665e00aa4071685d772ab3/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py#L157-L159), which runs on a regular schedule, to catch anything which fell through the cracks. The automated method does a certain amount of time/date math in order to calculate the entry point of the window based on the current runtime. This is, I assume, why it has some hardcoded logic; it's not at all simple to have a `cron`-run management command running on a regular cadence that can do the same time logic. ```py if options['auto']: options['end_date'] = datetime.now().replace(minute=0, second=0, microsecond=0) options['start_date'] = options['end_date'] - timedelta(hours=4) ``` However, it is not ideal that the actual time window of 4 hours is hardcoded directly into `edx-platform`. This fix * creates an overridable `NOTIFY_CREDENTIALS_FREQUENCY` that defaults to 14400 seconds (4 hours). * pulls that frequency into the quoted code Using seconds allows maximum flexibility. FIXES: APER-3383 --- cms/envs/common.py | 2 ++ lms/envs/common.py | 2 ++ .../management/commands/notify_credentials.py | 4 +++- .../commands/tests/test_notify_credentials.py | 16 +++++++++++++++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 591247388a9d..0919834a9b64 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2520,6 +2520,8 @@ CREDENTIALS_INTERNAL_SERVICE_URL = 'http://localhost:8005' CREDENTIALS_PUBLIC_SERVICE_URL = 'http://localhost:8005' CREDENTIALS_SERVICE_USERNAME = 'credentials_service_user' +# time between scheduled runs, in seconds +NOTIFY_CREDENTIALS_FREQUENCY = 14400 ANALYTICS_DASHBOARD_URL = 'http://localhost:18110/courses' ANALYTICS_DASHBOARD_NAME = 'Your Platform Name Here Insights' diff --git a/lms/envs/common.py b/lms/envs/common.py index 76127d062d81..c57a08b617db 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -4331,6 +4331,8 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring CREDENTIALS_INTERNAL_SERVICE_URL = 'http://localhost:8005' CREDENTIALS_PUBLIC_SERVICE_URL = 'http://localhost:8005' +# time between scheduled runs, in seconds +NOTIFY_CREDENTIALS_FREQUENCY = 14400 COMMENTS_SERVICE_URL = 'http://localhost:18080' COMMENTS_SERVICE_KEY = 'password' diff --git a/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py index 1b7ff682baf6..f79ead24d9b0 100644 --- a/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py +++ b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py @@ -13,6 +13,7 @@ from datetime import datetime, timedelta import dateutil.parser +from django.conf import settings from django.core.management.base import BaseCommand, CommandError from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey @@ -157,8 +158,9 @@ def handle(self, *args, **options): options = self.get_args_from_database() if options["auto"]: + run_frequency = settings.NOTIFY_CREDENTIALS_FREQUENCY options["end_date"] = datetime.now().replace(minute=0, second=0, microsecond=0) - options["start_date"] = options["end_date"] - timedelta(hours=4) + options["start_date"] = options["end_date"] - timedelta(seconds=run_frequency) log.info( f"notify_credentials starting, dry-run={options['dry_run']}, site={options['site']}, " diff --git a/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py b/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py index e50173001fd3..f7bed3446aca 100644 --- a/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py +++ b/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py @@ -7,7 +7,7 @@ from django.core.management import call_command from django.core.management.base import CommandError -from django.test import TestCase, override_settings # lint-amnesty, pylint: disable=unused-import +from django.test import TestCase, override_settings from freezegun import freeze_time from openedx.core.djangoapps.catalog.tests.factories import ProgramFactory, CourseFactory, CourseRunFactory @@ -125,6 +125,7 @@ def test_multiple_programs_uuid_args(self, mock_get_programs, mock_task): @freeze_time(datetime(2017, 5, 1, 4)) def test_auto_execution(self, mock_task): + """Verify that an automatic execution designed for scheduled windows works correctly""" self.expected_options['auto'] = True self.expected_options['start_date'] = datetime(2017, 5, 1, 0, 0) self.expected_options['end_date'] = datetime(2017, 5, 1, 4, 0) @@ -133,6 +134,19 @@ def test_auto_execution(self, mock_task): assert mock_task.called assert mock_task.call_args[0][0] == self.expected_options + @override_settings(NOTIFY_CREDENTIALS_FREQUENCY=3600) + @freeze_time(datetime(2017, 5, 1, 4)) + def test_auto_execution_different_schedule(self, mock_task): + """Verify that an automatic execution designed for scheduled windows + works correctly if the window frequency has been changed""" + self.expected_options["auto"] = True + self.expected_options["start_date"] = datetime(2017, 5, 1, 3, 0) + self.expected_options["end_date"] = datetime(2017, 5, 1, 4, 0) + + call_command(Command(), "--auto") + assert mock_task.called + assert mock_task.call_args[0][0] == self.expected_options + def test_date_args(self, mock_task): self.expected_options['start_date'] = datetime(2017, 1, 31, 0, 0, tzinfo=timezone.utc) call_command(Command(), '--start-date', '2017-01-31') From 98a4a32805b28a8eec7b5719782d0dd4fe1f4920 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 29 Jan 2025 11:05:57 -0500 Subject: [PATCH 23/30] feat: Make dump for fns, classes more stable and helpful (#36188) The `dump_settings` command currently prints out the raw `repr(...)`s for defined functions, e.g.: "WIKI_CAN_ASSIGN": "", In addition to being uninformative, these `at 0x74ce...` hashes change every run, so they appear in the diff as having "changed" every time. With this commit, here's what `dump_settings` will print out for a function instead: "WIKI_CAN_ASSIGN": { "module": "lms.djangoapps.course_wiki.settings", "qualname": "CAN_ASSIGN" }, --- .../util/management/commands/dump_settings.py | 25 ++++++++++++------- .../util/tests/test_dump_settings.py | 8 +++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/openedx/core/djangoapps/util/management/commands/dump_settings.py b/openedx/core/djangoapps/util/management/commands/dump_settings.py index 1f9949ffbdaf..965ea16c6d27 100644 --- a/openedx/core/djangoapps/util/management/commands/dump_settings.py +++ b/openedx/core/djangoapps/util/management/commands/dump_settings.py @@ -33,13 +33,14 @@ class Command(BaseCommand): * _("hello") # <-- this will become just "hello" * "hello" - Furthermore, objects which are not easily JSON-ifiable will stringified using their `repr(...)`, e.g.: - * "Path('my/path')" # a Path object - * "" # some random class instance - * "<_io.TextIOWrapper name='' mode='w' encoding='utf-8'>" # sys.stderr + Furthermore, functions and classes are printed as JSON objects like: + { + "module": "path.to.module", + "qualname": "MyClass.MyInnerClass.my_method", // Or, "" + "source_hint": "MY_SETTING = lambda: x + y", // For s only + } - and lambdas are printed by *roughly* printing out their source lines (it's impossible in Python to get the *exact* - source code, as it's been compiled into bytecode). + And everything else will be stringified as its `repr(...)`. """ def handle(self, *args, **kwargs): @@ -82,12 +83,18 @@ def _to_json_friendly_repr(value: object) -> object: # Print gettext_lazy as simply the wrapped string return proxy_args[0] try: + module = value.__module__ qualname = value.__qualname__ except AttributeError: pass else: - if qualname == "": - # Handle lambdas by printing the source lines - return "lambda defined with line(s): " + inspect.getsource(value).strip() + # Handle functions and classes by printing their location (plus approximate source, for lambdas) + return { + "module": module, + "qualname": qualname, + **({ + "source_hint": inspect.getsource(value).strip(), + } if qualname == "" else {}), + } # For all other objects, print the repr return repr(value) diff --git a/openedx/core/djangoapps/util/tests/test_dump_settings.py b/openedx/core/djangoapps/util/tests/test_dump_settings.py index b8712e9aed1c..90171eb48c95 100644 --- a/openedx/core/djangoapps/util/tests/test_dump_settings.py +++ b/openedx/core/djangoapps/util/tests/test_dump_settings.py @@ -26,8 +26,8 @@ def test_for_lms_settings(capsys): # Check: tuples are converted to lists assert isinstance(dump['XBLOCK_MIXINS'], list) - # Check: objects (like classes) are repr'd - assert "" in dump['XBLOCK_MIXINS'] + # Check: classes are converted to dicts of info on the class location + assert {"module": "xmodule.x_module", "qualname": "XModuleMixin"} in dump['XBLOCK_MIXINS'] # Check: nested dictionaries come through OK, and int'l strings are just strings assert dump['COURSE_ENROLLMENT_MODES']['audit']['display_name'] == "Audit" @@ -46,8 +46,8 @@ def test_for_cms_settings(capsys): # Check: tuples are converted to lists assert isinstance(dump['XBLOCK_MIXINS'], list) - # Check: objects (like classes) are repr'd - assert "" in dump['XBLOCK_MIXINS'] + # Check: classes are converted to dicts of info on the class location + assert {"module": "xmodule.x_module", "qualname": "XModuleMixin"} in dump['XBLOCK_MIXINS'] # Check: nested dictionaries come through OK, and int'l strings are just strings assert dump['COURSE_ENROLLMENT_MODES']['audit']['display_name'] == "Audit" From 5054ef13c136f56ace142048711819737a291b6e Mon Sep 17 00:00:00 2001 From: pwnage101 <85151+pwnage101@users.noreply.github.com> Date: Wed, 29 Jan 2025 21:13:51 +0000 Subject: [PATCH 24/30] feat: Upgrade Python dependency edx-enterprise Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index ec078202c16a..38089f4cce1c 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -80,7 +80,7 @@ django-storages<1.14.4 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==5.6.5 +edx-enterprise==5.6.6 # Date: 2024-05-09 # This has to be constrained as well because newer versions of edx-i18n-tools need the diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 034f03afb0de..28b2f2f3b87c 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -467,7 +467,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.5 +edx-enterprise==5.6.6 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 5deaa2bcce64..93bd005966a0 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -745,7 +745,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.5 +edx-enterprise==5.6.6 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index b54a871071b4..7b5bb8207338 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -552,7 +552,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.5 +edx-enterprise==5.6.6 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 15c73b94b0fe..9ea71f258ccb 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -574,7 +574,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.6.5 +edx-enterprise==5.6.6 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From b6828cecaabe54856716e6e7db083b3e06156573 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Thu, 30 Jan 2025 17:15:33 +0500 Subject: [PATCH 25/30] fix: enable pylint warnings (#36195) * fix: enable pylint warnings --- .../commands/backfill_course_tabs.py | 3 +-- .../commands/export_content_library.py | 25 +++++++++---------- .../commands/bulk_change_enrollment_csv.py | 2 +- common/djangoapps/track/views/segmentio.py | 5 ++-- lms/djangoapps/discussion/rest_api/api.py | 2 +- lms/djangoapps/experiments/flags.py | 2 +- lms/djangoapps/static_template_view/views.py | 3 +-- openedx/core/djangoapps/credit/tasks.py | 3 +-- openedx/core/djangoapps/enrollments/data.py | 3 +-- .../djangoapps/safe_sessions/middleware.py | 15 ++++++----- .../commands/bulk_user_org_email_optout.py | 9 +++---- openedx/core/lib/api/view_utils.py | 3 +-- openedx/core/lib/celery/task_utils.py | 5 ++-- pylintrc | 7 +----- pylintrc_tweaks | 6 ----- xmodule/capa/tests/test_input_templates.py | 3 +-- xmodule/split_test_block.py | 5 ++-- 17 files changed, 39 insertions(+), 62 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py b/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py index 878a8dabaa27..768c3a53f7ac 100644 --- a/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py +++ b/cms/djangoapps/contentstore/management/commands/backfill_course_tabs.py @@ -71,6 +71,5 @@ def handle(self, *args, **options): if error_keys: msg = 'The following courses encountered errors and were not updated:\n' - for error_key in error_keys: - msg += f' - {error_key}\n' + msg += '\n'.join(f' - {error_key}' for error_key in error_keys) logger.info(msg) diff --git a/cms/djangoapps/contentstore/management/commands/export_content_library.py b/cms/djangoapps/contentstore/management/commands/export_content_library.py index 0b4cbfb1fbad..b56c172e374e 100644 --- a/cms/djangoapps/contentstore/management/commands/export_content_library.py +++ b/cms/djangoapps/contentstore/management/commands/export_content_library.py @@ -51,16 +51,15 @@ def handle(self, *args, **options): tarball = tasks.create_export_tarball(library, library_key, {}, None) except Exception as e: raise CommandError(f'Failed to export "{library_key}" with "{e}"') # lint-amnesty, pylint: disable=raise-missing-from - else: - with tarball: - # Save generated archive with keyed filename - prefix, suffix, n = str(library_key).replace(':', '+'), '.tar.gz', 0 - while os.path.exists(prefix + suffix): - n += 1 - prefix = '{}_{}'.format(prefix.rsplit('_', 1)[0], n) if n > 1 else f'{prefix}_1' - filename = prefix + suffix - target = os.path.join(dest_path, filename) - tarball.file.seek(0) - with open(target, 'wb') as f: - shutil.copyfileobj(tarball.file, f) - print(f'Library "{library.location.library_key}" exported to "{target}"') + with tarball: + # Save generated archive with keyed filename + prefix, suffix, n = str(library_key).replace(':', '+'), '.tar.gz', 0 + while os.path.exists(prefix + suffix): + n += 1 + prefix = '{}_{}'.format(prefix.rsplit('_', 1)[0], n) if n > 1 else f'{prefix}_1' + filename = prefix + suffix + target = os.path.join(dest_path, filename) + tarball.file.seek(0) + with open(target, 'wb') as f: + shutil.copyfileobj(tarball.file, f) + print(f'Library "{library.location.library_key}" exported to "{target}"') diff --git a/common/djangoapps/student/management/commands/bulk_change_enrollment_csv.py b/common/djangoapps/student/management/commands/bulk_change_enrollment_csv.py index a9c1d5349858..69f2e35de44a 100644 --- a/common/djangoapps/student/management/commands/bulk_change_enrollment_csv.py +++ b/common/djangoapps/student/management/commands/bulk_change_enrollment_csv.py @@ -81,7 +81,7 @@ def handle(self, *args, **options): self.change_enrollments(csv_file) else: - CommandError('No file is provided. File is required') + raise CommandError('No file is provided. File is required') def change_enrollments(self, csv_file): """ change the enrollments of the learners. """ diff --git a/common/djangoapps/track/views/segmentio.py b/common/djangoapps/track/views/segmentio.py index 2ab5306232c5..8a35a0e421cf 100644 --- a/common/djangoapps/track/views/segmentio.py +++ b/common/djangoapps/track/views/segmentio.py @@ -205,9 +205,8 @@ def track_segmentio_event(request): # pylint: disable=too-many-statements raise EventValidationError(ERROR_USER_NOT_EXIST) # lint-amnesty, pylint: disable=raise-missing-from except ValueError: raise EventValidationError(ERROR_INVALID_USER_ID) # lint-amnesty, pylint: disable=raise-missing-from - else: - context['user_id'] = user.id - context['username'] = user.username + context['user_id'] = user.id + context['username'] = user.username # course_id is expected to be provided in the context when applicable course_id = context.get('course_id') diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index a517e00dff34..7123ca9e3fb4 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -1010,7 +1010,7 @@ def get_thread_list( if view in ["unread", "unanswered", "unresponded"]: query_params[view] = "true" else: - ValidationError({ + raise ValidationError({ "view": [f"Invalid value. '{view}' must be 'unread' or 'unanswered'"] }) diff --git a/lms/djangoapps/experiments/flags.py b/lms/djangoapps/experiments/flags.py index b83a7751fae7..3b45bcee5b5c 100644 --- a/lms/djangoapps/experiments/flags.py +++ b/lms/djangoapps/experiments/flags.py @@ -245,7 +245,7 @@ def get_bucket(self, course_key=None, track=True): if ( track and hasattr(request, 'session') and session_key not in request.session and - not masquerading_as_specific_student and not anonymous + not masquerading_as_specific_student and not anonymous # pylint: disable=used-before-assignment ): segment.track( user_id=user.id, diff --git a/lms/djangoapps/static_template_view/views.py b/lms/djangoapps/static_template_view/views.py index a788f77a95fd..f8962897c033 100644 --- a/lms/djangoapps/static_template_view/views.py +++ b/lms/djangoapps/static_template_view/views.py @@ -92,8 +92,7 @@ def render_press_release(request, slug): resp = render_to_response('static_templates/press_releases/' + template, {}) except TemplateDoesNotExist: raise Http404 # lint-amnesty, pylint: disable=raise-missing-from - else: - return resp + return resp @fix_crum_request diff --git a/openedx/core/djangoapps/credit/tasks.py b/openedx/core/djangoapps/credit/tasks.py index 312e278a985f..79ef613e3d19 100644 --- a/openedx/core/djangoapps/credit/tasks.py +++ b/openedx/core/djangoapps/credit/tasks.py @@ -41,8 +41,7 @@ def update_credit_course_requirements(course_id): except (InvalidKeyError, ItemNotFoundError, InvalidCreditRequirements) as exc: LOGGER.error('Error on adding the requirements for course %s - %s', course_id, str(exc)) raise update_credit_course_requirements.retry(args=[course_id], exc=exc) - else: - LOGGER.info('Requirements added for course %s', course_id) + LOGGER.info('Requirements added for course %s', course_id) def _get_course_credit_requirements(course_key): diff --git a/openedx/core/djangoapps/enrollments/data.py b/openedx/core/djangoapps/enrollments/data.py index 9986830a3491..b76042f72c9d 100644 --- a/openedx/core/djangoapps/enrollments/data.py +++ b/openedx/core/djangoapps/enrollments/data.py @@ -341,8 +341,7 @@ def get_course_enrollment_info(course_id, include_expired=False): msg = f"Requested enrollment information for unknown course {course_id}" log.warning(msg) raise CourseNotFoundError(msg) # lint-amnesty, pylint: disable=raise-missing-from - else: - return CourseSerializer(course, include_expired=include_expired).data + return CourseSerializer(course, include_expired=include_expired).data def get_user_roles(username): diff --git a/openedx/core/djangoapps/safe_sessions/middleware.py b/openedx/core/djangoapps/safe_sessions/middleware.py index f3948217efd9..950a3e08c54f 100644 --- a/openedx/core/djangoapps/safe_sessions/middleware.py +++ b/openedx/core/djangoapps/safe_sessions/middleware.py @@ -244,14 +244,13 @@ def parse(cls, safe_cookie_string): raise SafeCookieError( # lint-amnesty, pylint: disable=raise-missing-from f"SafeCookieData BWC parse error: {safe_cookie_string!r}." ) - else: - if safe_cookie_data.version != cls.CURRENT_VERSION: - raise SafeCookieError( - "SafeCookieData version {!r} is not supported. Current version is {}.".format( - safe_cookie_data.version, - cls.CURRENT_VERSION, - )) - return safe_cookie_data + if safe_cookie_data.version != cls.CURRENT_VERSION: + raise SafeCookieError( + "SafeCookieData version {!r} is not supported. Current version is {}.".format( + safe_cookie_data.version, + cls.CURRENT_VERSION, + )) + return safe_cookie_data def __str__(self): """ diff --git a/openedx/core/djangoapps/user_api/management/commands/bulk_user_org_email_optout.py b/openedx/core/djangoapps/user_api/management/commands/bulk_user_org_email_optout.py index e465ff5610ed..d194e58ee880 100644 --- a/openedx/core/djangoapps/user_api/management/commands/bulk_user_org_email_optout.py +++ b/openedx/core/djangoapps/user_api/management/commands/bulk_user_org_email_optout.py @@ -135,11 +135,10 @@ def handle(self, *args, **options): optout_rows[end_idx][0], optout_rows[end_idx][1], str(err)) raise - else: - cursor.execute('COMMIT;') - log.info("Committed opt-out for rows (%s, %s) through (%s, %s).", - optout_rows[start_idx][0], optout_rows[start_idx][1], - optout_rows[end_idx][0], optout_rows[end_idx][1]) + cursor.execute('COMMIT;') + log.info("Committed opt-out for rows (%s, %s) through (%s, %s).", + optout_rows[start_idx][0], optout_rows[start_idx][1], + optout_rows[end_idx][0], optout_rows[end_idx][1]) log.info("Sleeping %s seconds...", sleep_between) time.sleep(sleep_between) curr_row_idx += chunk_size diff --git a/openedx/core/lib/api/view_utils.py b/openedx/core/lib/api/view_utils.py index 054755ae3cc1..d876e49ae579 100644 --- a/openedx/core/lib/api/view_utils.py +++ b/openedx/core/lib/api/view_utils.py @@ -265,8 +265,7 @@ def __len__(self): def __iter__(self): # Yield all the known data first - for item in self._data: - yield item + yield from self._data # Capture and yield data from the underlying iterator # until it is exhausted diff --git a/openedx/core/lib/celery/task_utils.py b/openedx/core/lib/celery/task_utils.py index 738f074be68c..9a54f1b3a550 100644 --- a/openedx/core/lib/celery/task_utils.py +++ b/openedx/core/lib/celery/task_utils.py @@ -50,9 +50,8 @@ def emulate_http_request(site=None, user=None, middleware_classes=None): for middleware in reversed(middleware_instances): _run_method_if_implemented(middleware, 'process_exception', request, exc) raise - else: - for middleware in reversed(middleware_instances): - _run_method_if_implemented(middleware, 'process_response', request, response) + for middleware in reversed(middleware_instances): + _run_method_if_implemented(middleware, 'process_response', request, response) def _run_method_if_implemented(instance, method_name, *args, **kwargs): diff --git a/pylintrc b/pylintrc index 81b984a23cf8..e06941853774 100644 --- a/pylintrc +++ b/pylintrc @@ -321,11 +321,6 @@ disable = superfluous-parens, logging-not-lazy, broad-exception-caught, - no-else-raise, - pointless-exception-statement, - consider-using-join, - use-yield-from, - used-before-assignment, [REPORTS] output-format = text @@ -422,4 +417,4 @@ int-import-graph = [EXCEPTIONS] overgeneral-exceptions = builtins.Exception -# 86de3cc2b03ecf7e6f6f9f2d5f790bb9e7c3eb4c +# 85c3d025c367597a0a7b23a05fdde9d8c63e1374 diff --git a/pylintrc_tweaks b/pylintrc_tweaks index bfd6127880a8..fb0bebfd2a12 100644 --- a/pylintrc_tweaks +++ b/pylintrc_tweaks @@ -40,12 +40,6 @@ disable+ = superfluous-parens, logging-not-lazy, broad-exception-caught, - no-else-raise, - pointless-exception-statement, - consider-using-join, - use-yield-from, - used-before-assignment, - [BASIC] attr-rgx = [a-z_][a-z0-9_]{2,40}$ diff --git a/xmodule/capa/tests/test_input_templates.py b/xmodule/capa/tests/test_input_templates.py index 4b14bd5ef86c..3048f62095de 100644 --- a/xmodule/capa/tests/test_input_templates.py +++ b/xmodule/capa/tests/test_input_templates.py @@ -76,8 +76,7 @@ def render_to_xml(self, context_dict): except Exception as exc: raise TemplateError("Could not parse XML from '{0}': {1}".format( # lint-amnesty, pylint: disable=raise-missing-from xml_str, str(exc))) - else: - return xml + return xml def assert_has_xpath(self, xml_root, xpath, context_dict, exact_num=1): """ diff --git a/xmodule/split_test_block.py b/xmodule/split_test_block.py index 05ca3a5db454..52f40547879a 100644 --- a/xmodule/split_test_block.py +++ b/xmodule/split_test_block.py @@ -419,9 +419,8 @@ def log_child_render(self, request, suffix=''): # lint-amnesty, pylint: disable ) ) raise - else: - self.runtime.publish(self, 'xblock.split_test.child_render', {'child_id': child_id}) - return Response() + self.runtime.publish(self, 'xblock.split_test.child_render', {'child_id': child_id}) + return Response() def get_icon_class(self): return self.child.get_icon_class() if self.child else 'other' From a81493ce7f2c735a513e3edaf877d39641311414 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Thu, 16 Jan 2025 15:19:47 -0500 Subject: [PATCH 26/30] revert: revert: refactor: Clean up lms/envs/production.py cruft This reintroduces commit 15939232d547b751a06e9cb0c1c39ba78144bade, which was reverted due to a typo. The typo is fixed in the commit immediately following this one. Co-Authored-By: Feanil Patel --- lms/envs/common.py | 28 +- lms/envs/production.py | 875 ++++++++++------------------------------- 2 files changed, 225 insertions(+), 678 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index c57a08b617db..d08d0ea0f154 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -3392,9 +3392,35 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring CSRF_COOKIE_SECURE = False CSRF_TRUSTED_ORIGINS = [] CSRF_TRUSTED_ORIGINS_WITH_SCHEME = [] -CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = '' + +# If setting a cross-domain cookie, it's really important to choose +# a name for the cookie that is DIFFERENT than the cookies used +# by each subdomain. For example, suppose the applications +# at these subdomains are configured to use the following cookie names: +# +# 1) foo.example.com --> "csrftoken" +# 2) baz.example.com --> "csrftoken" +# 3) bar.example.com --> "csrftoken" +# +# For the cross-domain version of the CSRF cookie, you need to choose +# a name DIFFERENT than "csrftoken"; otherwise, the new token configured +# for ".example.com" could conflict with the other cookies, +# non-deterministically causing 403 responses. CROSS_DOMAIN_CSRF_COOKIE_NAME = '' +# When setting the domain for the "cross-domain" version of the CSRF +# cookie, you should choose something like: ".example.com" +# (note the leading dot), where both the referer and the host +# are subdomains of "example.com". +# +# Browser security rules require that +# the cookie domain matches the domain of the server; otherwise +# the cookie won't get set. And once the cookie gets set, the client +# needs to be on a domain that matches the cookie domain, otherwise +# the client won't be able to read the cookie. +CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = '' + + ######################### Django Rest Framework ######################## REST_FRAMEWORK = { diff --git a/lms/envs/production.py b/lms/envs/production.py index addcea0e6028..3ea1c952e479 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -22,7 +22,6 @@ import os import yaml -import django from django.core.exceptions import ImproperlyConfigured from edx_django_utils.plugins import add_plugins from openedx_events.event_bus import merge_producer_configs @@ -44,7 +43,11 @@ def get_env_setting(setting): error_msg = "Set the %s env variable" % setting raise ImproperlyConfigured(error_msg) # lint-amnesty, pylint: disable=raise-missing-from -################################ ALWAYS THE SAME ############################## + +################################################# PRODUCTION DEFAULTS ################################################ +# We configure some defaults (beyond what has already been configured in common.py) before loading the YAML file below. +# DO NOT ADD NEW DEFAULTS HERE! Put any new setting defaults in common.py instead, along with a setting annotation. +# TODO: Move all these defaults into common.py. DEBUG = False DEFAULT_TEMPLATE_ENGINE['OPTIONS']['debug'] = False @@ -58,7 +61,72 @@ def get_env_setting(setting): # https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header # for other warnings. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -################################ END ALWAYS THE SAME ############################## + +CELERY_RESULT_BACKEND = 'django-cache' +BROKER_HEARTBEAT = 60.0 +BROKER_HEARTBEAT_CHECKRATE = 2 +STATIC_ROOT_BASE = None +STATIC_URL_BASE = None +EMAIL_HOST = 'localhost' +EMAIL_PORT = 25 +EMAIL_USE_TLS = False +SESSION_COOKIE_DOMAIN = None +SESSION_COOKIE_HTTPONLY = True +AWS_SES_REGION_NAME = 'us-east-1' +AWS_SES_REGION_ENDPOINT = 'email.us-east-1.amazonaws.com' +REGISTRATION_EMAIL_PATTERNS_ALLOWED = None +LMS_ROOT_URL = None +CMS_BASE = 'studio.edx.org' +CELERY_EVENT_QUEUE_TTL = None +COMPREHENSIVE_THEME_LOCALE_PATHS = [] +PREPEND_LOCALE_PATHS = [] +COURSE_LISTINGS = {} +COMMENTS_SERVICE_URL = '' +COMMENTS_SERVICE_KEY = '' +CERT_QUEUE = 'test-pull' +PYTHON_LIB_FILENAME = 'python_lib.zip' +VIDEO_CDN_URL = {} +HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS = {} +AWS_STORAGE_BUCKET_NAME = 'edxuploads' +# Disabling querystring auth instructs Boto to exclude the querystring parameters (e.g. signature, access key) it +# normally appends to every returned URL. +AWS_QUERYSTRING_AUTH = True +AWS_S3_CUSTOM_DOMAIN = 'edxuploads.s3.amazonaws.com' +MONGODB_LOG = {} +ZENDESK_USER = None +ZENDESK_API_KEY = None +EDX_API_KEY = None +CELERY_BROKER_TRANSPORT = "" +CELERY_BROKER_HOSTNAME = "" +CELERY_BROKER_VHOST = "" +CELERY_BROKER_USER = "" +CELERY_BROKER_PASSWORD = "" +BROKER_USE_SSL = False +SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = None +ENABLE_REQUIRE_THIRD_PARTY_AUTH = False +GOOGLE_ANALYTICS_TRACKING_ID = None +GOOGLE_ANALYTICS_LINKEDIN = None +GOOGLE_SITE_VERIFICATION_ID = None +BRANCH_IO_KEY = None +REGISTRATION_CODE_LENGTH = 8 +FACEBOOK_API_VERSION = None +FACEBOOK_APP_SECRET = None +FACEBOOK_APP_ID = None +API_ACCESS_MANAGER_EMAIL = None +API_ACCESS_FROM_EMAIL = None +CHAT_COMPLETION_API = '' +CHAT_COMPLETION_API_KEY = '' +OPENAPI_CACHE_TIMEOUT = 60 * 60 +MAINTENANCE_BANNER_TEXT = None +DASHBOARD_COURSE_LIMIT = None + +# TODO: We believe these were part of the DEPR'd sysadmin dashboard, and can likely be removed. +SSL_AUTH_EMAIL_DOMAIN = "MIT.EDU" +SSL_AUTH_DN_FORMAT_STRING = ( + "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" +) + +####################################################################################################################### # A file path to a YAML file from which to load all the configuration for the edx platform CONFIG_FILE = get_env_setting('LMS_CFG') @@ -66,10 +134,13 @@ def get_env_setting(setting): with codecs.open(CONFIG_FILE, encoding='utf-8') as f: __config__ = yaml.safe_load(f) - # ENV_TOKENS and AUTH_TOKENS are included for reverse compatibility. - # Removing them may break plugins that rely on them. - ENV_TOKENS = __config__ - AUTH_TOKENS = __config__ + # _YAML_TOKENS contains the exact contents of the LMS_CFG YAML file. + # We do splat the entirety of the LMS_CFG YAML file (except KEYS_WITH_MERGED_VALUES) into this module. + # However, for precise backwards compatibility, we need to reference _YAML_TOKENS directly a few times, + # particularly we need to derive Django setting values from YAML values. + # This pattern is confusing and we discourage it. Rather than adding more _YAML_TOKENS references, please + # consider just referencing this module's variables directly. + _YAML_TOKENS = __config__ # Add the key/values from config into the global namespace of this module. # But don't override the FEATURES dict because we do that in an additive way. @@ -82,7 +153,6 @@ def get_env_setting(setting): 'JWT_AUTH', 'CELERY_QUEUES', 'MKTG_URL_LINK_MAP', - 'MKTG_URL_OVERRIDES', 'REST_FRAMEWORK', 'EVENT_BUS_PRODUCER_CONFIG', ] @@ -111,20 +181,11 @@ def get_env_setting(setting): BROKER_POOL_LIMIT = 0 BROKER_CONNECTION_TIMEOUT = 1 -# Allow env to configure celery result backend with default set to django-cache -CELERY_RESULT_BACKEND = ENV_TOKENS.get('CELERY_RESULT_BACKEND', 'django-cache') - -# When the broker is behind an ELB, use a heartbeat to refresh the -# connection and to detect if it has been dropped. -BROKER_HEARTBEAT = ENV_TOKENS.get('BROKER_HEARTBEAT', 60.0) -BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2) - # Each worker should only fetch one message at a time CELERYD_PREFETCH_MULTIPLIER = 1 # STATIC_ROOT specifies the directory where static files are # collected -STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None) if STATIC_ROOT_BASE: STATIC_ROOT = path(STATIC_ROOT_BASE) WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" @@ -132,83 +193,34 @@ def get_env_setting(setting): # STATIC_URL_BASE specifies the base url to use for static files -STATIC_URL_BASE = ENV_TOKENS.get('STATIC_URL_BASE', None) if STATIC_URL_BASE: STATIC_URL = STATIC_URL_BASE if not STATIC_URL.endswith("/"): STATIC_URL += "/" -# Allow overriding build profile used by RequireJS with one -# contained on a custom theme -REQUIRE_BUILD_PROFILE = ENV_TOKENS.get('REQUIRE_BUILD_PROFILE', REQUIRE_BUILD_PROFILE) - -# The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text -# values when the variable is an empty string. Therefore, setting these variable as empty text in related -# json files will make the system reads their values from django translation files -PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME') or PLATFORM_NAME -PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION') or PLATFORM_DESCRIPTION - -DATA_DIR = path(ENV_TOKENS.get('DATA_DIR', DATA_DIR)) -CC_MERCHANT_NAME = ENV_TOKENS.get('CC_MERCHANT_NAME', PLATFORM_NAME) -EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', DATA_DIR / "emails" / "lms") -EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', 'localhost') # django default is localhost -EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', 25) # django default is 25 -EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', False) # django default is False -SITE_NAME = ENV_TOKENS.get('SITE_NAME', SITE_NAME) -SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN') -SESSION_COOKIE_HTTPONLY = ENV_TOKENS.get('SESSION_COOKIE_HTTPONLY', True) - -DCS_SESSION_COOKIE_SAMESITE = ENV_TOKENS.get('DCS_SESSION_COOKIE_SAMESITE', DCS_SESSION_COOKIE_SAMESITE) -DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL = ENV_TOKENS.get('DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL', DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL) # lint-amnesty, pylint: disable=line-too-long - -# As django-cookies-samesite package is set to be removed from base requirements when we upgrade to Django 3.2, -# we should follow the settings name provided by Django. -# https://docs.djangoproject.com/en/3.2/ref/settings/#session-cookie-samesite -SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE - -AWS_SES_REGION_NAME = ENV_TOKENS.get('AWS_SES_REGION_NAME', 'us-east-1') -AWS_SES_REGION_ENDPOINT = ENV_TOKENS.get('AWS_SES_REGION_ENDPOINT', 'email.us-east-1.amazonaws.com') +DATA_DIR = path(DATA_DIR) +CC_MERCHANT_NAME = _YAML_TOKENS.get('CC_MERCHANT_NAME', PLATFORM_NAME) +EMAIL_FILE_PATH = _YAML_TOKENS.get('EMAIL_FILE_PATH', DATA_DIR / "emails" / "lms") -REGISTRATION_EMAIL_PATTERNS_ALLOWED = ENV_TOKENS.get('REGISTRATION_EMAIL_PATTERNS_ALLOWED') - -LMS_ROOT_URL = ENV_TOKENS.get('LMS_ROOT_URL') -LMS_INTERNAL_ROOT_URL = ENV_TOKENS.get('LMS_INTERNAL_ROOT_URL', LMS_ROOT_URL) +# TODO: This was for backwards compatibility back when installed django-cookie-samesite (not since 2022). +# The DCS_ version of the setting can be DEPR'd at this point. +SESSION_COOKIE_SAMESITE = DCS_SESSION_COOKIE_SAMESITE -# List of logout URIs for each IDA that the learner should be logged out of when they logout of the LMS. Only applies to -# IDA for which the social auth flow uses DOT (Django OAuth Toolkit). -IDA_LOGOUT_URI_LIST = ENV_TOKENS.get('IDA_LOGOUT_URI_LIST', []) +LMS_INTERNAL_ROOT_URL = _YAML_TOKENS.get('LMS_INTERNAL_ROOT_URL', LMS_ROOT_URL) -ENV_FEATURES = ENV_TOKENS.get('FEATURES', {}) -for feature, value in ENV_FEATURES.items(): +for feature, value in _YAML_TOKENS.get('FEATURES', {}).items(): FEATURES[feature] = value -CMS_BASE = ENV_TOKENS.get('CMS_BASE', 'studio.edx.org') - ALLOWED_HOSTS = [ "*", - ENV_TOKENS.get('LMS_BASE'), + _YAML_TOKENS.get('LMS_BASE'), FEATURES['PREVIEW_LMS_BASE'], ] -# Sometimes, OAuth2 clients want the user to redirect back to their site after logout. But to determine if the given -# redirect URL/path is safe for redirection, the following variable is used by edX. -LOGIN_REDIRECT_WHITELIST = ENV_TOKENS.get( - 'LOGIN_REDIRECT_WHITELIST', - LOGIN_REDIRECT_WHITELIST -) - -# allow for environments to specify what cookie name our login subsystem should use -# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can -# happen with some browsers (e.g. Firefox) -if ENV_TOKENS.get('SESSION_COOKIE_NAME', None): - # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str() - SESSION_COOKIE_NAME = str(ENV_TOKENS.get('SESSION_COOKIE_NAME')) - # This is the domain that is used to set shared cookies between various sub-domains. # By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN, but we want to make it overrideable. -SHARED_COOKIE_DOMAIN = ENV_TOKENS.get('SHARED_COOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) +SHARED_COOKIE_DOMAIN = _YAML_TOKENS.get('SHARED_kCOOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) -CACHES = ENV_TOKENS.get('CACHES', CACHES) # Cache used for location mapping -- called many times with the same key/value # in a given request. if 'loc_cache' not in CACHES: @@ -224,206 +236,76 @@ def get_env_setting(setting): # we need to run asset collection twice, once for local disk and once for S3. # Once we have migrated to service assets off S3, then we can convert this back to # managed by the yaml file contents -STATICFILES_STORAGE = os.environ.get('STATICFILES_STORAGE', ENV_TOKENS.get('STATICFILES_STORAGE', STATICFILES_STORAGE)) - -# Load all AWS_ prefixed variables to allow an S3Boto3Storage to be configured -_locals = locals() -for key, value in ENV_TOKENS.items(): - if key.startswith('AWS_'): - _locals[key] = value - -# Currency -PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CURRENCY', - PAID_COURSE_REGISTRATION_CURRENCY) # We want Bulk Email running on the high-priority queue, so we define the # routing key that points to it. At the moment, the name is the same. # We have to reset the value here, since we have changed the value of the queue name. -BULK_EMAIL_ROUTING_KEY = ENV_TOKENS.get('BULK_EMAIL_ROUTING_KEY', HIGH_PRIORITY_QUEUE) +BULK_EMAIL_ROUTING_KEY = _YAML_TOKENS.get('BULK_EMAIL_ROUTING_KEY', HIGH_PRIORITY_QUEUE) # We can run smaller jobs on the low priority queue. See note above for why # we have to reset the value here. -BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = ENV_TOKENS.get('BULK_EMAIL_ROUTING_KEY_SMALL_JOBS', DEFAULT_PRIORITY_QUEUE) +BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = _YAML_TOKENS.get('BULK_EMAIL_ROUTING_KEY_SMALL_JOBS', DEFAULT_PRIORITY_QUEUE) # Queue to use for expiring old entitlements -ENTITLEMENTS_EXPIRATION_ROUTING_KEY = ENV_TOKENS.get('ENTITLEMENTS_EXPIRATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) +ENTITLEMENTS_EXPIRATION_ROUTING_KEY = _YAML_TOKENS.get('ENTITLEMENTS_EXPIRATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) -# Message expiry time in seconds -CELERY_EVENT_QUEUE_TTL = ENV_TOKENS.get('CELERY_EVENT_QUEUE_TTL', None) - -# Allow CELERY_QUEUES to be overwritten by ENV_TOKENS, -ENV_CELERY_QUEUES = ENV_TOKENS.get('CELERY_QUEUES', None) -if ENV_CELERY_QUEUES: - CELERY_QUEUES = {queue: {} for queue in ENV_CELERY_QUEUES} +# Build a CELERY_QUEUES dict the way that celery expects, based on a couple lists of queue names from the YAML. +_YAML_CELERY_QUEUES = _YAML_TOKENS.get('CELERY_QUEUES', None) +if _YAML_CELERY_QUEUES: + CELERY_QUEUES = {queue: {} for queue in _YAML_CELERY_QUEUES} # Then add alternate environment queues -ALTERNATE_QUEUE_ENVS = ENV_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() +_YAML_ALTERNATE_WORKER_QUEUES = _YAML_TOKENS.get('ALTERNATE_WORKER_QUEUES', '').split() ALTERNATE_QUEUES = [ DEFAULT_PRIORITY_QUEUE.replace(QUEUE_VARIANT, alternate + '.') - for alternate in ALTERNATE_QUEUE_ENVS + for alternate in _YAML_ALTERNATE_WORKER_QUEUES ] + CELERY_QUEUES.update( { alternate: {} for alternate in ALTERNATE_QUEUES - if alternate not in list(CELERY_QUEUES.keys()) + if alternate not in CELERY_QUEUES.keys() } ) -# following setting is for backward compatibility -if ENV_TOKENS.get('COMPREHENSIVE_THEME_DIR', None): - COMPREHENSIVE_THEME_DIR = ENV_TOKENS.get('COMPREHENSIVE_THEME_DIR') - - -# COMPREHENSIVE_THEME_LOCALE_PATHS contain the paths to themes locale directories e.g. -# "COMPREHENSIVE_THEME_LOCALE_PATHS" : [ -# "/edx/src/edx-themes/conf/locale" -# ], -COMPREHENSIVE_THEME_LOCALE_PATHS = ENV_TOKENS.get('COMPREHENSIVE_THEME_LOCALE_PATHS', []) - - -# PREPEND_LOCALE_PATHS contain the paths to locale directories to load first e.g. -# "PREPEND_LOCALE_PATHS" : [ -# "/edx/my-locale" -# ], -PREPEND_LOCALE_PATHS = ENV_TOKENS.get('PREPEND_LOCALE_PATHS', []) - - -MKTG_URL_LINK_MAP.update(ENV_TOKENS.get('MKTG_URL_LINK_MAP', {})) -ENTERPRISE_MARKETING_FOOTER_QUERY_PARAMS = ENV_TOKENS.get( - 'ENTERPRISE_MARKETING_FOOTER_QUERY_PARAMS', - ENTERPRISE_MARKETING_FOOTER_QUERY_PARAMS -) -# Marketing link overrides -MKTG_URL_OVERRIDES.update(ENV_TOKENS.get('MKTG_URL_OVERRIDES', MKTG_URL_OVERRIDES)) +MKTG_URL_LINK_MAP.update(_YAML_TOKENS.get('MKTG_URL_LINK_MAP', {})) # Intentional defaults. -ID_VERIFICATION_SUPPORT_LINK = ENV_TOKENS.get('ID_VERIFICATION_SUPPORT_LINK', SUPPORT_SITE_LINK) -PASSWORD_RESET_SUPPORT_LINK = ENV_TOKENS.get('PASSWORD_RESET_SUPPORT_LINK', SUPPORT_SITE_LINK) -ACTIVATION_EMAIL_SUPPORT_LINK = ENV_TOKENS.get('ACTIVATION_EMAIL_SUPPORT_LINK', SUPPORT_SITE_LINK) -LOGIN_ISSUE_SUPPORT_LINK = ENV_TOKENS.get('LOGIN_ISSUE_SUPPORT_LINK', SUPPORT_SITE_LINK) +ID_VERIFICATION_SUPPORT_LINK = _YAML_TOKENS.get('ID_VERIFICATION_SUPPORT_LINK', SUPPORT_SITE_LINK) +PASSWORD_RESET_SUPPORT_LINK = _YAML_TOKENS.get('PASSWORD_RESET_SUPPORT_LINK', SUPPORT_SITE_LINK) +ACTIVATION_EMAIL_SUPPORT_LINK = _YAML_TOKENS.get('ACTIVATION_EMAIL_SUPPORT_LINK', SUPPORT_SITE_LINK) +LOGIN_ISSUE_SUPPORT_LINK = _YAML_TOKENS.get('LOGIN_ISSUE_SUPPORT_LINK', SUPPORT_SITE_LINK) # Timezone overrides -TIME_ZONE = ENV_TOKENS.get('CELERY_TIMEZONE', CELERY_TIMEZONE) +TIME_ZONE = CELERY_TIMEZONE # Translation overrides LANGUAGE_DICT = dict(LANGUAGES) -LANGUAGE_COOKIE_NAME = ENV_TOKENS.get('LANGUAGE_COOKIE', None) or ENV_TOKENS.get( - 'LANGUAGE_COOKIE_NAME', LANGUAGE_COOKIE_NAME) +LANGUAGE_COOKIE_NAME = _YAML_TOKENS.get('LANGUAGE_COOKIE') or LANGUAGE_COOKIE_NAME # Additional installed apps -for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): +for app in _YAML_TOKENS.get('ADDL_INSTALLED_APPS', []): INSTALLED_APPS.append(app) - -local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') -LOG_DIR = ENV_TOKENS.get('LOG_DIR', LOG_DIR) - -LOGGING = get_logger_config(LOG_DIR, - logging_env=ENV_TOKENS.get('LOGGING_ENV', LOGGING_ENV), - local_loglevel=local_loglevel, - service_variant=SERVICE_VARIANT) - -COURSE_LISTINGS = ENV_TOKENS.get('COURSE_LISTINGS', {}) -COMMENTS_SERVICE_URL = ENV_TOKENS.get("COMMENTS_SERVICE_URL", '') -COMMENTS_SERVICE_KEY = ENV_TOKENS.get("COMMENTS_SERVICE_KEY", '') -CERT_QUEUE = ENV_TOKENS.get("CERT_QUEUE", 'test-pull') - -# Python lib settings -PYTHON_LIB_FILENAME = ENV_TOKENS.get('PYTHON_LIB_FILENAME', 'python_lib.zip') - -# Code jail settings -for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): - oldvalue = CODE_JAIL.get(name) - if isinstance(oldvalue, dict): - for subname, subvalue in value.items(): - oldvalue[subname] = subvalue - else: - CODE_JAIL[name] = value - -COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", []) - -# Event Tracking -if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS: - TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS") - -# SSL external authentication settings -SSL_AUTH_EMAIL_DOMAIN = ENV_TOKENS.get("SSL_AUTH_EMAIL_DOMAIN", "MIT.EDU") -SSL_AUTH_DN_FORMAT_STRING = ENV_TOKENS.get( - "SSL_AUTH_DN_FORMAT_STRING", - "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" +LOGGING = get_logger_config( + LOG_DIR, + logging_env=LOGGING_ENV, + local_loglevel=LOCAL_LOGLEVEL, + service_variant=SERVICE_VARIANT, ) -# Video Caching. Pairing country codes with CDN URLs. -# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='} -VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {}) - -# Determines whether the CSRF token can be transported on -# unencrypted channels. It is set to False here for backward compatibility, -# but it is highly recommended that this is True for environments accessed -# by end users. -CSRF_COOKIE_SECURE = ENV_TOKENS.get('CSRF_COOKIE_SECURE', False) - # Determines which origins are trusted for unsafe requests eg. POST requests. -CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS', []) -# values are already updated above with default CSRF_TRUSTED_ORIGINS values but in -# case of new django version these values will override. -if django.VERSION[0] >= 4: # for greater than django 3.2 use schemes. - CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', []) +CSRF_TRUSTED_ORIGINS = _YAML_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', []) -############# CORS headers for cross-domain requests ################# - -if FEATURES.get('ENABLE_CORS_HEADERS') or FEATURES.get('ENABLE_CROSS_DOMAIN_CSRF_COOKIE'): +if FEATURES['ENABLE_CORS_HEADERS'] or FEATURES.get('ENABLE_CROSS_DOMAIN_CSRF_COOKIE'): CORS_ALLOW_CREDENTIALS = True - CORS_ORIGIN_WHITELIST = ENV_TOKENS.get('CORS_ORIGIN_WHITELIST', ()) - - CORS_ORIGIN_ALLOW_ALL = ENV_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) - CORS_ALLOW_INSECURE = ENV_TOKENS.get('CORS_ALLOW_INSECURE', False) - - # If setting a cross-domain cookie, it's really important to choose - # a name for the cookie that is DIFFERENT than the cookies used - # by each subdomain. For example, suppose the applications - # at these subdomains are configured to use the following cookie names: - # - # 1) foo.example.com --> "csrftoken" - # 2) baz.example.com --> "csrftoken" - # 3) bar.example.com --> "csrftoken" - # - # For the cross-domain version of the CSRF cookie, you need to choose - # a name DIFFERENT than "csrftoken"; otherwise, the new token configured - # for ".example.com" could conflict with the other cookies, - # non-deterministically causing 403 responses. - # - # Because of the way Django stores cookies, the cookie name MUST - # be a `str`, not unicode. Otherwise there will `TypeError`s will be raised - # when Django tries to call the unicode `translate()` method with the wrong - # number of parameters. - CROSS_DOMAIN_CSRF_COOKIE_NAME = str(ENV_TOKENS.get('CROSS_DOMAIN_CSRF_COOKIE_NAME')) - - # When setting the domain for the "cross-domain" version of the CSRF - # cookie, you should choose something like: ".example.com" - # (note the leading dot), where both the referer and the host - # are subdomains of "example.com". - # - # Browser security rules require that - # the cookie domain matches the domain of the server; otherwise - # the cookie won't get set. And once the cookie gets set, the client - # needs to be on a domain that matches the cookie domain, otherwise - # the client won't be able to read the cookie. - CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = ENV_TOKENS.get('CROSS_DOMAIN_CSRF_COOKIE_DOMAIN') - - -# Field overrides. To use the IDDE feature, add -# 'courseware.student_field_overrides.IndividualStudentOverrideProvider'. -FIELD_OVERRIDE_PROVIDERS = tuple(ENV_TOKENS.get('FIELD_OVERRIDE_PROVIDERS', [])) - -############### XBlock filesystem field config ########## -if 'DJFS' in AUTH_TOKENS and AUTH_TOKENS['DJFS'] is not None: - DJFS = AUTH_TOKENS['DJFS'] - -############### Module Store Items ########## -HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS = ENV_TOKENS.get('HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS', {}) + CORS_ORIGIN_WHITELIST = _YAML_TOKENS.get('CORS_ORIGIN_WHITELIST', ()) + CORS_ORIGIN_ALLOW_ALL = _YAML_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) + CORS_ALLOW_INSECURE = _YAML_TOKENS.get('CORS_ALLOW_INSECURE', False) + CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = _YAML_TOKENS.get('CROSS_DOMAIN_CSRF_COOKIE_DOMAIN') + # PREVIEW DOMAIN must be present in HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS for the preview to show draft changes if 'PREVIEW_LMS_BASE' in FEATURES and FEATURES['PREVIEW_LMS_BASE'] != '': PREVIEW_DOMAIN = FEATURES['PREVIEW_LMS_BASE'].split(':')[0] @@ -432,26 +314,11 @@ def get_env_setting(setting): PREVIEW_DOMAIN: 'draft-preferred' }) -MODULESTORE_FIELD_OVERRIDE_PROVIDERS = ENV_TOKENS.get( - 'MODULESTORE_FIELD_OVERRIDE_PROVIDERS', - MODULESTORE_FIELD_OVERRIDE_PROVIDERS -) - -XBLOCK_FIELD_DATA_WRAPPERS = ENV_TOKENS.get( - 'XBLOCK_FIELD_DATA_WRAPPERS', - XBLOCK_FIELD_DATA_WRAPPERS -) - ############### Mixed Related(Secure/Not-Secure) Items ########## -LMS_SEGMENT_KEY = AUTH_TOKENS.get('SEGMENT_KEY') - -SECRET_KEY = AUTH_TOKENS['SECRET_KEY'] +LMS_SEGMENT_KEY = _YAML_TOKENS.get('SEGMENT_KEY') -AWS_ACCESS_KEY_ID = AUTH_TOKENS.get("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID) if AWS_ACCESS_KEY_ID == "": AWS_ACCESS_KEY_ID = None - -AWS_SECRET_ACCESS_KEY = AUTH_TOKENS.get("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY) if AWS_SECRET_ACCESS_KEY == "": AWS_SECRET_ACCESS_KEY = None @@ -460,24 +327,10 @@ def get_env_setting(setting): # same with upcoming version setting it to `public-read`. AWS_DEFAULT_ACL = 'public-read' AWS_BUCKET_ACL = AWS_DEFAULT_ACL -AWS_STORAGE_BUCKET_NAME = AUTH_TOKENS.get('AWS_STORAGE_BUCKET_NAME', 'edxuploads') -# Disabling querystring auth instructs Boto to exclude the querystring parameters (e.g. signature, access key) it -# normally appends to every returned URL. -AWS_QUERYSTRING_AUTH = AUTH_TOKENS.get('AWS_QUERYSTRING_AUTH', True) -AWS_S3_CUSTOM_DOMAIN = AUTH_TOKENS.get('AWS_S3_CUSTOM_DOMAIN', 'edxuploads.s3.amazonaws.com') - -if AUTH_TOKENS.get('DEFAULT_FILE_STORAGE'): - DEFAULT_FILE_STORAGE = AUTH_TOKENS.get('DEFAULT_FILE_STORAGE') -elif AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: +# Change to S3Boto3 if we haven't specified another default storage AND we have specified AWS creds. +if (not _YAML_TOKENS.get('DEFAULT_FILE_STORAGE')) and AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY: DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -else: - DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' - - -# If there is a database called 'read_replica', you can use the use_read_replica_if_available -# function in util/query.py, which is useful for very large database reads -DATABASES = AUTH_TOKENS.get('DATABASES', DATABASES) # The normal database user does not have enough permissions to run migrations. # Migrations are run with separate credentials, given as DB_MIGRATION_* @@ -493,11 +346,9 @@ def get_env_setting(setting): 'PORT': os.environ.get('DB_MIGRATION_PORT', database['PORT']), }) -XQUEUE_INTERFACE = AUTH_TOKENS.get('XQUEUE_INTERFACE', XQUEUE_INTERFACE) - # Get the MODULESTORE from auth.json, but if it doesn't exist, # use the one from common.py -MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE)) +MODULESTORE = convert_module_store_setting_if_needed(_YAML_TOKENS.get('MODULESTORE', MODULESTORE)) # After conversion above, the modulestore will have a "stores" list with all defined stores, for all stores, add the # fs_root entry to derived collection so that if it's a callable it can be resolved. We need to do this because the @@ -510,137 +361,34 @@ def get_env_setting(setting): if 'OPTIONS' in store and 'fs_root' in store["OPTIONS"]: derived_collection_entry('MODULESTORE', 'default', 'OPTIONS', 'stores', idx, 'OPTIONS', 'fs_root') -MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {}) - -EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is '' -EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', '') # django default is '' - -# Analytics API -ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", ANALYTICS_API_KEY) -ANALYTICS_API_URL = ENV_TOKENS.get("ANALYTICS_API_URL", ANALYTICS_API_URL) - -# Zendesk -ZENDESK_USER = AUTH_TOKENS.get("ZENDESK_USER") -ZENDESK_API_KEY = AUTH_TOKENS.get("ZENDESK_API_KEY") - -# API Key for inbound requests from Notifier service -EDX_API_KEY = AUTH_TOKENS.get("EDX_API_KEY") - -# Celery Broker -CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") -CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "") -CELERY_BROKER_VHOST = ENV_TOKENS.get("CELERY_BROKER_VHOST", "") -CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "") -CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "") - BROKER_URL = "{}://{}:{}@{}/{}".format(CELERY_BROKER_TRANSPORT, CELERY_BROKER_USER, CELERY_BROKER_PASSWORD, CELERY_BROKER_HOSTNAME, CELERY_BROKER_VHOST) -BROKER_USE_SSL = ENV_TOKENS.get('CELERY_BROKER_USE_SSL', False) - try: BROKER_TRANSPORT_OPTIONS = { 'fanout_patterns': True, 'fanout_prefix': True, - **ENV_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) + **_YAML_TOKENS.get('CELERY_BROKER_TRANSPORT_OPTIONS', {}) } except TypeError as exc: raise ImproperlyConfigured('CELERY_BROKER_TRANSPORT_OPTIONS must be a dict') from exc -# Block Structures - -# upload limits -STUDENT_FILEUPLOAD_MAX_SIZE = ENV_TOKENS.get("STUDENT_FILEUPLOAD_MAX_SIZE", STUDENT_FILEUPLOAD_MAX_SIZE) - # Event tracking -TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {})) -EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( - AUTH_TOKENS.get("EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST", [])) -TRACKING_SEGMENTIO_WEBHOOK_SECRET = AUTH_TOKENS.get( - "TRACKING_SEGMENTIO_WEBHOOK_SECRET", - TRACKING_SEGMENTIO_WEBHOOK_SECRET -) -TRACKING_SEGMENTIO_ALLOWED_TYPES = ENV_TOKENS.get("TRACKING_SEGMENTIO_ALLOWED_TYPES", TRACKING_SEGMENTIO_ALLOWED_TYPES) -TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES = ENV_TOKENS.get( - "TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES", - TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES +TRACKING_BACKENDS.update(_YAML_TOKENS.get("TRACKING_BACKENDS", {})) +EVENT_TRACKING_BACKENDS['tracking_logs']['OPTIONS']['backends'].update( + _YAML_TOKENS.get("EVENT_TRACKING_BACKENDS", {}) ) -TRACKING_SEGMENTIO_SOURCE_MAP = ENV_TOKENS.get("TRACKING_SEGMENTIO_SOURCE_MAP", TRACKING_SEGMENTIO_SOURCE_MAP) - -# Heartbeat -HEARTBEAT_CELERY_ROUTING_KEY = ENV_TOKENS.get('HEARTBEAT_CELERY_ROUTING_KEY', HEARTBEAT_CELERY_ROUTING_KEY) - -# Student identity verification settings -VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT) -DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH = ENV_TOKENS.get( - "DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH", - DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH +EVENT_TRACKING_BACKENDS['segmentio']['OPTIONS']['processors'][0]['OPTIONS']['whitelist'].extend( + EVENT_TRACKING_SEGMENTIO_EMIT_WHITELIST ) # Grades download -GRADES_DOWNLOAD_ROUTING_KEY = ENV_TOKENS.get('GRADES_DOWNLOAD_ROUTING_KEY', HIGH_MEM_QUEUE) - -GRADES_DOWNLOAD = ENV_TOKENS.get("GRADES_DOWNLOAD", GRADES_DOWNLOAD) - -# Rate limit for regrading tasks that a grading policy change can kick off - -# financial reports -FINANCIAL_REPORTS = ENV_TOKENS.get("FINANCIAL_REPORTS", FINANCIAL_REPORTS) - -##### ORA2 ###### -# Prefix for uploads of example-based assessment AI classifiers -# This can be used to separate uploads for different environments -# within the same S3 bucket. -ORA2_FILE_PREFIX = ENV_TOKENS.get("ORA2_FILE_PREFIX", ORA2_FILE_PREFIX) - -##### ACCOUNT LOCKOUT DEFAULT PARAMETERS ##### -MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = ENV_TOKENS.get( - "MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED", MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED -) - -MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = ENV_TOKENS.get( - "MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS", MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS -) - -##### LOGISTRATION RATE LIMIT SETTINGS ##### -LOGISTRATION_RATELIMIT_RATE = ENV_TOKENS.get('LOGISTRATION_RATELIMIT_RATE', LOGISTRATION_RATELIMIT_RATE) -LOGISTRATION_API_RATELIMIT = ENV_TOKENS.get('LOGISTRATION_API_RATELIMIT', LOGISTRATION_API_RATELIMIT) -LOGIN_AND_REGISTER_FORM_RATELIMIT = ENV_TOKENS.get( - 'LOGIN_AND_REGISTER_FORM_RATELIMIT', LOGIN_AND_REGISTER_FORM_RATELIMIT -) -RESET_PASSWORD_TOKEN_VALIDATE_API_RATELIMIT = ENV_TOKENS.get( - 'RESET_PASSWORD_TOKEN_VALIDATE_API_RATELIMIT', RESET_PASSWORD_TOKEN_VALIDATE_API_RATELIMIT -) -RESET_PASSWORD_API_RATELIMIT = ENV_TOKENS.get('RESET_PASSWORD_API_RATELIMIT', RESET_PASSWORD_API_RATELIMIT) - -##### REGISTRATION RATE LIMIT SETTINGS ##### -REGISTRATION_VALIDATION_RATELIMIT = ENV_TOKENS.get( - 'REGISTRATION_VALIDATION_RATELIMIT', REGISTRATION_VALIDATION_RATELIMIT -) - -REGISTRATION_RATELIMIT = ENV_TOKENS.get('REGISTRATION_RATELIMIT', REGISTRATION_RATELIMIT) - -#### PASSWORD POLICY SETTINGS ##### -AUTH_PASSWORD_VALIDATORS = ENV_TOKENS.get("AUTH_PASSWORD_VALIDATORS", AUTH_PASSWORD_VALIDATORS) - -### INACTIVITY SETTINGS #### -SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = AUTH_TOKENS.get("SESSION_INACTIVITY_TIMEOUT_IN_SECONDS") - -##### LMS DEADLINE DISPLAY TIME_ZONE ####### -TIME_ZONE_DISPLAYED_FOR_DEADLINES = ENV_TOKENS.get("TIME_ZONE_DISPLAYED_FOR_DEADLINES", - TIME_ZONE_DISPLAYED_FOR_DEADLINES) - -#### PROCTORED EXAM SETTINGS #### -PROCTORED_EXAM_VIEWABLE_PAST_DUE = ENV_TOKENS.get('PROCTORED_EXAM_VIEWABLE_PAST_DUE', False) - -##### Third-party auth options ################################################ -ENABLE_REQUIRE_THIRD_PARTY_AUTH = ENV_TOKENS.get('ENABLE_REQUIRE_THIRD_PARTY_AUTH', False) +GRADES_DOWNLOAD_ROUTING_KEY = _YAML_TOKENS.get('GRADES_DOWNLOAD_ROUTING_KEY', HIGH_MEM_QUEUE) if FEATURES.get('ENABLE_THIRD_PARTY_AUTH'): - tmp_backends = ENV_TOKENS.get('THIRD_PARTY_AUTH_BACKENDS', [ + AUTHENTICATION_BACKENDS = _YAML_TOKENS.get('THIRD_PARTY_AUTH_BACKENDS', [ 'social_core.backends.google.GoogleOAuth2', 'social_core.backends.linkedin.LinkedinOAuth2', 'social_core.backends.facebook.FacebookOAuth2', @@ -649,136 +397,66 @@ def get_env_setting(setting): 'common.djangoapps.third_party_auth.identityserver3.IdentityServer3', 'common.djangoapps.third_party_auth.saml.SAMLAuthBackend', 'common.djangoapps.third_party_auth.lti.LTIAuthBackend', - ]) - - AUTHENTICATION_BACKENDS = list(tmp_backends) + list(AUTHENTICATION_BACKENDS) - del tmp_backends + ]) + list(AUTHENTICATION_BACKENDS) # The reduced session expiry time during the third party login pipeline. (Value in seconds) - SOCIAL_AUTH_PIPELINE_TIMEOUT = ENV_TOKENS.get('SOCIAL_AUTH_PIPELINE_TIMEOUT', 600) - - # Most provider configuration is done via ConfigurationModels but for a few sensitive values - # we allow configuration via AUTH_TOKENS instead (optionally). - # The SAML private/public key values do not need the delimiter lines (such as - # "-----BEGIN PRIVATE KEY-----", "-----END PRIVATE KEY-----" etc.) but they may be included - # if you want (though it's easier to format the key values as JSON without the delimiters). - SOCIAL_AUTH_SAML_SP_PRIVATE_KEY = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PRIVATE_KEY', '') - SOCIAL_AUTH_SAML_SP_PUBLIC_CERT = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PUBLIC_CERT', '') - SOCIAL_AUTH_SAML_SP_PRIVATE_KEY_DICT = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PRIVATE_KEY_DICT', {}) - SOCIAL_AUTH_SAML_SP_PUBLIC_CERT_DICT = AUTH_TOKENS.get('SOCIAL_AUTH_SAML_SP_PUBLIC_CERT_DICT', {}) - SOCIAL_AUTH_OAUTH_SECRETS = AUTH_TOKENS.get('SOCIAL_AUTH_OAUTH_SECRETS', {}) - SOCIAL_AUTH_LTI_CONSUMER_SECRETS = AUTH_TOKENS.get('SOCIAL_AUTH_LTI_CONSUMER_SECRETS', {}) + SOCIAL_AUTH_PIPELINE_TIMEOUT = _YAML_TOKENS.get('SOCIAL_AUTH_PIPELINE_TIMEOUT', 600) + + # TODO: Would it be safe to just set this default in common.py, even if ENABLE_THIRD_PARTY_AUTH is False? + SOCIAL_AUTH_LTI_CONSUMER_SECRETS = _YAML_TOKENS.get('SOCIAL_AUTH_LTI_CONSUMER_SECRETS', {}) # third_party_auth config moved to ConfigurationModels. This is for data migration only: - THIRD_PARTY_AUTH_OLD_CONFIG = AUTH_TOKENS.get('THIRD_PARTY_AUTH', None) + THIRD_PARTY_AUTH_OLD_CONFIG = _YAML_TOKENS.get('THIRD_PARTY_AUTH', None) - if ENV_TOKENS.get('THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS', 24) is not None: + # TODO: This logic is somewhat insane. We're not sure if it's intentional or not. We've left it + # as-is for strict backwards compatibility, but it's worth revisiting. + if hours := _YAML_TOKENS.get('THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS', 24): + # If we didn't override the value in YAML, OR we overrode it to a truthy value, + # then update CELERYBEAT_SCHEDULE. CELERYBEAT_SCHEDULE['refresh-saml-metadata'] = { 'task': 'common.djangoapps.third_party_auth.fetch_saml_metadata', - 'schedule': datetime.timedelta(hours=ENV_TOKENS.get('THIRD_PARTY_AUTH_SAML_FETCH_PERIOD_HOURS', 24)), + 'schedule': datetime.timedelta(hours=hours), } # The following can be used to integrate a custom login form with third_party_auth. # It should be a dict where the key is a word passed via ?auth_entry=, and the value is a # dict with an arbitrary 'secret_key' and a 'url'. - THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS = AUTH_TOKENS.get('THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS', {}) + THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS = _YAML_TOKENS.get('THIRD_PARTY_AUTH_CUSTOM_AUTH_FORMS', {}) ##### OAUTH2 Provider ############## -if FEATURES.get('ENABLE_OAUTH2_PROVIDER'): - OAUTH_ENFORCE_SECURE = ENV_TOKENS.get('OAUTH_ENFORCE_SECURE', True) - OAUTH_ENFORCE_CLIENT_SECURE = ENV_TOKENS.get('OAUTH_ENFORCE_CLIENT_SECURE', True) +if FEATURES['ENABLE_OAUTH2_PROVIDER']: + OAUTH_ENFORCE_SECURE = True + OAUTH_ENFORCE_CLIENT_SECURE = True # Defaults for the following are defined in lms.envs.common - OAUTH_EXPIRE_DELTA = datetime.timedelta( - days=ENV_TOKENS.get('OAUTH_EXPIRE_CONFIDENTIAL_CLIENT_DAYS', OAUTH_EXPIRE_CONFIDENTIAL_CLIENT_DAYS) - ) - OAUTH_EXPIRE_DELTA_PUBLIC = datetime.timedelta( - days=ENV_TOKENS.get('OAUTH_EXPIRE_PUBLIC_CLIENT_DAYS', OAUTH_EXPIRE_PUBLIC_CLIENT_DAYS) - ) - - -##### GOOGLE ANALYTICS IDS ##### -GOOGLE_ANALYTICS_ACCOUNT = AUTH_TOKENS.get('GOOGLE_ANALYTICS_ACCOUNT') -GOOGLE_ANALYTICS_TRACKING_ID = AUTH_TOKENS.get('GOOGLE_ANALYTICS_TRACKING_ID') -GOOGLE_ANALYTICS_LINKEDIN = AUTH_TOKENS.get('GOOGLE_ANALYTICS_LINKEDIN') -GOOGLE_SITE_VERIFICATION_ID = ENV_TOKENS.get('GOOGLE_SITE_VERIFICATION_ID') -GOOGLE_ANALYTICS_4_ID = AUTH_TOKENS.get('GOOGLE_ANALYTICS_4_ID') - -##### BRANCH.IO KEY ##### -BRANCH_IO_KEY = AUTH_TOKENS.get('BRANCH_IO_KEY') - -#### Course Registration Code length #### -REGISTRATION_CODE_LENGTH = ENV_TOKENS.get('REGISTRATION_CODE_LENGTH', 8) - -# Which access.py permission names to check; -# We default this to the legacy permission 'see_exists'. -COURSE_CATALOG_VISIBILITY_PERMISSION = ENV_TOKENS.get( - 'COURSE_CATALOG_VISIBILITY_PERMISSION', - COURSE_CATALOG_VISIBILITY_PERMISSION -) -COURSE_ABOUT_VISIBILITY_PERMISSION = ENV_TOKENS.get( - 'COURSE_ABOUT_VISIBILITY_PERMISSION', - COURSE_ABOUT_VISIBILITY_PERMISSION -) - -DEFAULT_COURSE_VISIBILITY_IN_CATALOG = ENV_TOKENS.get( - 'DEFAULT_COURSE_VISIBILITY_IN_CATALOG', - DEFAULT_COURSE_VISIBILITY_IN_CATALOG -) - -DEFAULT_MOBILE_AVAILABLE = ENV_TOKENS.get( - 'DEFAULT_MOBILE_AVAILABLE', - DEFAULT_MOBILE_AVAILABLE -) - -# Enrollment API Cache Timeout -ENROLLMENT_COURSE_DETAILS_CACHE_TIMEOUT = ENV_TOKENS.get('ENROLLMENT_COURSE_DETAILS_CACHE_TIMEOUT', 60) - -# Ecommerce Orders API Cache Timeout -ECOMMERCE_ORDERS_API_CACHE_TIMEOUT = ENV_TOKENS.get('ECOMMERCE_ORDERS_API_CACHE_TIMEOUT', 3600) - -if FEATURES.get('ENABLE_COURSEWARE_SEARCH') or \ - FEATURES.get('ENABLE_DASHBOARD_SEARCH') or \ - FEATURES.get('ENABLE_COURSE_DISCOVERY') or \ - FEATURES.get('ENABLE_TEAMS'): + OAUTH_EXPIRE_DELTA = datetime.timedelta(days=OAUTH_EXPIRE_CONFIDENTIAL_CLIENT_DAYS) + OAUTH_EXPIRE_DELTA_PUBLIC = datetime.timedelta(days=OAUTH_EXPIRE_PUBLIC_CLIENT_DAYS) + +if ( + FEATURES['ENABLE_COURSEWARE_SEARCH'] or + FEATURES['ENABLE_DASHBOARD_SEARCH'] or + FEATURES['ENABLE_COURSE_DISCOVERY'] or + FEATURES['ENABLE_TEAMS'] + ): # Use ElasticSearch as the search engine herein SEARCH_ENGINE = "search.elastic.ElasticSearchEngine" - SEARCH_FILTER_GENERATOR = ENV_TOKENS.get('SEARCH_FILTER_GENERATOR', SEARCH_FILTER_GENERATOR) - -SEARCH_SKIP_INVITATION_ONLY_FILTERING = ENV_TOKENS.get( - 'SEARCH_SKIP_INVITATION_ONLY_FILTERING', - SEARCH_SKIP_INVITATION_ONLY_FILTERING, -) -SEARCH_SKIP_SHOW_IN_CATALOG_FILTERING = ENV_TOKENS.get( - 'SEARCH_SKIP_SHOW_IN_CATALOG_FILTERING', - SEARCH_SKIP_SHOW_IN_CATALOG_FILTERING, -) - -SEARCH_COURSEWARE_CONTENT_LOG_PARAMS = ENV_TOKENS.get( - 'SEARCH_COURSEWARE_CONTENT_LOG_PARAMS', - SEARCH_COURSEWARE_CONTENT_LOG_PARAMS, -) # TODO: Once we have successfully upgraded to ES7, switch this back to ELASTIC_SEARCH_CONFIG. -ELASTIC_SEARCH_CONFIG = ENV_TOKENS.get('ELASTIC_SEARCH_CONFIG_ES7', [{}]) - -# Facebook app -FACEBOOK_API_VERSION = AUTH_TOKENS.get("FACEBOOK_API_VERSION") -FACEBOOK_APP_SECRET = AUTH_TOKENS.get("FACEBOOK_APP_SECRET") -FACEBOOK_APP_ID = AUTH_TOKENS.get("FACEBOOK_APP_ID") +ELASTIC_SEARCH_CONFIG = _YAML_TOKENS.get('ELASTIC_SEARCH_CONFIG_ES7', [{}]) -XBLOCK_SETTINGS = ENV_TOKENS.get('XBLOCK_SETTINGS', {}) -XBLOCK_SETTINGS.setdefault("VideoBlock", {})["licensing_enabled"] = FEATURES.get("LICENSING", False) -XBLOCK_SETTINGS.setdefault("VideoBlock", {})['YOUTUBE_API_KEY'] = AUTH_TOKENS.get('YOUTUBE_API_KEY', YOUTUBE_API_KEY) +XBLOCK_SETTINGS.setdefault("VideoBlock", {})["licensing_enabled"] = FEATURES["LICENSING"] +XBLOCK_SETTINGS.setdefault("VideoBlock", {})['YOUTUBE_API_KEY'] = YOUTUBE_API_KEY ##### Custom Courses for EdX ##### -if FEATURES.get('CUSTOM_COURSES_EDX'): +if FEATURES['CUSTOM_COURSES_EDX']: INSTALLED_APPS += ['lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig'] MODULESTORE_FIELD_OVERRIDE_PROVIDERS += ( 'lms.djangoapps.ccx.overrides.CustomCoursesForEdxOverrideProvider', ) +FIELD_OVERRIDE_PROVIDERS = tuple(FIELD_OVERRIDE_PROVIDERS) + ##### Individual Due Date Extensions ##### -if FEATURES.get('INDIVIDUAL_DUE_DATES'): +if FEATURES['INDIVIDUAL_DUE_DATES']: FIELD_OVERRIDE_PROVIDERS += ( 'lms.djangoapps.courseware.student_field_overrides.IndividualStudentOverrideProvider', ) @@ -799,52 +477,30 @@ def get_env_setting(setting): # PROFILE IMAGE CONFIG PROFILE_IMAGE_DEFAULT_FILENAME = 'images/profiles/default' -PROFILE_IMAGE_SIZES_MAP = ENV_TOKENS.get( - 'PROFILE_IMAGE_SIZES_MAP', - PROFILE_IMAGE_SIZES_MAP -) ##### Credit Provider Integration ##### -CREDIT_PROVIDER_SECRET_KEYS = AUTH_TOKENS.get("CREDIT_PROVIDER_SECRET_KEYS", {}) - ##################### LTI Provider ##################### -if FEATURES.get('ENABLE_LTI_PROVIDER'): +if FEATURES['ENABLE_LTI_PROVIDER']: INSTALLED_APPS.append('lms.djangoapps.lti_provider.apps.LtiProviderConfig') AUTHENTICATION_BACKENDS.append('lms.djangoapps.lti_provider.users.LtiBackend') -LTI_USER_EMAIL_DOMAIN = ENV_TOKENS.get('LTI_USER_EMAIL_DOMAIN', 'lti.example.com') - -# For more info on this, see the notes in common.py -LTI_AGGREGATE_SCORE_PASSBACK_DELAY = ENV_TOKENS.get( - 'LTI_AGGREGATE_SCORE_PASSBACK_DELAY', LTI_AGGREGATE_SCORE_PASSBACK_DELAY -) - ##################### Credit Provider help link #################### #### JWT configuration #### -JWT_AUTH.update(ENV_TOKENS.get('JWT_AUTH', {})) -JWT_AUTH.update(AUTH_TOKENS.get('JWT_AUTH', {})) - -# Offset for pk of courseware.StudentModuleHistoryExtended -STUDENTMODULEHISTORYEXTENDED_OFFSET = ENV_TOKENS.get( - 'STUDENTMODULEHISTORYEXTENDED_OFFSET', STUDENTMODULEHISTORYEXTENDED_OFFSET -) +JWT_AUTH.update(_YAML_TOKENS.get('JWT_AUTH', {})) ################################ Settings for Credentials Service ################################ -CREDENTIALS_GENERATION_ROUTING_KEY = ENV_TOKENS.get('CREDENTIALS_GENERATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) +CREDENTIALS_GENERATION_ROUTING_KEY = _YAML_TOKENS.get('CREDENTIALS_GENERATION_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) -# Queue to use for award program certificates -PROGRAM_CERTIFICATES_ROUTING_KEY = ENV_TOKENS.get('PROGRAM_CERTIFICATES_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) -SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = ENV_TOKENS.get( +PROGRAM_CERTIFICATES_ROUTING_KEY = _YAML_TOKENS.get('PROGRAM_CERTIFICATES_ROUTING_KEY', DEFAULT_PRIORITY_QUEUE) + +SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = _YAML_TOKENS.get( 'SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY', HIGH_PRIORITY_QUEUE ) -API_ACCESS_MANAGER_EMAIL = ENV_TOKENS.get('API_ACCESS_MANAGER_EMAIL') -API_ACCESS_FROM_EMAIL = ENV_TOKENS.get('API_ACCESS_FROM_EMAIL') - ############## OPEN EDX ENTERPRISE SERVICE CONFIGURATION ###################### # The Open edX Enterprise service is currently hosted via the LMS container/process. # However, for all intents and purposes this service is treated as a standalone IDA. @@ -852,47 +508,17 @@ def get_env_setting(setting): # not find references to them within the edx-platform project. # Publicly-accessible enrollment URL, for use on the client side. -ENTERPRISE_PUBLIC_ENROLLMENT_API_URL = ENV_TOKENS.get( +ENTERPRISE_PUBLIC_ENROLLMENT_API_URL = _YAML_TOKENS.get( 'ENTERPRISE_PUBLIC_ENROLLMENT_API_URL', (LMS_ROOT_URL or '') + LMS_ENROLLMENT_API_PATH ) # Enrollment URL used on the server-side. -ENTERPRISE_ENROLLMENT_API_URL = ENV_TOKENS.get( +ENTERPRISE_ENROLLMENT_API_URL = _YAML_TOKENS.get( 'ENTERPRISE_ENROLLMENT_API_URL', (LMS_INTERNAL_ROOT_URL or '') + LMS_ENROLLMENT_API_PATH ) -# Enterprise logo image size limit in KB's -ENTERPRISE_CUSTOMER_LOGO_IMAGE_SIZE = ENV_TOKENS.get( - 'ENTERPRISE_CUSTOMER_LOGO_IMAGE_SIZE', - ENTERPRISE_CUSTOMER_LOGO_IMAGE_SIZE -) - -# Course enrollment modes to be hidden in the Enterprise enrollment page -# if the "Hide audit track" flag is enabled for an EnterpriseCustomer -ENTERPRISE_COURSE_ENROLLMENT_AUDIT_MODES = ENV_TOKENS.get( - 'ENTERPRISE_COURSE_ENROLLMENT_AUDIT_MODES', - ENTERPRISE_COURSE_ENROLLMENT_AUDIT_MODES -) - -# A support URL used on Enterprise landing pages for when a warning -# message goes off. -ENTERPRISE_SUPPORT_URL = ENV_TOKENS.get( - 'ENTERPRISE_SUPPORT_URL', - ENTERPRISE_SUPPORT_URL -) - -# A default dictionary to be used for filtering out enterprise customer catalog. -ENTERPRISE_CUSTOMER_CATALOG_DEFAULT_CONTENT_FILTER = ENV_TOKENS.get( - 'ENTERPRISE_CUSTOMER_CATALOG_DEFAULT_CONTENT_FILTER', - ENTERPRISE_CUSTOMER_CATALOG_DEFAULT_CONTENT_FILTER -) -INTEGRATED_CHANNELS_API_CHUNK_TRANSMISSION_LIMIT = ENV_TOKENS.get( - 'INTEGRATED_CHANNELS_API_CHUNK_TRANSMISSION_LIMIT', - INTEGRATED_CHANNELS_API_CHUNK_TRANSMISSION_LIMIT -) - ############## ENTERPRISE SERVICE API CLIENT CONFIGURATION ###################### # The LMS communicates with the Enterprise service via the requests.Session() client # The below environmental settings are utilized by the LMS when interacting with @@ -901,97 +527,24 @@ def get_env_setting(setting): DEFAULT_ENTERPRISE_API_URL = None if LMS_INTERNAL_ROOT_URL is not None: DEFAULT_ENTERPRISE_API_URL = LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/' -ENTERPRISE_API_URL = ENV_TOKENS.get('ENTERPRISE_API_URL', DEFAULT_ENTERPRISE_API_URL) +ENTERPRISE_API_URL = _YAML_TOKENS.get('ENTERPRISE_API_URL', DEFAULT_ENTERPRISE_API_URL) DEFAULT_ENTERPRISE_CONSENT_API_URL = None if LMS_INTERNAL_ROOT_URL is not None: DEFAULT_ENTERPRISE_CONSENT_API_URL = LMS_INTERNAL_ROOT_URL + '/consent/api/v1/' -ENTERPRISE_CONSENT_API_URL = ENV_TOKENS.get('ENTERPRISE_CONSENT_API_URL', DEFAULT_ENTERPRISE_CONSENT_API_URL) - -ENTERPRISE_SERVICE_WORKER_USERNAME = ENV_TOKENS.get( - 'ENTERPRISE_SERVICE_WORKER_USERNAME', - ENTERPRISE_SERVICE_WORKER_USERNAME -) -ENTERPRISE_API_CACHE_TIMEOUT = ENV_TOKENS.get( - 'ENTERPRISE_API_CACHE_TIMEOUT', - ENTERPRISE_API_CACHE_TIMEOUT -) -ENTERPRISE_CATALOG_INTERNAL_ROOT_URL = ENV_TOKENS.get( - 'ENTERPRISE_CATALOG_INTERNAL_ROOT_URL', - ENTERPRISE_CATALOG_INTERNAL_ROOT_URL -) - -CHAT_COMPLETION_API = ENV_TOKENS.get('CHAT_COMPLETION_API', '') -CHAT_COMPLETION_API_KEY = ENV_TOKENS.get('CHAT_COMPLETION_API_KEY', '') -LEARNER_ENGAGEMENT_PROMPT_FOR_ACTIVE_CONTRACT = ENV_TOKENS.get('LEARNER_ENGAGEMENT_PROMPT_FOR_ACTIVE_CONTRACT', '') -LEARNER_ENGAGEMENT_PROMPT_FOR_NON_ACTIVE_CONTRACT = ENV_TOKENS.get( - 'LEARNER_ENGAGEMENT_PROMPT_FOR_NON_ACTIVE_CONTRACT', - '' -) -LEARNER_PROGRESS_PROMPT_FOR_ACTIVE_CONTRACT = ENV_TOKENS.get('LEARNER_PROGRESS_PROMPT_FOR_ACTIVE_CONTRACT', '') -LEARNER_PROGRESS_PROMPT_FOR_NON_ACTIVE_CONTRACT = ENV_TOKENS.get('LEARNER_PROGRESS_PROMPT_FOR_NON_ACTIVE_CONTRACT', '') +ENTERPRISE_CONSENT_API_URL = _YAML_TOKENS.get('ENTERPRISE_CONSENT_API_URL', DEFAULT_ENTERPRISE_CONSENT_API_URL) ############## ENTERPRISE SERVICE LMS CONFIGURATION ################################## # The LMS has some features embedded that are related to the Enterprise service, but # which are not provided by the Enterprise service. These settings override the # base values for the parameters as defined in common.py -ENTERPRISE_PLATFORM_WELCOME_TEMPLATE = ENV_TOKENS.get( - 'ENTERPRISE_PLATFORM_WELCOME_TEMPLATE', - ENTERPRISE_PLATFORM_WELCOME_TEMPLATE -) -ENTERPRISE_SPECIFIC_BRANDED_WELCOME_TEMPLATE = ENV_TOKENS.get( - 'ENTERPRISE_SPECIFIC_BRANDED_WELCOME_TEMPLATE', - ENTERPRISE_SPECIFIC_BRANDED_WELCOME_TEMPLATE -) -ENTERPRISE_TAGLINE = ENV_TOKENS.get( - 'ENTERPRISE_TAGLINE', - ENTERPRISE_TAGLINE -) -ENTERPRISE_EXCLUDED_REGISTRATION_FIELDS = set( - ENV_TOKENS.get( - 'ENTERPRISE_EXCLUDED_REGISTRATION_FIELDS', - ENTERPRISE_EXCLUDED_REGISTRATION_FIELDS - ) -) -BASE_COOKIE_DOMAIN = ENV_TOKENS.get( - 'BASE_COOKIE_DOMAIN', - BASE_COOKIE_DOMAIN -) -SYSTEM_TO_FEATURE_ROLE_MAPPING = ENV_TOKENS.get( - 'SYSTEM_TO_FEATURE_ROLE_MAPPING', - SYSTEM_TO_FEATURE_ROLE_MAPPING -) - -# Add an ICP license for serving content in China if your organization is registered to do so -ICP_LICENSE = ENV_TOKENS.get('ICP_LICENSE', None) -ICP_LICENSE_INFO = ENV_TOKENS.get('ICP_LICENSE_INFO', {}) - -# How long to cache OpenAPI schemas and UI, in seconds. -OPENAPI_CACHE_TIMEOUT = ENV_TOKENS.get('OPENAPI_CACHE_TIMEOUT', 60 * 60) - -########################## Parental controls config ####################### - -# The age at which a learner no longer requires parental consent, or None -# if parental consent is never required. -PARENTAL_CONSENT_AGE_LIMIT = ENV_TOKENS.get( - 'PARENTAL_CONSENT_AGE_LIMIT', - PARENTAL_CONSENT_AGE_LIMIT -) +ENTERPRISE_EXCLUDED_REGISTRATION_FIELDS = set(ENTERPRISE_EXCLUDED_REGISTRATION_FIELDS) ########################## Extra middleware classes ####################### # Allow extra middleware classes to be added to the app through configuration. -MIDDLEWARE.extend(ENV_TOKENS.get('EXTRA_MIDDLEWARE_CLASSES', [])) - -################# Settings for the maintenance banner ################# -MAINTENANCE_BANNER_TEXT = ENV_TOKENS.get('MAINTENANCE_BANNER_TEXT', None) - -########################## limiting dashboard courses ###################### -DASHBOARD_COURSE_LIMIT = ENV_TOKENS.get('DASHBOARD_COURSE_LIMIT', None) - -######################## Setting for content libraries ######################## -MAX_BLOCKS_PER_CONTENT_LIBRARY = ENV_TOKENS.get('MAX_BLOCKS_PER_CONTENT_LIBRARY', MAX_BLOCKS_PER_CONTENT_LIBRARY) +MIDDLEWARE.extend(_YAML_TOKENS.get('EXTRA_MIDDLEWARE_CLASSES', [])) ########################## Derive Any Derived Settings ####################### @@ -1001,23 +554,15 @@ def get_env_setting(setting): # This is at the bottom because it is going to load more settings after base settings are loaded +# ENV_TOKENS and AUTH_TOKENS are included for reverse compatibility. +# Removing them may break plugins that rely on them. +# Please do not add new references to them... just use `django.conf.settings` instead. +ENV_TOKENS = __config__ +AUTH_TOKENS = __config__ + # Load production.py in plugins add_plugins(__name__, ProjectType.LMS, SettingsType.PRODUCTION) -############## Settings for Completion API ######################### - -# Once a user has watched this percentage of a video, mark it as complete: -# (0.0 = 0%, 1.0 = 100%) -COMPLETION_VIDEO_COMPLETE_PERCENTAGE = ENV_TOKENS.get('COMPLETION_VIDEO_COMPLETE_PERCENTAGE', - COMPLETION_VIDEO_COMPLETE_PERCENTAGE) -COMPLETION_BY_VIEWING_DELAY_MS = ENV_TOKENS.get('COMPLETION_BY_VIEWING_DELAY_MS', - COMPLETION_BY_VIEWING_DELAY_MS) - -################# Settings for brand logos. ################# -LOGO_URL = ENV_TOKENS.get('LOGO_URL', LOGO_URL) -LOGO_URL_PNG = ENV_TOKENS.get('LOGO_URL_PNG', LOGO_URL_PNG) -LOGO_TRADEMARK_URL = ENV_TOKENS.get('LOGO_TRADEMARK_URL', LOGO_TRADEMARK_URL) -FAVICON_URL = ENV_TOKENS.get('FAVICON_URL', FAVICON_URL) ######################## CELERY ROUTING ######################## @@ -1077,56 +622,32 @@ def get_env_setting(setting): } -LOGO_IMAGE_EXTRA_TEXT = ENV_TOKENS.get('LOGO_IMAGE_EXTRA_TEXT', '') - ############## XBlock extra mixins ############################ XBLOCK_MIXINS += tuple(XBLOCK_EXTRA_MIXINS) -############## Settings for course import olx validation ############################ -COURSE_OLX_VALIDATION_STAGE = ENV_TOKENS.get('COURSE_OLX_VALIDATION_STAGE', COURSE_OLX_VALIDATION_STAGE) -COURSE_OLX_VALIDATION_IGNORE_LIST = ENV_TOKENS.get( - 'COURSE_OLX_VALIDATION_IGNORE_LIST', - COURSE_OLX_VALIDATION_IGNORE_LIST -) - -################# show account activate cta after register ######################## -SHOW_ACCOUNT_ACTIVATION_CTA = ENV_TOKENS.get('SHOW_ACCOUNT_ACTIVATION_CTA', SHOW_ACCOUNT_ACTIVATION_CTA) - -################# Discussions micro frontend URL ######################## -DISCUSSIONS_MICROFRONTEND_URL = ENV_TOKENS.get('DISCUSSIONS_MICROFRONTEND_URL', DISCUSSIONS_MICROFRONTEND_URL) - -################### Discussions micro frontend Feedback URL################### -DISCUSSIONS_MFE_FEEDBACK_URL = ENV_TOKENS.get('DISCUSSIONS_MFE_FEEDBACK_URL', DISCUSSIONS_MFE_FEEDBACK_URL) - -############################ AI_TRANSLATIONS URL ################################## -AI_TRANSLATIONS_API_URL = ENV_TOKENS.get('AI_TRANSLATIONS_API_URL', AI_TRANSLATIONS_API_URL) - ############## DRF overrides ############## -REST_FRAMEWORK.update(ENV_TOKENS.get('REST_FRAMEWORK', {})) +REST_FRAMEWORK.update(_YAML_TOKENS.get('REST_FRAMEWORK', {})) ############################# CELERY ############################ -CELERY_IMPORTS.extend(ENV_TOKENS.get('CELERY_EXTRA_IMPORTS', [])) +CELERY_IMPORTS.extend(_YAML_TOKENS.get('CELERY_EXTRA_IMPORTS', [])) # keys for big blue button live provider +# TODO: This should not be in the core platform. If it has to stay for now, though, then we should move these +# defaults into common.py COURSE_LIVE_GLOBAL_CREDENTIALS["BIG_BLUE_BUTTON"] = { - "KEY": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_KEY', None), - "SECRET": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_SECRET', None), - "URL": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_URL', None), + "KEY": _YAML_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_KEY'), + "SECRET": _YAML_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_SECRET'), + "URL": _YAML_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_URL'), } -AVAILABLE_DISCUSSION_TOURS = ENV_TOKENS.get('AVAILABLE_DISCUSSION_TOURS', []) - -############## NOTIFICATIONS EXPIRY ############## -NOTIFICATIONS_EXPIRY = ENV_TOKENS.get('NOTIFICATIONS_EXPIRY', NOTIFICATIONS_EXPIRY) - ############## Event bus producer ############## -EVENT_BUS_PRODUCER_CONFIG = merge_producer_configs(EVENT_BUS_PRODUCER_CONFIG, - ENV_TOKENS.get('EVENT_BUS_PRODUCER_CONFIG', {})) -BEAMER_PRODUCT_ID = ENV_TOKENS.get('BEAMER_PRODUCT_ID', BEAMER_PRODUCT_ID) - -# .. setting_name: DISABLED_COUNTRIES -# .. setting_default: [] -# .. setting_description: List of country codes that should be disabled -# .. for now it wil impact country listing in auth flow and user profile. -# .. eg ['US', 'CA'] -DISABLED_COUNTRIES = ENV_TOKENS.get('DISABLED_COUNTRIES', []) +EVENT_BUS_PRODUCER_CONFIG = merge_producer_configs( + EVENT_BUS_PRODUCER_CONFIG, + _YAML_TOKENS.get('EVENT_BUS_PRODUCER_CONFIG', {}) +) + +##################################################################################################### +# HEY! Don't add anything to the end of this file. +# Add your defaults to common.py instead! +# If you really need to add post-YAML logic, add it above the "Derive Any Derived Settings" section. +###################################################################################################### From 4449f43c4e63c2b8fa22d0e6a6583b71354f7d06 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Fri, 17 Jan 2025 14:40:02 -0500 Subject: [PATCH 27/30] fix: SHARED_kCOOKIE_DOMAIN -> SHARED_COOKIE_DOMAIN --- lms/envs/production.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/envs/production.py b/lms/envs/production.py index 3ea1c952e479..cbe10b180af2 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -219,7 +219,7 @@ def get_env_setting(setting): # This is the domain that is used to set shared cookies between various sub-domains. # By default, it's set to the same thing as the SESSION_COOKIE_DOMAIN, but we want to make it overrideable. -SHARED_COOKIE_DOMAIN = _YAML_TOKENS.get('SHARED_kCOOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) +SHARED_COOKIE_DOMAIN = _YAML_TOKENS.get('SHARED_COOKIE_DOMAIN', SESSION_COOKIE_DOMAIN) # Cache used for location mapping -- called many times with the same key/value # in a given request. From 0ea4bae7b32cf373dce5def32f25e88f02061000 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Tue, 28 Jan 2025 15:15:21 -0500 Subject: [PATCH 28/30] fix: Exactly preserve legacy settings dicts; rm KEYS_WITH_MERGED_VALUES In the near term, we wish to precisely preserve the existing values of all Django settings exposed by lms/envs/production.py in order to avoid breaking legacy Django plugins without a proper announcement. That includes preserving the behavior of these old, redundant dicts: * ENV_TOKENS * AUTH_TOKENS * ENV_FEATURES * ENV_CELERY_QUEUES * ALTERNATE_QUEUE_ENVS Particularly, it means we need to ensure that updates to Django settings are reflected in these dicts. The most reliable way to do that is to change the yaml-loading logic so that these values are aliased to the corresponding values in the global namespace rather than deep-copied. Finally, we remove KEYS_WITH_MERGED_VALUES from the global namespace, and inline the remaining list. We have modified the list (specifically, we dropped the no-op MKTG_URL_OVERRIDES). Plugins should not be counting on the value of the list, so we remove it. --- lms/envs/production.py | 76 ++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/lms/envs/production.py b/lms/envs/production.py index cbe10b180af2..ed705ad14bc7 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -17,7 +17,6 @@ import codecs -import copy import datetime import os @@ -128,40 +127,40 @@ def get_env_setting(setting): ####################################################################################################################### -# A file path to a YAML file from which to load all the configuration for the edx platform +# A file path to a YAML file from which to load configuration overrides for LMS. CONFIG_FILE = get_env_setting('LMS_CFG') with codecs.open(CONFIG_FILE, encoding='utf-8') as f: - __config__ = yaml.safe_load(f) - - # _YAML_TOKENS contains the exact contents of the LMS_CFG YAML file. - # We do splat the entirety of the LMS_CFG YAML file (except KEYS_WITH_MERGED_VALUES) into this module. - # However, for precise backwards compatibility, we need to reference _YAML_TOKENS directly a few times, - # particularly we need to derive Django setting values from YAML values. - # This pattern is confusing and we discourage it. Rather than adding more _YAML_TOKENS references, please - # consider just referencing this module's variables directly. - _YAML_TOKENS = __config__ - - # Add the key/values from config into the global namespace of this module. - # But don't override the FEATURES dict because we do that in an additive way. - __config_copy__ = copy.deepcopy(__config__) - - KEYS_WITH_MERGED_VALUES = [ - 'FEATURES', - 'TRACKING_BACKENDS', - 'EVENT_TRACKING_BACKENDS', - 'JWT_AUTH', - 'CELERY_QUEUES', - 'MKTG_URL_LINK_MAP', - 'REST_FRAMEWORK', - 'EVENT_BUS_PRODUCER_CONFIG', - ] - for key in KEYS_WITH_MERGED_VALUES: - if key in __config_copy__: - del __config_copy__[key] - - vars().update(__config_copy__) + # _YAML_TOKENS starts out with the exact contents of the LMS_CFG YAML file. + # Please avoid adding new references to _YAML_TOKENS. Such references make our settings logic more complex. + # Instead, just reference the Django settings, which we define in the next step... + _YAML_TOKENS = yaml.safe_load(f) + + # Update the global namespace of this module with the key-value pairs from _YAML_TOKENS. + # In other words: For (almost) every YAML key-value pair, define/update a Django setting with that name and value. + vars().update({ + + # Note: If `value` is a mutable object (e.g., a dict), then it will be aliased between the global namespace and + # _YAML_TOKENS. In other words, updates to `value` will manifest in _YAML_TOKENS as well. This is intentional, + # in order to maintain backwards compatibility with old Django plugins which use _YAML_TOKENS. + key: value + for key, value in _YAML_TOKENS.items() + + # Do NOT define/update Django settings for these particular special keys. + # We handle each of these with its special logic (below, in this same module). + # For example, we need to *update* the default FEATURES dict rather than wholesale *override* it. + if key not in [ + 'FEATURES', + 'TRACKING_BACKENDS', + 'EVENT_TRACKING_BACKENDS', + 'JWT_AUTH', + 'CELERY_QUEUES', + 'MKTG_URL_LINK_MAP', + 'REST_FRAMEWORK', + 'EVENT_BUS_PRODUCER_CONFIG', + ] + }) try: # A file path to a YAML file from which to load all the code revisions currently deployed @@ -554,11 +553,16 @@ def get_env_setting(setting): # This is at the bottom because it is going to load more settings after base settings are loaded -# ENV_TOKENS and AUTH_TOKENS are included for reverse compatibility. -# Removing them may break plugins that rely on them. -# Please do not add new references to them... just use `django.conf.settings` instead. -ENV_TOKENS = __config__ -AUTH_TOKENS = __config__ +# These dicts are defined solely for BACKWARDS COMPATIBILITY with existing plugins which may theoretically +# rely upon them. Please do not add new references to these dicts! +# - If you need to access the YAML values in this module, use _YAML_TOKENS. +# - If you need to access to these values elsewhere, use the corresponding rendered `settings.*` +# value rathering than diving into these dicts. +ENV_TOKENS = _YAML_TOKENS +AUTH_TOKENS = _YAML_TOKENS +ENV_FEATURES = _YAML_TOKENS.get("FEATURES", {}) +ENV_CELERY_QUEUES = _YAML_CELERY_QUEUES +ALTERNATE_QUEUE_ENVS = _YAML_ALTERNATE_WORKER_QUEUES # Load production.py in plugins add_plugins(__name__, ProjectType.LMS, SettingsType.PRODUCTION) From 5e51e2d5b75d4caf3b23b704e9b3de4b1885f413 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Thu, 30 Jan 2025 20:32:18 +0530 Subject: [PATCH 29/30] fix: swagger docs ref_name conflicts (#36189) * fix: swagger docs ref_name conflicts * fix: swagger auto doc errors * chore: bumps openedx-learning==0.18.2 --------- Co-authored-by: Jillian Vogel --- cms/djangoapps/api/v1/views/course_runs.py | 1 + .../contentstore/api/views/course_import.py | 2 +- .../contentstore/api/views/course_quality.py | 5 +++++ .../api/views/course_validation.py | 5 +++++ .../v0/serializers/authoring_grading.py | 6 ++++++ .../contentstore/rest_api/v0/urls.py | 2 +- .../rest_api/v0/views/authoring_videos.py | 10 +++++++++ cms/urls.py | 2 +- .../djangoapps/content_libraries/views.py | 21 +++++++++++++++++++ requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 14 files changed, 56 insertions(+), 8 deletions(-) diff --git a/cms/djangoapps/api/v1/views/course_runs.py b/cms/djangoapps/api/v1/views/course_runs.py index b405207bd9c6..7b27193d175f 100644 --- a/cms/djangoapps/api/v1/views/course_runs.py +++ b/cms/djangoapps/api/v1/views/course_runs.py @@ -23,6 +23,7 @@ class CourseRunViewSet(viewsets.GenericViewSet): # lint-amnesty, pylint: disabl lookup_value_regex = settings.COURSE_KEY_REGEX permission_classes = (permissions.IsAdminUser,) serializer_class = CourseRunSerializer + queryset = [] def get_object(self): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field diff --git a/cms/djangoapps/contentstore/api/views/course_import.py b/cms/djangoapps/contentstore/api/views/course_import.py index dd7828c2d94f..3027b1926d0f 100644 --- a/cms/djangoapps/contentstore/api/views/course_import.py +++ b/cms/djangoapps/contentstore/api/views/course_import.py @@ -106,7 +106,7 @@ class CourseImportView(CourseImportExportViewMixin, GenericAPIView): # TODO: ARCH-91 # This view is excluded from Swagger doc generation because it # does not specify a serializer class. - exclude_from_schema = True + swagger_schema = None @course_author_access_required def post(self, request, course_key): diff --git a/cms/djangoapps/contentstore/api/views/course_quality.py b/cms/djangoapps/contentstore/api/views/course_quality.py index 8f647accaa91..b301f5ac1420 100644 --- a/cms/djangoapps/contentstore/api/views/course_quality.py +++ b/cms/djangoapps/contentstore/api/views/course_quality.py @@ -77,6 +77,11 @@ class CourseQualityView(DeveloperErrorViewMixin, GenericAPIView): * mode """ + # TODO: ARCH-91 + # This view is excluded from Swagger doc generation because it + # does not specify a serializer class. + swagger_schema = None + @course_author_access_required def get(self, request, course_key): """ diff --git a/cms/djangoapps/contentstore/api/views/course_validation.py b/cms/djangoapps/contentstore/api/views/course_validation.py index 0fa8d9041c1b..d1c2c2b8c46f 100644 --- a/cms/djangoapps/contentstore/api/views/course_validation.py +++ b/cms/djangoapps/contentstore/api/views/course_validation.py @@ -65,6 +65,11 @@ class CourseValidationView(DeveloperErrorViewMixin, GenericAPIView): * has_proctoring_escalation_email - whether the course has a proctoring escalation email """ + # TODO: ARCH-91 + # This view is excluded from Swagger doc generation because it + # does not specify a serializer class. + swagger_schema = None + @course_author_access_required def get(self, request, course_key): """ diff --git a/cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py b/cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py index e3dd070573aa..e42c3e2ee397 100644 --- a/cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py +++ b/cms/djangoapps/contentstore/rest_api/v0/serializers/authoring_grading.py @@ -14,7 +14,13 @@ class GradersSerializer(serializers.Serializer): weight = serializers.IntegerField() id = serializers.IntegerField() + class Meta: + ref_name = "authoring_grading.Graders.v0" + class CourseGradingModelSerializer(serializers.Serializer): """ Serializer for course grading model data """ graders = GradersSerializer(many=True, allow_null=True, allow_empty=True) + + class Meta: + ref_name = "authoring_grading.CourseGrading.v0" diff --git a/cms/djangoapps/contentstore/rest_api/v0/urls.py b/cms/djangoapps/contentstore/rest_api/v0/urls.py index cc1e13b0929c..6e0c11d22b8f 100644 --- a/cms/djangoapps/contentstore/rest_api/v0/urls.py +++ b/cms/djangoapps/contentstore/rest_api/v0/urls.py @@ -63,7 +63,7 @@ authoring_videos.VideoEncodingsDownloadView.as_view(), name='cms_api_videos_encodings' ), re_path( - fr'grading/{settings.COURSE_ID_PATTERN}', + fr'grading/{settings.COURSE_ID_PATTERN}$', AuthoringGradingView.as_view(), name='cms_api_update_grading' ), path( diff --git a/cms/djangoapps/contentstore/rest_api/v0/views/authoring_videos.py b/cms/djangoapps/contentstore/rest_api/v0/views/authoring_videos.py index 972b6229f55a..8fb66070f3bf 100644 --- a/cms/djangoapps/contentstore/rest_api/v0/views/authoring_videos.py +++ b/cms/djangoapps/contentstore/rest_api/v0/views/authoring_videos.py @@ -128,6 +128,11 @@ class VideoEncodingsDownloadView(DeveloperErrorViewMixin, RetrieveAPIView): course_key: required argument, needed to authorize course authors and identify relevant videos. """ + # TODO: ARCH-91 + # This view is excluded from Swagger doc generation because it + # does not specify a serializer class. + swagger_schema = None + def dispatch(self, request, *args, **kwargs): # TODO: probably want to refactor this to a decorator. """ @@ -151,6 +156,11 @@ class VideoFeaturesView(DeveloperErrorViewMixin, RetrieveAPIView): public rest API endpoint providing a list of enabled video features. """ + # TODO: ARCH-91 + # This view is excluded from Swagger doc generation because it + # does not specify a serializer class. + swagger_schema = None + def dispatch(self, request, *args, **kwargs): # TODO: probably want to refactor this to a decorator. """ diff --git a/cms/urls.py b/cms/urls.py index 2e64d4bbeb79..58503f9ed92f 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -168,7 +168,7 @@ contentstore_views.textbooks_detail_handler, name='textbooks_detail_handler'), re_path(fr'^videos/{settings.COURSE_KEY_PATTERN}(?:/(?P[-\w]+))?$', contentstore_views.videos_handler, name='videos_handler'), - re_path(fr'^generate_video_upload_link/{settings.COURSE_KEY_PATTERN}', + re_path(fr'^generate_video_upload_link/{settings.COURSE_KEY_PATTERN}$', contentstore_views.generate_video_upload_link_handler, name='generate_video_upload_link'), re_path(fr'^video_images/{settings.COURSE_KEY_PATTERN}(?:/(?P[-\w]+))?$', contentstore_views.video_images_handler, name='video_images_handler'), diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index 048226c5b16c..a1b7a2602450 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -81,6 +81,7 @@ from django.views.decorators.clickjacking import xframe_options_exempt from django.views.decorators.csrf import csrf_exempt from django.views.generic.base import TemplateResponseMixin, View +from drf_yasg.utils import swagger_auto_schema from pylti1p3.contrib.django import DjangoCacheDataStorage, DjangoDbToolConf, DjangoMessageLaunch, DjangoOIDCLogin from pylti1p3.exception import LtiException, OIDCException @@ -201,8 +202,10 @@ class LibraryRootView(GenericAPIView): """ Views to list, search for, and create content libraries. """ + serializer_class = ContentLibraryMetadataSerializer @apidocs.schema( + responses={200: ContentLibraryMetadataSerializer(many=True)}, parameters=[ *LibraryApiPaginationDocs.apidoc_params, apidocs.query_parameter( @@ -530,7 +533,13 @@ class LibraryPasteClipboardView(GenericAPIView): """ Paste content of clipboard into Library. """ + serializer_class = LibraryXBlockMetadataSerializer + @convert_exceptions + @swagger_auto_schema( + request_body=LibraryPasteClipboardSerializer, + responses={200: LibraryXBlockMetadataSerializer} + ) def post(self, request, lib_key_str): """ Import the contents of the user's clipboard and paste them into the Library @@ -558,6 +567,7 @@ class LibraryBlocksView(GenericAPIView): """ Views to work with XBlocks in a specific content library. """ + serializer_class = LibraryXBlockMetadataSerializer @apidocs.schema( parameters=[ @@ -595,6 +605,10 @@ def get(self, request, lib_key_str): return self.get_paginated_response(serializer.data) @convert_exceptions + @swagger_auto_schema( + request_body=LibraryXBlockCreationSerializer, + responses={200: LibraryXBlockMetadataSerializer} + ) def post(self, request, lib_key_str): """ Add a new XBlock to this content library @@ -870,6 +884,9 @@ class LibraryImportTaskViewSet(GenericViewSet): Import blocks from Courseware through modulestore. """ + queryset = [] # type: ignore[assignment] + serializer_class = ContentLibraryBlockImportTaskSerializer + @convert_exceptions def list(self, request, lib_key_str): """ @@ -889,6 +906,10 @@ def list(self, request, lib_key_str): ) @convert_exceptions + @swagger_auto_schema( + request_body=ContentLibraryBlockImportTaskCreateSerializer, + responses={200: ContentLibraryBlockImportTaskSerializer} + ) def create(self, request, lib_key_str): """ Create and queue an import tasks for this library. diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 38089f4cce1c..94d960359a91 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -131,7 +131,7 @@ optimizely-sdk<5.0 # Date: 2023-09-18 # pinning this version to avoid updates while the library is being developed # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269 -openedx-learning==0.18.1 +openedx-learning==0.18.2 # Date: 2023-11-29 # Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 28b2f2f3b87c..66428919cc61 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -827,7 +827,7 @@ openedx-filters==1.12.0 # ora2 openedx-forum==0.1.6 # via -r requirements/edx/kernel.in -openedx-learning==0.18.1 +openedx-learning==0.18.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 93bd005966a0..faab27d2a6ba 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1381,7 +1381,7 @@ openedx-forum==0.1.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-learning==0.18.1 +openedx-learning==0.18.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 7b5bb8207338..0f927c5218f8 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1000,7 +1000,7 @@ openedx-filters==1.12.0 # ora2 openedx-forum==0.1.6 # via -r requirements/edx/base.txt -openedx-learning==0.18.1 +openedx-learning==0.18.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 9ea71f258ccb..5c991ae3e7d6 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1048,7 +1048,7 @@ openedx-filters==1.12.0 # ora2 openedx-forum==0.1.6 # via -r requirements/edx/base.txt -openedx-learning==0.18.1 +openedx-learning==0.18.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 4f13ee0171e5c7b6e7e9f51c3b23fdddb5c43555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Thu, 30 Jan 2025 14:36:21 -0300 Subject: [PATCH 30/30] fix: advanced editor styling on library authoring [FC-0076] (#36146) Fixes the styles for the advanced editors (poll, survey, LTI Provider, etc). Updated the code if `xblock_v2/xblock_iframe.html` to use `course-unit-mfe-iframe-bundle.scss` --- cms/envs/common.py | 1 + cms/envs/production.py | 1 + cms/envs/test.py | 3 ++ .../sass/course-unit-mfe-iframe-bundle.scss | 33 ++++++++++++++++++- common/templates/xblock_v2/xblock_iframe.html | 24 +++++++++----- .../core/djangoapps/xblock/rest_api/views.py | 2 ++ 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 0919834a9b64..7a36bfab3d96 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -843,6 +843,7 @@ # Public domain name of Studio (should be resolvable from the end-user's browser) CMS_BASE = 'localhost:18010' +CMS_ROOT_URL = "https://localhost:18010" LOG_DIR = '/edx/var/log/edx' diff --git a/cms/envs/production.py b/cms/envs/production.py index ad7667772f9a..da5642b53c6c 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -163,6 +163,7 @@ def get_env_setting(setting): CMS_BASE = ENV_TOKENS.get('CMS_BASE') LMS_BASE = ENV_TOKENS.get('LMS_BASE') LMS_ROOT_URL = ENV_TOKENS.get('LMS_ROOT_URL') +CMS_ROOT_URL = ENV_TOKENS.get('CMS_ROOT_URL') LMS_INTERNAL_ROOT_URL = ENV_TOKENS.get('LMS_INTERNAL_ROOT_URL', LMS_ROOT_URL) ENTERPRISE_API_URL = ENV_TOKENS.get('ENTERPRISE_API_URL', LMS_INTERNAL_ROOT_URL + '/enterprise/api/v1/') ENTERPRISE_CONSENT_API_URL = ENV_TOKENS.get('ENTERPRISE_CONSENT_API_URL', LMS_INTERNAL_ROOT_URL + '/consent/api/v1/') diff --git a/cms/envs/test.py b/cms/envs/test.py index 49db50608858..6a7c17b001fe 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -136,6 +136,9 @@ LMS_ROOT_URL = f"http://{LMS_BASE}" FEATURES['PREVIEW_LMS_BASE'] = "preview.localhost" +CMS_BASE = "localhost:8001" +CMS_ROOT_URL = f"http://{CMS_BASE}" + COURSE_AUTHORING_MICROFRONTEND_URL = "http://course-authoring-mfe" DISCUSSIONS_MICROFRONTEND_URL = "http://discussions-mfe" diff --git a/cms/static/sass/course-unit-mfe-iframe-bundle.scss b/cms/static/sass/course-unit-mfe-iframe-bundle.scss index bc0c3901b147..1f813fac253d 100644 --- a/cms/static/sass/course-unit-mfe-iframe-bundle.scss +++ b/cms/static/sass/course-unit-mfe-iframe-bundle.scss @@ -200,7 +200,7 @@ body { } } - .modal-window.modal-editor { + .modal-window.modal-editor, &.xblock-iframe-content { background-color: $white; border-radius: 6px; @@ -381,6 +381,37 @@ body { .modal-lg.modal-window.confirm.openassessment_modal_window { height: 635px; } + + // Additions for the xblock editor on the Library Authoring + &.xblock-iframe-content { + // Reset the max-height to allow the settings list to grow + .wrapper-comp-settings .list-input.settings-list { + max-height: unset; + } + + // For Google Docs and Google Calendar xblock editor + .google-edit-wrapper .xblock-inputs { + position: unset; + overflow-y: unset; + } + + .xblock-actions { + padding: ($baseline*0.75) 2% ($baseline/2) 2%; + position: sticky; + bottom: 0; + + .action-item { + @extend %t-action3; + + display: inline-block; + margin-right: ($baseline*0.75); + + &:last-child { + margin-right: 0; + } + } + } + } } .view-container .content-primary { diff --git a/common/templates/xblock_v2/xblock_iframe.html b/common/templates/xblock_v2/xblock_iframe.html index 07d81b962a65..57ff4684ddbd 100644 --- a/common/templates/xblock_v2/xblock_iframe.html +++ b/common/templates/xblock_v2/xblock_iframe.html @@ -1,6 +1,7 @@ - + + @@ -81,8 +82,11 @@ href="https://stackpath.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"> - - + + + + +