Merge branch 'master' into EDX-PLATFORM-35175-deprecate-assertDictCon…

…tainsSubset

.github/workflows/unit-test-shards.json

@@ -238,6 +238,7 @@
       "cms/djangoapps/cms_user_tasks/",
       "cms/djangoapps/course_creators/",
       "cms/djangoapps/export_course_metadata/",
+      "cms/djangoapps/maintenance/",
       "cms/djangoapps/models/",
       "cms/djangoapps/pipeline_js/",
       "cms/djangoapps/xblock_config/",

cms/djangoapps/contentstore/admin.py

@@ -13,6 +13,8 @@
 from cms.djangoapps.contentstore.models import (
     BackfillCourseTabsConfig,
     CleanStaleCertificateAvailabilityDatesConfig,
+    LearningContextLinksStatus,
+    PublishableEntityLink,
     VideoUploadConfig
 )
 from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate
@@ -86,6 +88,71 @@ class CleanStaleCertificateAvailabilityDatesConfigAdmin(ConfigurationModelAdmin)
     pass
 
 
+@admin.register(PublishableEntityLink)
+class PublishableEntityLinkAdmin(admin.ModelAdmin):
+    """
+    PublishableEntityLink admin.
+    """
+    fields = (
+        "uuid",
+        "upstream_block",
+        "upstream_usage_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+        "version_synced",
+        "version_declined",
+        "created",
+        "updated",
+    )
+    readonly_fields = fields
+    list_display = [
+        "upstream_block",
+        "upstream_usage_key",
+        "downstream_usage_key",
+        "version_synced",
+        "updated",
+    ]
+    search_fields = [
+        "upstream_usage_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+    ]
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
+@admin.register(LearningContextLinksStatus)
+class LearningContextLinksStatusAdmin(admin.ModelAdmin):
+    """
+    LearningContextLinksStatus admin.
+    """
+    fields = (
+        "context_key",
+        "status",
+        "created",
+        "updated",
+    )
+    readonly_fields = ("created", "updated")
+    list_display = (
+        "context_key",
+        "status",
+        "created",
+        "updated",
+    )
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
 admin.site.register(BackfillCourseTabsConfig, ConfigurationModelAdmin)
 admin.site.register(VideoUploadConfig, ConfigurationModelAdmin)
 admin.site.register(CourseOutlineRegenerate, CourseOutlineRegenerateAdmin)

cms/djangoapps/contentstore/asset_storage_handlers.py

@@ -25,7 +25,7 @@
 from common.djangoapps.util.json_request import JsonResponse
 from openedx.core.djangoapps.contentserver.caching import del_cached_content
 from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
-from openedx_filters.course_authoring.filters import LMSPageURLRequested
+from openedx_filters.content_authoring.filters import LMSPageURLRequested
 from xmodule.contentstore.content import StaticContent  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.contentstore.django import contentstore  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.exceptions import NotFoundError  # lint-amnesty, pylint: disable=wrong-import-order
@@ -717,7 +717,7 @@ def get_asset_json(display_name, content_type, date, location, thumbnail_locatio
     asset_url = StaticContent.serialize_asset_key_with_slash(location)
 
     ## .. filter_implemented_name: LMSPageURLRequested
-    ## .. filter_type: org.openedx.course_authoring.lms.page.url.requested.v1
+    ## .. filter_type: org.openedx.content_authoring.lms.page.url.requested.v1
     lms_root, _ = LMSPageURLRequested.run_filter(
         url=configuration_helpers.get_value('LMS_ROOT_URL', settings.LMS_ROOT_URL),
         org=location.org,

cms/djangoapps/contentstore/exams.py

@@ -71,7 +71,16 @@ def register_exams(course_key):
             timed_exam.is_onboarding_exam
         )
 
-        due_date = timed_exam.due.isoformat() if timed_exam.due else (course.end.isoformat() if course.end else None)
+        # Exams in courses not using an LTI based proctoring provider should use the original definition of due_date
+        # from contentstore/proctoring.py. These exams are powered by the edx-proctoring plugin and not the edx-exams
+        # microservice.
+        if course.proctoring_provider == 'lti_external':
+            due_date = (
+                timed_exam.due.isoformat() if timed_exam.due
+                else (course.end.isoformat() if course.end else None)
+            )
+        else:
+            due_date = timed_exam.due if not course.self_paced else None
 
         exams_list.append({
             'course_id': str(course_key),

cms/djangoapps/contentstore/helpers.py

@@ -10,6 +10,7 @@
 import re
 
 from attrs import frozen, Factory
+from django.core.files.base import ContentFile
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.utils.translation import gettext as _
@@ -23,6 +24,11 @@
 from xmodule.exceptions import NotFoundError
 from xmodule.modulestore.django import modulestore
 from xmodule.xml_block import XmlMixin
+from xmodule.video_block.transcripts_utils import Transcript, build_components_import_path
+from edxval.api import (
+    create_external_video,
+    create_or_update_video_transcript,
+)
 
 from cms.djangoapps.models.settings.course_grading import CourseGradingModel
 from cms.lib.xblock.upstream_sync import UpstreamLink, UpstreamLinkException, fetch_customizable_fields
@@ -274,8 +280,14 @@ def _insert_static_files_into_downstream_xblock(
         course_key=downstream_xblock.context_key,
         staged_content_id=staged_content_id,
         static_files=static_files,
-        usage_key=downstream_xblock.scope_ids.usage_id,
+        usage_key=downstream_xblock.usage_key,
     )
+    if downstream_xblock.usage_key.block_type == 'video':
+        _import_transcripts(
+            downstream_xblock,
+            staged_content_id=staged_content_id,
+            static_files=static_files,
+        )
 
     # Rewrite the OLX's static asset references to point to the new
     # locations for those assets. See _import_files_into_course for more
@@ -331,6 +343,13 @@ def import_staged_content_from_user_clipboard(parent_key: UsageKey, request) ->
             tags=user_clipboard.content.tags,
         )
 
+        usage_key = new_xblock.usage_key
+        if usage_key.block_type == 'video':
+            # The edx_video_id must always be new so as not
+            # to interfere with the data of the copied block
+            new_xblock.edx_video_id = create_external_video(display_name='external video')
+            store.update_item(new_xblock, request.user.id)
+
         notices = _insert_static_files_into_downstream_xblock(new_xblock, user_clipboard.content.id, request)
 
     return new_xblock, notices
@@ -630,8 +649,8 @@ def _import_file_into_course(
     # we're not going to attempt to change.
     if clipboard_file_path.startswith('static/'):
         # If it's in this form, it came from a library and assumes component-local assets
-        file_path = clipboard_file_path.lstrip('static/')
-        import_path = f"components/{usage_key.block_type}/{usage_key.block_id}/{file_path}"
+        file_path = clipboard_file_path.removeprefix('static/')
+        import_path = build_components_import_path(usage_key, file_path)
         filename = pathlib.Path(file_path).name
         new_key = course_key.make_asset_key("asset", import_path.replace("/", "_"))
     else:
@@ -672,6 +691,50 @@ def _import_file_into_course(
         return False, {}
 
 
+def _import_transcripts(
+    block: XBlock,
+    staged_content_id: int,
+    static_files: list[content_staging_api.StagedContentFileData],
+):
+    """
+    Adds transcripts to VAL using the new edx_video_id.
+    """
+    for file_data_obj in static_files:
+        clipboard_file_path = file_data_obj.filename
+        data = content_staging_api.get_staged_content_static_file_data(
+            staged_content_id,
+            clipboard_file_path
+        )
+        if data is None:
+            raise NotFoundError(file_data_obj.source_key)
+
+        if clipboard_file_path.startswith('static/'):
+            # If it's in this form, it came from a library and assumes component-local assets
+            file_path = clipboard_file_path.removeprefix('static/')
+        else:
+            # Otherwise it came from a course...
+            file_path = clipboard_file_path
+
+        filename = pathlib.Path(file_path).name
+
+        language_code = next((k for k, v in block.transcripts.items() if v == filename), None)
+        if language_code:
+            sjson_subs = Transcript.convert(
+                content=data,
+                input_format=Transcript.SRT,
+                output_format=Transcript.SJSON
+            ).encode()
+            create_or_update_video_transcript(
+                video_id=block.edx_video_id,
+                language_code=language_code,
+                metadata={
+                    'file_format': Transcript.SJSON,
+                    'language_code': language_code
+                },
+                file_data=ContentFile(sjson_subs),
+            )
+
+
 def is_item_in_course_tree(item):
     """
     Check that the item is in the course tree.

cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py

@@ -0,0 +1,94 @@
+"""
+Management command to recreate upstream-dowstream links in PublishableEntityLink for course(s).
+
+This command can be run for all the courses or for given list of courses.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import gettext as _
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+
+from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
+
+from ...tasks import create_or_update_upstream_links
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Recreate links for course(s) in PublishableEntityLink table.
+
+    Examples:
+        # Recreate upstream links for two courses.
+        $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
+        --course course-v1:edX+DemoX.2+2015
+        # Force recreate upstream links for one or more courses including processed ones.
+        $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
+        --course course-v1:edX+DemoX.2+2015 --force
+        # Recreate upstream links for all courses.
+        $ ./manage.py cms recreate_upstream_links --all
+        # Force recreate links for all courses including completely processed ones.
+        $ ./manage.py cms recreate_upstream_links --all --force
+        # Delete all links and force recreate links for all courses
+        $ ./manage.py cms recreate_upstream_links --all --force --replace
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--course',
+            metavar=_('COURSE_KEY'),
+            action='append',
+            help=_('Recreate links for xblocks under given course keys. For eg. course-v1:edX+DemoX.1+2014'),
+            default=[],
+        )
+        parser.add_argument(
+            '--all',
+            action='store_true',
+            help=_(
+                'Recreate links for xblocks under all courses. NOTE: this can take long time depending'
+                ' on number of course and xblocks'
+            ),
+        )
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            help=_('Recreate links even for completely processed courses.'),
+        )
+        parser.add_argument(
+            '--replace',
+            action='store_true',
+            help=_('Delete all and create links for given course(s).'),
+        )
+
+    def handle(self, *args, **options):
+        """
+        Handle command
+        """
+        courses = options['course']
+        should_process_all = options['all']
+        force = options['force']
+        replace = options['replace']
+        time_now = datetime.now(tz=timezone.utc)
+        if not courses and not should_process_all:
+            raise CommandError('Either --course or --all argument should be provided.')
+
+        if should_process_all and courses:
+            raise CommandError('Only one of --course or --all argument should be provided.')
+
+        if should_process_all:
+            courses = CourseOverview.get_all_course_keys()
+        for course in courses:
+            log.info(f"Start processing upstream->dowstream links in course: {course}")
+            try:
+                CourseKey.from_string(str(course))
+            except InvalidKeyError:
+                log.error(f"Invalid course key: {course}, skipping..")
+                continue
+            create_or_update_upstream_links.delay(str(course), force=force, replace=replace, created=time_now)