Merge branch 'master' of https://github.com/openedx/edx-enterprise in…

…to MueezKhan/Replace-Moodle-Unencrypted-Columns-ENT8032

CHANGELOG.rst

@@ -16,6 +16,23 @@ Change Log
 Unreleased
 ----------
 
+[4.10.1]
+--------
+
+feat: added json field in learner transmission audit to record 3 most latest response statuses
+
+[4.10.0]
+--------
+
+feat: enrollment API enhancements
+
+- Allows Enrollment API Admin to see all enrollments.
+- Makes the endpoint return more fields, such as: enrollment_track,
+  enrollment_date, user_email, course_start and course_end.
+- Changes EnterpriseCourseEnrollment's default ordering from 'created'
+  to 'id', which equivalent, but faster in some cases (due to the
+  existing indes on 'id').
+
 [4.9.5]
 --------
 

enterprise/__init__.py

@@ -2,4 +2,4 @@
 Your project description goes here.
 """
 
-__version__ = "4.9.5"
+__version__ = "4.10.1"

enterprise/api/filters.py

@@ -7,7 +7,7 @@
 
 from django.contrib import auth
 
-from enterprise.models import EnterpriseCustomerUser, SystemWideEnterpriseUserRoleAssignment
+from enterprise.models import EnterpriseCustomer, EnterpriseCustomerUser, SystemWideEnterpriseUserRoleAssignment
 
 User = auth.get_user_model()
 
@@ -33,6 +33,36 @@ def filter_queryset(self, request, queryset, view):
         return queryset
 
 
+class EnterpriseCourseEnrollmentFilterBackend(filters.BaseFilterBackend):
+    """
+    Filter backend to return enrollments under the user's enterprise(s) only.
+
+    * Staff users will bypass this filter.
+    * Non-staff users will receive enrollments under their linked enterprises,
+      only if they have the `enterprise.can_enroll_learners` permission.
+    * Non-staff users without the `enterprise.can_enroll_learners` permission
+      will receive only their own enrollments.
+    """
+
+    def filter_queryset(self, request, queryset, view):
+        """
+        Filter out enrollments if learner is not linked
+        """
+
+        if request.user.is_staff:
+            return queryset
+
+        if request.user.has_perm('enterprise.can_enroll_learners'):
+            enterprise_customers = EnterpriseCustomer.objects.filter(enterprise_customer_users__user_id=request.user.id)
+            return queryset.filter(enterprise_customer_user__enterprise_customer__in=enterprise_customers)
+
+        filter_kwargs = {
+            view.USER_ID_FILTER: request.user.id,
+        }
+
+        return queryset.filter(**filter_kwargs)
+
+
 class EnterpriseCustomerUserFilterBackend(filters.BaseFilterBackend):
     """
     Allow filtering on the enterprise customer user api endpoint.

enterprise/api/v1/serializers.py

@@ -356,6 +356,32 @@ class Meta:
         )
 
 
+class EnterpriseCourseEnrollmentWithAdditionalFieldsReadOnlySerializer(EnterpriseCourseEnrollmentReadOnlySerializer):
+    """
+    Serializer for EnterpriseCourseEnrollment model with additional fields.
+    """
+
+    class Meta:
+        model = models.EnterpriseCourseEnrollment
+        fields = (
+            'enterprise_customer_user',
+            'course_id',
+            'created',
+            'unenrolled_at',
+            'enrollment_date',
+            'enrollment_track',
+            'user_email',
+            'course_start',
+            'course_end',
+        )
+
+    enrollment_track = serializers.CharField()
+    enrollment_date = serializers.DateTimeField()
+    user_email = serializers.EmailField()
+    course_start = serializers.DateTimeField()
+    course_end = serializers.DateTimeField()
+
+
 class EnterpriseCourseEnrollmentWriteSerializer(serializers.ModelSerializer):
     """
     Serializer for writing to the EnterpriseCourseEnrollment model.

enterprise/api/v1/views/enterprise_course_enrollment.py

@@ -1,17 +1,68 @@
 """
 Views for the ``enterprise-course-enrollment`` API endpoint.
 """
+from django_filters.rest_framework import DjangoFilterBackend
+from edx_rest_framework_extensions.paginators import DefaultPagination
+from rest_framework import filters
+
+from django.core.paginator import Paginator
+from django.utils.functional import cached_property
+
 from enterprise import models
+from enterprise.api.filters import EnterpriseCourseEnrollmentFilterBackend
 from enterprise.api.v1 import serializers
 from enterprise.api.v1.views.base_views import EnterpriseReadWriteModelViewSet
 
+try:
+    from common.djangoapps.util.query import read_replica_or_default
+except ImportError:
+    def read_replica_or_default():
+        return None
+
+
+class PaginatorWithOptimizedCount(Paginator):
+    """
+    Django < 4.2 ORM doesn't strip unused annotations from count queries.
+
+    For example, if we execute this query:
+
+        Book.objects.annotate(Count('chapters')).count()
+
+    it will generate SQL like this:
+
+        SELECT COUNT(*) FROM (SELECT COUNT(...), ... FROM ...) subquery
+
+    This isn't optimal on its own, but it's not a big deal. However, this
+    becomes problematic when annotations use subqueries, because it's terribly
+    inefficient to execute the subquery for every row in the outer query.
+
+    This class overrides the count() method of Django's Paginator to strip
+    unused annotations from the query by requesting only the primary key
+    instead of all fields.
+
+    This is a temporary workaround until Django is updated to 4.2, which will
+    include a fix for this issue.
+
+    See https://code.djangoproject.com/ticket/32169 for more details.
+
+    TODO: remove this class once Django is updated to 4.2 or higher.
+    """
+    @cached_property
+    def count(self):
+        return self.object_list.values("pk").count()
+
+
+class EnterpriseCourseEnrollmentPagination(DefaultPagination):
+    django_paginator_class = PaginatorWithOptimizedCount
+
 
 class EnterpriseCourseEnrollmentViewSet(EnterpriseReadWriteModelViewSet):
     """
     API views for the ``enterprise-course-enrollment`` API endpoint.
     """
 
-    queryset = models.EnterpriseCourseEnrollment.objects.all()
+    queryset = models.EnterpriseCourseEnrollment.with_additional_fields.all()
+    filter_backends = (filters.OrderingFilter, DjangoFilterBackend, EnterpriseCourseEnrollmentFilterBackend)
 
     USER_ID_FILTER = 'enterprise_customer_user__user_id'
     FIELDS = (
@@ -20,10 +71,18 @@ class EnterpriseCourseEnrollmentViewSet(EnterpriseReadWriteModelViewSet):
     filterset_fields = FIELDS
     ordering_fields = FIELDS
 
+    pagination_class = EnterpriseCourseEnrollmentPagination
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        if self.request.method == 'GET':
+            queryset = queryset.using(read_replica_or_default())
+        return queryset
+
     def get_serializer_class(self):
         """
         Use a special serializer for any requests that aren't read-only.
         """
         if self.request.method in ('GET',):
-            return serializers.EnterpriseCourseEnrollmentReadOnlySerializer
+            return serializers.EnterpriseCourseEnrollmentWithAdditionalFieldsReadOnlySerializer
         return serializers.EnterpriseCourseEnrollmentWriteSerializer

enterprise/migrations/0198_alter_enterprisecourseenrollment_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2 on 2023-12-29 17:03
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('enterprise', '0197_auto_20231130_2239'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='enterprisecourseenrollment',
+            options={'ordering': ['id']},
+        ),
+    ]

enterprise/models.py

@@ -93,6 +93,11 @@
 except ImportError:
     CourseEntitlement = None
 
+try:
+    from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
+except ImportError:
+    CourseOverview = None
+
 LOGGER = getEnterpriseLogger(__name__)
 User = auth.get_user_model()
 mark_safe_lazy = lazy(mark_safe, str)
@@ -1857,11 +1862,61 @@ def get_queryset(self):
         """
         Override to return only those enrollment records for which learner is linked to an enterprise.
         """
+
         return super().get_queryset().select_related('enterprise_customer_user').filter(
             enterprise_customer_user__linked=True
         )
 
 
+class EnterpriseCourseEnrollmentWithAdditionalFieldsManager(models.Manager):
+    """
+    Model manager for `EnterpriseCourseEnrollment`.
+    """
+
+    def get_queryset(self):
+        """
+        Override to return only those enrollment records for which learner is linked to an enterprise.
+        """
+
+        return super().get_queryset().select_related('enterprise_customer_user').filter(
+            enterprise_customer_user__linked=True
+        ).annotate(**self._get_additional_data_annotations())
+
+    def _get_additional_data_annotations(self):
+        """
+        Return annotations with additional data for the queryset.
+        Additional fields are None in the test environment, where platform models are not available.
+        """
+
+        if not CourseEnrollment or not CourseOverview:
+            return {
+                'enrollment_track': models.Value(None, output_field=models.CharField()),
+                'enrollment_date': models.Value(None, output_field=models.DateTimeField()),
+                'user_email': models.Value(None, output_field=models.EmailField()),
+                'course_start': models.Value(None, output_field=models.DateTimeField()),
+                'course_end': models.Value(None, output_field=models.DateTimeField()),
+            }
+
+        enrollment_subquery = CourseEnrollment.objects.filter(
+            user=models.OuterRef('enterprise_customer_user__user_id'),
+            course_id=models.OuterRef('course_id'),
+        )
+        user_subquery = auth.get_user_model().objects.filter(
+            id=models.OuterRef('enterprise_customer_user__user_id'),
+        ).values('email')[:1]
+        course_subquery = CourseOverview.objects.filter(
+            id=models.OuterRef('course_id'),
+        )
+
+        return {
+            'enrollment_track': models.Subquery(enrollment_subquery.values('mode')[:1]),
+            'enrollment_date': models.Subquery(enrollment_subquery.values('created')[:1]),
+            'user_email': models.Subquery(user_subquery),
+            'course_start': models.Subquery(course_subquery.values('start')[:1]),
+            'course_end': models.Subquery(course_subquery.values('end')[:1]),
+        }
+
+
 class EnterpriseCourseEnrollment(TimeStampedModel):
     """
     Store information about the enrollment of a user in a course.
@@ -1881,11 +1936,15 @@ class EnterpriseCourseEnrollment(TimeStampedModel):
     """
 
     objects = EnterpriseCourseEnrollmentManager()
+    with_additional_fields = EnterpriseCourseEnrollmentWithAdditionalFieldsManager()
 
     class Meta:
         unique_together = (('enterprise_customer_user', 'course_id',),)
         app_label = 'enterprise'
-        ordering = ['created']
+        # Originally, we were ordering by 'created', but there was never an index on that column. To avoid creating
+        # an index on that column, we are ordering by 'id' instead, which is indexed by default and is equivalent to
+        # ordering by 'created' in this case.
+        ordering = ['id']
 
     enterprise_customer_user = models.ForeignKey(
         EnterpriseCustomerUser,

integrated_channels/blackboard/migrations/0018_auto_20240110_1301.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-01-10 13:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('blackboard', '0017_alter_historicalblackboardenterprisecustomerconfiguration_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='blackboardlearnerassessmentdatatransmissionaudit',
+            name='transmission_status',
+            field=models.JSONField(blank=True, default=list, null=True),
+        ),
+        migrations.AddField(
+            model_name='blackboardlearnerdatatransmissionaudit',
+            name='transmission_status',
+            field=models.JSONField(blank=True, default=list, null=True),
+        ),
+    ]

integrated_channels/canvas/migrations/0033_auto_20240110_1301.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-01-10 13:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('canvas', '0032_alter_historicalcanvasenterprisecustomerconfiguration_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='canvaslearnerassessmentdatatransmissionaudit',
+            name='transmission_status',
+            field=models.JSONField(blank=True, default=list, null=True),
+        ),
+        migrations.AddField(
+            model_name='canvaslearnerdatatransmissionaudit',
+            name='transmission_status',
+            field=models.JSONField(blank=True, default=list, null=True),
+        ),
+    ]

integrated_channels/cornerstone/migrations/0032_cornerstonelearnerdatatransmissionaudit_transmission_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-16 07:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cornerstone', '0031_cornerstoneapirequestlogs'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='cornerstonelearnerdatatransmissionaudit',
+            name='transmission_status',
+            field=models.JSONField(blank=True, default=list, null=True),
+        ),
+    ]

integrated_channels/degreed/migrations/0032_degreedlearnerdatatransmissionaudit_transmission_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-10 13:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('degreed', '0031_alter_historicaldegreedenterprisecustomerconfiguration_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='degreedlearnerdatatransmissionaudit',
+            name='transmission_status',
+            field=models.JSONField(blank=True, default=list, null=True),
+        ),
+    ]