feat: address feedback

enterprise_data/api/v1/serializers.py

@@ -27,7 +27,7 @@ class Meta:
         model = EnterpriseLearnerEnrollment
         # Do not change the order of fields below. Ordering is important becuase `progress_v3`
         # csv generated in `enterprise_reporting` should be same as csv generated on `admin-portal`
-        # Order and field names below should match with `EnterpriseLearnerEnrollmentViewSet.header`
+        # Order and field names below should match with `EnrollmentsCSVRenderer.header`
         fields = (
             'enrollment_id', 'enterprise_enrollment_id', 'is_consent_granted', 'paid_by',
             'user_current_enrollment_mode', 'enrollment_date', 'unenrollment_date',

enterprise_data/api/v1/views.py

@@ -18,6 +18,7 @@
 from rest_framework.status import HTTP_200_OK, HTTP_404_NOT_FOUND
 from rest_framework.views import APIView
 
+from django.conf import settings
 from django.core.paginator import Paginator
 from django.db.models import Count, Max, OuterRef, Prefetch, Q, Subquery, Value
 from django.db.models.fields import IntegerField
@@ -84,6 +85,7 @@ class EnterpriseLearnerEnrollmentViewSet(EnterpriseViewSetMixin, viewsets.ReadOn
     ENROLLMENT_MODE_FILTER = 'user_current_enrollment_mode'
     COUPON_CODE_FILTER = 'coupon_code'
     OFFER_FILTER = 'offer_type'
+    # TODO: Remove after we release the streaming csv changes
     # This will be used as CSV header for csv generated from `admin-portal`.
     # Do not change the order of fields below. Ordering is important because csv generated
     # on `admin-portal` should match `progress_v3` csv generated in `enterprise_reporting`
@@ -104,6 +106,7 @@ class EnterpriseLearnerEnrollmentViewSet(EnterpriseViewSetMixin, viewsets.ReadOn
         'course_product_line', 'budget_id'
     ]
 
+    # TODO: Remove after we release the streaming csv changes
     def get_renderer_context(self):
         renderer_context = super().get_renderer_context()
         renderer_context['header'] = self.header
@@ -127,18 +130,7 @@ def get_queryset(self):
         if cached_response.is_found:
             return cached_response.value
         else:
-            enterprise = EnterpriseLearner.objects.filter(enterprise_customer_uuid=enterprise_customer_uuid).exists()
-
-            if not enterprise:
-                LOGGER.warning(
-                    "[Data Overview Failure] Wrong Enterprise UUID. UUID [%s], Endpoint ['%s'], User: [%s]",
-                    enterprise_customer_uuid,
-                    self.request.get_full_path(),
-                    self.request.user.username,
-                )
-
             enrollments = EnterpriseLearnerEnrollment.objects.filter(enterprise_customer_uuid=enterprise_customer_uuid)
-
             enrollments = self.apply_filters(enrollments)
             TieredCache.set_all_tiers(cache_key, enrollments, DEFAULT_LEARNER_CACHE_TIMEOUT)
             return enrollments
@@ -147,26 +139,24 @@ def list(self, request, *args, **kwargs):
         """
         Override the list method to handle streaming CSV download.
         """
-        queryset = self.filter_queryset(self.get_queryset())
-
-        page = self.paginate_queryset(queryset)
-        if page is not None:
-            serializer = self.get_serializer(page, many=True)
-            return self.get_paginated_response(serializer.data)
-
-        def data_gen(queryset):
-            paginator = Paginator(queryset, per_page=10000)
-            for page_number in paginator.page_range:
-                enrollments = paginator.page(page_number)
-                serializer = self.get_serializer(enrollments, many=True)
-                yield serializer.data
-
-        if self.request.query_params.get('data') == 'csv':
-            data = EnrollmentsCSVRenderer().render(row for chunk in data_gen(queryset) for row in chunk)
-            return StreamingHttpResponse(data, content_type='text/csv')
+        if self.request.query_params.get('streaming_csv_enabled') == 'true':
+            if request.accepted_renderer.format == 'csv':
+                return StreamingHttpResponse(
+                    EnrollmentsCSVRenderer().render(self._stream_serialized_data()),
+                    content_type='text/csv'
+                )
+
+        return super().list(request, *args, **kwargs)
 
-        serializer = self.get_serializer(queryset, many=True)
-        return Response(serializer.data)
+    def _stream_serialized_data(self):
+        """
+        Stream the serialized data.
+        """
+        queryset = self.filter_queryset(self.get_queryset())
+        serializer = self.get_serializer_class()
+        paginator = Paginator(queryset, per_page=settings.ENROLLMENTS_PAGE_SIZE)
+        for page_number in paginator.page_range:
+            yield from serializer(paginator.page(page_number).object_list, many=True).data
 
     def apply_filters(self, queryset):
         """

enterprise_data/filters.py

@@ -90,9 +90,8 @@ def filter_queryset(self, request, queryset, view):
         """
         Filter out queryset for results where enrollment mode is `audit`.
         """
-        enterprise_uuid = view.kwargs['enterprise_id']
-
         if self.exclude_audit_enrollments(view):
+            enterprise_uuid = view.kwargs['enterprise_id']
             LOGGER.info(f'[AuditEnrollmentsFilterBackend] excluding audit enrollments for: {enterprise_uuid}')
             # Filter out enrollments that have audit mode and do not have a coupon code or an offer.
             filter_query = {

enterprise_data/renderers.py

@@ -2,6 +2,10 @@
 
 
 class EnrollmentsCSVRenderer(CSVStreamingRenderer):
+    # This will be used as CSV header for csv generated from `admin-portal`.
+    # Do not change the order of fields below. Ordering is important because csv generated
+    # on `admin-portal` should match `progress_v3` csv generated in `enterprise_reporting`
+    # Order and field names below should match with `EnterpriseLearnerEnrollmentSerializer.fields`
     header = [
         'enrollment_id', 'enterprise_enrollment_id', 'is_consent_granted', 'paid_by',
         'user_current_enrollment_mode', 'enrollment_date', 'unenrollment_date',
@@ -17,4 +21,3 @@ class EnrollmentsCSVRenderer(CSVStreamingRenderer):
         'enterprise_sso_uid', 'created', 'course_api_url', 'total_learning_time_hours', 'is_subsidy',
         'course_product_line', 'budget_id'
     ]
-

enterprise_data/settings/test.py

@@ -104,6 +104,7 @@ def root(*args):
 SITE_NAME = 'analytics-data-api'
 
 ENTERPRISE_REPORTING_DB_ALIAS = 'default'
+ENROLLMENTS_PAGE_SIZE = 10000
 
 # Required for use with edx-drf-extensions JWT functionality:
 # USER_SETTINGS overrides for djangorestframework-jwt APISettings class