feat: stream csv downloads

enterprise_data/api/v1/views.py

@@ -21,6 +21,7 @@
 from django.db.models import Count, Max, OuterRef, Prefetch, Q, Subquery, Value
 from django.db.models.fields import IntegerField
 from django.db.models.functions import Coalesce
+from django.http import StreamingHttpResponse
 from django.utils import timezone
 
 from enterprise_data.api.v1 import serializers
@@ -69,6 +70,27 @@ def paginate_queryset(self, queryset):
         return super().paginate_queryset(queryset)
 
 
+from rest_framework_csv.renderers import CSVStreamingRenderer
+
+
+class EnrollmentsCSVRenderer(CSVStreamingRenderer):
+    header = [
+        'enrollment_id', 'enterprise_enrollment_id', 'is_consent_granted', 'paid_by',
+        'user_current_enrollment_mode', 'enrollment_date', 'unenrollment_date',
+        'unenrollment_end_within_date', 'is_refunded', 'seat_delivery_method',
+        'offer_id', 'offer_name', 'offer_type', 'coupon_code', 'coupon_name', 'contract_id',
+        'course_list_price', 'amount_learner_paid', 'course_key', 'courserun_key',
+        'course_title', 'course_pacing_type', 'course_start_date', 'course_end_date',
+        'course_duration_weeks', 'course_max_effort', 'course_min_effort',
+        'course_primary_program', 'primary_program_type', 'course_primary_subject', 'has_passed',
+        'last_activity_date', 'progress_status', 'passed_date', 'current_grade',
+        'letter_grade', 'enterprise_user_id', 'user_email', 'user_account_creation_date',
+        'user_country_code', 'user_username', 'enterprise_name', 'enterprise_customer_uuid',
+        'enterprise_sso_uid', 'created', 'course_api_url', 'total_learning_time_hours', 'is_subsidy',
+        'course_product_line', 'budget_id'
+    ]
+
+
 class EnterpriseLearnerEnrollmentViewSet(EnterpriseViewSetMixin, viewsets.ReadOnlyModelViewSet):
     """
     Viewset for routes related to Enterprise course enrollments.
@@ -140,6 +162,25 @@ def get_queryset(self):
             TieredCache.set_all_tiers(cache_key, enrollments, DEFAULT_LEARNER_CACHE_TIMEOUT)
             return enrollments
 
+    def list(self, request, *args, **kwargs):
+        """
+        Override the list method to handle streaming CSV download.
+        """
+        queryset = self.filter_queryset(self.get_queryset())
+
+        page = self.paginate_queryset(queryset)
+        if page is not None:
+            serializer = self.get_serializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+
+        serializer = self.get_serializer(queryset, many=True)
+
+        if self.request.query_params.get('data') == 'csv':
+            data = EnrollmentsCSVRenderer().render(serializer.data)
+            return StreamingHttpResponse(data, content_type='text/csv')
+
+        return Response(serializer.data)
+
     def apply_filters(self, queryset):
         """
         Filters enrollments based on query params.

enterprise_data/filters.py

@@ -73,14 +73,26 @@ class AuditEnrollmentsFilterBackend(filters.BaseFilterBackend, FiltersMixin):
     `user_current_enrollment_mode` field.
     """
 
+    def exclude_audit_enrollments(self, view):
+        """
+        Determine if audit enrollments should be excluded.
+        """
+        # this will be passed from admin-portal to avoid api call to lms
+        audit_enrollments = view.request.query_params.get('audit_enrollments')
+        if audit_enrollments:
+            return audit_enrollments == 'false'
+
+        enterprise_uuid = view.kwargs['enterprise_id']
+        enterprise_customer = self.get_enterprise_customer(enterprise_uuid)
+        return enterprise_customer.get('enable_audit_data_reporting') == False
+
     def filter_queryset(self, request, queryset, view):
         """
         Filter out queryset for results where enrollment mode is `audit`.
         """
         enterprise_uuid = view.kwargs['enterprise_id']
-        enterprise_customer = self.get_enterprise_customer(enterprise_uuid)
 
-        if not enterprise_customer.get('enable_audit_data_reporting'):
+        if self.exclude_audit_enrollments(view):
             LOGGER.info(f'[AuditEnrollmentsFilterBackend] excluding audit enrollments for: {enterprise_uuid}')
             # Filter out enrollments that have audit mode and do not have a coupon code or an offer.
             filter_query = {

requirements/base.in

@@ -7,6 +7,7 @@ edx-drf-extensions
 edx-opaque-keys
 Django
 django-fernet-fields-v2
+djangorestframework-csv
 django-filter
 django-model-utils
 edx-rbac

requirements/base.txt

@@ -10,22 +10,22 @@ asgiref==3.7.2
     # via django
 asn1crypto==1.5.1
     # via snowflake-connector-python
-awscli==1.32.24
+awscli==1.32.35
     # via -r requirements/reporting.in
 bcrypt==4.1.2
     # via paramiko
 billiard==3.6.4.0
     # via celery
-boto3==1.34.24
+boto3==1.34.35
     # via -r requirements/reporting.in
-botocore==1.34.24
+botocore==1.34.35
     # via
     #   awscli
     #   boto3
     #   s3transfer
 celery==4.4.7
     # via -r requirements/reporting.in
-certifi==2023.11.17
+certifi==2024.2.2
     # via
     #   py2neo
     #   requests
@@ -84,8 +84,11 @@ django-waffle==4.1.0
     #   edx-drf-extensions
 djangorestframework==3.14.0
     # via
+    #   djangorestframework-csv
     #   drf-jwt
     #   edx-drf-extensions
+djangorestframework-csv==3.0.2
+    # via -r requirements/base.in
 docutils==0.16
     # via awscli
 drf-jwt==1.19.2
@@ -95,7 +98,7 @@ edx-django-utils==5.10.1
     #   -r requirements/base.in
     #   edx-drf-extensions
     #   edx-rest-api-client
-edx-drf-extensions==9.1.2
+edx-drf-extensions==10.2.0
     # via
     #   -r requirements/base.in
     #   edx-rbac
@@ -109,7 +112,7 @@ edx-rest-api-client==5.6.1
     # via -r requirements/base.in
 factory-boy==3.3.0
     # via -r requirements/base.in
-faker==22.5.0
+faker==22.7.0
     # via factory-boy
 filelock==3.13.1
     # via snowflake-connector-python
@@ -127,7 +130,7 @@ kombu==4.6.11
     # via celery
 monotonic==1.6
     # via py2neo
-newrelic==9.5.0
+newrelic==9.6.0
     # via edx-django-utils
 packaging==23.2
     # via
@@ -176,7 +179,7 @@ python-dateutil==2.8.2
     #   botocore
     #   faker
     #   vertica-python
-pytz==2023.3.post1
+pytz==2024.1
     # via
     #   celery
     #   django
@@ -212,7 +215,7 @@ six==1.16.0
     #   vertica-python
 slumber==0.7.1
     # via edx-rest-api-client
-snowflake-connector-python==3.6.0
+snowflake-connector-python==3.7.0
     # via -r requirements/reporting.in
 sortedcontainers==2.4.0
     # via snowflake-connector-python

requirements/ci.txt

@@ -4,7 +4,7 @@
 #
 #    make upgrade
 #
-coverage==7.4.0
+coverage==7.4.1
     # via -r requirements/ci.in
 distlib==0.3.8
     # via virtualenv
@@ -14,9 +14,9 @@ filelock==3.13.1
     #   virtualenv
 packaging==23.2
     # via tox
-platformdirs==4.1.0
+platformdirs==4.2.0
     # via virtualenv
-pluggy==1.3.0
+pluggy==1.4.0
     # via tox
 py==1.11.0
     # via tox

requirements/dev.txt

@@ -10,19 +10,19 @@ asgiref==3.7.2
     # via django
 asn1crypto==1.5.1
     # via snowflake-connector-python
-astroid==3.0.2
+astroid==3.0.3
     # via
     #   pylint
     #   pylint-celery
-awscli==1.32.24
+awscli==1.32.35
     # via -r requirements/reporting.in
 bcrypt==4.1.2
     # via paramiko
 billiard==3.6.4.0
     # via celery
-boto3==1.34.24
+boto3==1.34.35
     # via -r requirements/reporting.in
-botocore==1.34.24
+botocore==1.34.35
     # via
     #   awscli
     #   boto3
@@ -31,7 +31,7 @@ build==1.0.3
     # via pip-tools
 celery==4.4.7
     # via -r requirements/reporting.in
-certifi==2023.11.17
+certifi==2024.2.2
     # via
     #   py2neo
     #   requests
@@ -56,7 +56,7 @@ click==8.1.7
     #   pip-tools
 click-log==0.4.0
     # via edx-lint
-code-annotations==1.5.0
+code-annotations==1.6.0
     # via edx-lint
 colorama==0.4.4
     # via awscli
@@ -68,11 +68,10 @@ cryptography==41.0.7
     #   pgpy
     #   pyjwt
     #   pyopenssl
-    #   secretstorage
     #   snowflake-connector-python
 diff-cover==8.0.3
     # via -r requirements/dev-enterprise_data.in
-dill==0.3.7
+dill==0.3.8
     # via pylint
 distlib==0.3.8
     # via virtualenv
@@ -109,8 +108,11 @@ django-waffle==4.1.0
     #   edx-drf-extensions
 djangorestframework==3.14.0
     # via
+    #   djangorestframework-csv
     #   drf-jwt
     #   edx-drf-extensions
+djangorestframework-csv==3.0.2
+    # via -r requirements/base.in
 docutils==0.16
     # via
     #   awscli
@@ -122,7 +124,7 @@ edx-django-utils==5.10.1
     #   -r requirements/base.in
     #   edx-drf-extensions
     #   edx-rest-api-client
-edx-drf-extensions==9.1.2
+edx-drf-extensions==10.2.0
     # via
     #   -r requirements/base.in
     #   edx-rbac
@@ -142,7 +144,7 @@ edx-rest-api-client==5.6.1
     # via -r requirements/base.in
 factory-boy==3.3.0
     # via -r requirements/base.in
-faker==22.5.0
+faker==22.7.0
     # via factory-boy
 filelock==3.13.1
     # via
@@ -168,10 +170,6 @@ isort==5.13.2
     #   pylint
 jaraco-classes==3.3.0
     # via keyring
-jeepney==0.8.0
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.1.3
     # via
     #   code-annotations
@@ -188,7 +186,7 @@ lxml==5.1.0
     # via edx-i18n-tools
 markdown-it-py==3.0.0
     # via rich
-markupsafe==2.1.4
+markupsafe==2.1.5
     # via jinja2
 mccabe==0.7.0
     # via pylint
@@ -198,7 +196,7 @@ monotonic==1.6
     # via py2neo
 more-itertools==10.2.0
     # via jaraco-classes
-newrelic==9.5.0
+newrelic==9.6.0
     # via edx-django-utils
 nh3==0.2.15
     # via readme-renderer
@@ -212,7 +210,7 @@ pansi==2020.7.3
     # via py2neo
 paramiko==3.4.0
     # via -r requirements/reporting.in
-path==16.9.0
+path==16.10.0
     # via edx-i18n-tools
 pbr==6.0.0
     # via stevedore
@@ -227,7 +225,7 @@ platformdirs==3.11.0
     #   pylint
     #   snowflake-connector-python
     #   virtualenv
-pluggy==1.3.0
+pluggy==1.4.0
     # via
     #   diff-cover
     #   tox
@@ -292,9 +290,9 @@ python-dateutil==2.8.2
     #   botocore
     #   faker
     #   vertica-python
-python-slugify==8.0.1
+python-slugify==8.0.3
     # via code-annotations
-pytz==2023.3.post1
+pytz==2024.1
     # via
     #   celery
     #   django
@@ -331,8 +329,6 @@ s3transfer==0.10.0
     # via
     #   awscli
     #   boto3
-secretstorage==3.3.3
-    # via keyring
 semantic-version==2.10.0
     # via edx-drf-extensions
 six==1.16.0
@@ -349,7 +345,7 @@ slumber==0.7.1
     # via edx-rest-api-client
 snowballstemmer==2.2.0
     # via pydocstyle
-snowflake-connector-python==3.6.0
+snowflake-connector-python==3.7.0
     # via -r requirements/reporting.in
 sortedcontainers==2.4.0
     # via snowflake-connector-python

requirements/pip.txt

@@ -8,7 +8,7 @@ wheel==0.42.0
     # via -r requirements/pip.in
 
 # The following packages are considered to be unsafe in a requirements file:
-pip==23.3.2
+pip==24.0
     # via -r requirements/pip.in
 setuptools==69.0.3
     # via -r requirements/pip.in