From 08c116b700ce718507fa341bb50caf93370d4cb9 Mon Sep 17 00:00:00 2001 From: muhammad-ammar Date: Thu, 8 Feb 2024 21:49:39 +0500 Subject: [PATCH] feat: address feedback --- enterprise_data/api/v1/serializers.py | 2 +- enterprise_data/api/v1/views.py | 50 +++++++++++---------------- enterprise_data/filters.py | 3 +- enterprise_data/renderers.py | 5 ++- enterprise_data/settings/test.py | 1 + 5 files changed, 27 insertions(+), 34 deletions(-) diff --git a/enterprise_data/api/v1/serializers.py b/enterprise_data/api/v1/serializers.py index e3cb9d8c..3e035455 100644 --- a/enterprise_data/api/v1/serializers.py +++ b/enterprise_data/api/v1/serializers.py @@ -27,7 +27,7 @@ class Meta: model = EnterpriseLearnerEnrollment # Do not change the order of fields below. Ordering is important becuase `progress_v3` # csv generated in `enterprise_reporting` should be same as csv generated on `admin-portal` - # Order and field names below should match with `EnterpriseLearnerEnrollmentViewSet.header` + # Order and field names below should match with `EnrollmentsCSVRenderer.header` fields = ( 'enrollment_id', 'enterprise_enrollment_id', 'is_consent_granted', 'paid_by', 'user_current_enrollment_mode', 'enrollment_date', 'unenrollment_date', diff --git a/enterprise_data/api/v1/views.py b/enterprise_data/api/v1/views.py index 38bae0c4..e2d1addb 100644 --- a/enterprise_data/api/v1/views.py +++ b/enterprise_data/api/v1/views.py @@ -18,6 +18,7 @@ from rest_framework.status import HTTP_200_OK, HTTP_404_NOT_FOUND from rest_framework.views import APIView +from django.conf import settings from django.core.paginator import Paginator from django.db.models import Count, Max, OuterRef, Prefetch, Q, Subquery, Value from django.db.models.fields import IntegerField @@ -84,6 +85,7 @@ class EnterpriseLearnerEnrollmentViewSet(EnterpriseViewSetMixin, viewsets.ReadOn ENROLLMENT_MODE_FILTER = 'user_current_enrollment_mode' COUPON_CODE_FILTER = 'coupon_code' OFFER_FILTER = 'offer_type' + # TODO: Remove after we release the streaming csv changes # This will be used as CSV header for csv generated from `admin-portal`. # Do not change the order of fields below. Ordering is important because csv generated # on `admin-portal` should match `progress_v3` csv generated in `enterprise_reporting` @@ -104,6 +106,7 @@ class EnterpriseLearnerEnrollmentViewSet(EnterpriseViewSetMixin, viewsets.ReadOn 'course_product_line', 'budget_id' ] + # TODO: Remove after we release the streaming csv changes def get_renderer_context(self): renderer_context = super().get_renderer_context() renderer_context['header'] = self.header @@ -127,18 +130,7 @@ def get_queryset(self): if cached_response.is_found: return cached_response.value else: - enterprise = EnterpriseLearner.objects.filter(enterprise_customer_uuid=enterprise_customer_uuid).exists() - - if not enterprise: - LOGGER.warning( - "[Data Overview Failure] Wrong Enterprise UUID. UUID [%s], Endpoint ['%s'], User: [%s]", - enterprise_customer_uuid, - self.request.get_full_path(), - self.request.user.username, - ) - enrollments = EnterpriseLearnerEnrollment.objects.filter(enterprise_customer_uuid=enterprise_customer_uuid) - enrollments = self.apply_filters(enrollments) TieredCache.set_all_tiers(cache_key, enrollments, DEFAULT_LEARNER_CACHE_TIMEOUT) return enrollments @@ -147,26 +139,24 @@ def list(self, request, *args, **kwargs): """ Override the list method to handle streaming CSV download. """ - queryset = self.filter_queryset(self.get_queryset()) - - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - def data_gen(queryset): - paginator = Paginator(queryset, per_page=10000) - for page_number in paginator.page_range: - enrollments = paginator.page(page_number) - serializer = self.get_serializer(enrollments, many=True) - yield serializer.data - - if self.request.query_params.get('data') == 'csv': - data = EnrollmentsCSVRenderer().render(row for chunk in data_gen(queryset) for row in chunk) - return StreamingHttpResponse(data, content_type='text/csv') + if self.request.query_params.get('streaming_csv_enabled') == 'true': + if request.accepted_renderer.format == 'csv': + return StreamingHttpResponse( + EnrollmentsCSVRenderer().render(self._stream_serialized_data()), + content_type='text/csv' + ) + + return super().list(request, *args, **kwargs) - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) + def _stream_serialized_data(self): + """ + Stream the serialized data. + """ + queryset = self.filter_queryset(self.get_queryset()) + serializer = self.get_serializer_class() + paginator = Paginator(queryset, per_page=settings.ENROLLMENTS_PAGE_SIZE) + for page_number in paginator.page_range: + yield from serializer(paginator.page(page_number).object_list, many=True).data def apply_filters(self, queryset): """ diff --git a/enterprise_data/filters.py b/enterprise_data/filters.py index de420c27..ee8d8347 100644 --- a/enterprise_data/filters.py +++ b/enterprise_data/filters.py @@ -90,9 +90,8 @@ def filter_queryset(self, request, queryset, view): """ Filter out queryset for results where enrollment mode is `audit`. """ - enterprise_uuid = view.kwargs['enterprise_id'] - if self.exclude_audit_enrollments(view): + enterprise_uuid = view.kwargs['enterprise_id'] LOGGER.info(f'[AuditEnrollmentsFilterBackend] excluding audit enrollments for: {enterprise_uuid}') # Filter out enrollments that have audit mode and do not have a coupon code or an offer. filter_query = { diff --git a/enterprise_data/renderers.py b/enterprise_data/renderers.py index 90f32943..da445180 100644 --- a/enterprise_data/renderers.py +++ b/enterprise_data/renderers.py @@ -2,6 +2,10 @@ class EnrollmentsCSVRenderer(CSVStreamingRenderer): + # This will be used as CSV header for csv generated from `admin-portal`. + # Do not change the order of fields below. Ordering is important because csv generated + # on `admin-portal` should match `progress_v3` csv generated in `enterprise_reporting` + # Order and field names below should match with `EnterpriseLearnerEnrollmentSerializer.fields` header = [ 'enrollment_id', 'enterprise_enrollment_id', 'is_consent_granted', 'paid_by', 'user_current_enrollment_mode', 'enrollment_date', 'unenrollment_date', @@ -17,4 +21,3 @@ class EnrollmentsCSVRenderer(CSVStreamingRenderer): 'enterprise_sso_uid', 'created', 'course_api_url', 'total_learning_time_hours', 'is_subsidy', 'course_product_line', 'budget_id' ] - \ No newline at end of file diff --git a/enterprise_data/settings/test.py b/enterprise_data/settings/test.py index fc05f8da..168115c0 100644 --- a/enterprise_data/settings/test.py +++ b/enterprise_data/settings/test.py @@ -104,6 +104,7 @@ def root(*args): SITE_NAME = 'analytics-data-api' ENTERPRISE_REPORTING_DB_ALIAS = 'default' +ENROLLMENTS_PAGE_SIZE = 10000 # Required for use with edx-drf-extensions JWT functionality: # USER_SETTINGS overrides for djangorestframework-jwt APISettings class