Merge branch 'master' of github.com:openedx/edx-enterprise into hamza…

…/ENT-8277-record-blackboard-client-calls

CHANGELOG.rst

@@ -16,15 +16,25 @@ Change Log
 Unreleased
 ----------
 
-[4.10.13]
+[4.11.4]
 ---------
+* feat: update blackboard client to store API calls in DB
 
-feat: update blackboard client to store API calls in DB
+[4.11.3]
+---------
+* feat: update cornerstone client to store API calls in DB
+
+[4.11.2]
+---------
+* feat: added caching for fetching degreed course id 
 
-[4.10.12]
+[4.11.1]
 ---------
+* Added management command to fix `LearnerDataTransmissionAudit` table records.
 
-feat: update cornerstone client to store API calls in DB
+[4.11.0]
+---------
+* Added the ability for enterprise customers to enable/disable academies.
 
 [4.10.11]
 ---------

enterprise/__init__.py

@@ -2,4 +2,4 @@
 Your project description goes here.
 """
 
-__version__ = "4.10.13"
+__version__ = "4.11.4"

enterprise/admin/__init__.py

@@ -214,7 +214,7 @@ class EnterpriseCustomerAdmin(DjangoObjectActions, SimpleHistoryAdmin):
                        'enable_executive_education_2U_fulfillment',
                        'enable_career_engagement_network_on_learner_portal',
                        'career_engagement_network_message', 'enable_pathways', 'enable_programs',
-                       'enable_demo_data_for_analytics_and_lpr'),
+                       'enable_demo_data_for_analytics_and_lpr', 'enable_academies'),
             'description': ('The following default settings should be the same for '
                             'the majority of enterprise customers, '
                             'and are either rarely used, unlikely to be sold, '

enterprise/api/v1/serializers.py

@@ -222,7 +222,7 @@ class Meta:
             'enterprise_customer_catalogs', 'reply_to', 'enterprise_notification_banner', 'hide_labor_market_data',
             'modified', 'enable_universal_link', 'enable_browse_and_request', 'admin_users',
             'enable_career_engagement_network_on_learner_portal', 'career_engagement_network_message',
-            'enable_pathways', 'enable_programs', 'enable_demo_data_for_analytics_and_lpr',
+            'enable_pathways', 'enable_programs', 'enable_demo_data_for_analytics_and_lpr', 'enable_academies',
         )
 
     identity_providers = EnterpriseCustomerIdentityProviderSerializer(many=True, read_only=True)

enterprise/migrations/0199_auto_20240130_0628.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-01-30 06:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('enterprise', '0198_alter_enterprisecourseenrollment_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='enterprisecustomer',
+            name='enable_academies',
+            field=models.BooleanField(default=False, help_text='If checked, the learners will be able to see the academies on the learner portal dashboard.', verbose_name='Display academies screen'),
+        ),
+        migrations.AddField(
+            model_name='historicalenterprisecustomer',
+            name='enable_academies',
+            field=models.BooleanField(default=False, help_text='If checked, the learners will be able to see the academies on the learner portal dashboard.', verbose_name='Display academies screen'),
+        ),
+    ]

enterprise/models.py

@@ -396,6 +396,14 @@ class Meta:
         )
     )
 
+    enable_academies = models.BooleanField(
+        verbose_name="Display academies screen",
+        default=False,
+        help_text=_(
+            "If checked, the learners will be able to see the academies on the learner portal dashboard."
+        )
+    )
+
     enable_analytics_screen = models.BooleanField(
         verbose_name="Display analytics page",
         default=True,

integrated_channels/blackboard/client.py

@@ -17,10 +17,7 @@
 from integrated_channels.blackboard.exporters.content_metadata import BLACKBOARD_COURSE_CONTENT_NAME
 from integrated_channels.exceptions import ClientError
 from integrated_channels.integrated_channel.client import IntegratedChannelApiClient
-from integrated_channels.utils import (
-    generate_formatted_log,
-    refresh_session_if_expired,
-)
+from integrated_channels.utils import generate_formatted_log, refresh_session_if_expired
 
 LOGGER = logging.getLogger(__name__)
 
@@ -58,9 +55,6 @@ def __init__(self, enterprise_configuration):
             'blackboard',
             'BlackboardGlobalConfiguration'
         )
-        self.IntegratedChannelAPIRequestLogs = apps.get_model(
-            "integrated_channel", "IntegratedChannelAPIRequestLogs"
-        )
         self.global_blackboard_config = BlackboardGlobalConfiguration.current()
         self.config = apps.get_app_config('blackboard')
         self.session = None
@@ -604,6 +598,10 @@ def generate_course_content_delete_url(self, course_id, content_id):
     def stringify_and_store_api_record(
         self, url, data, time_taken, status_code, response_body
     ):
+        """
+        Helper method to stringify `data` arg and create new record in
+        `IntegratedChannelAPIRequestLogs` model
+        """
         if data is not None:
             # Convert data to string if it's not already a string
             if not isinstance(data, str):
@@ -614,11 +612,23 @@ def stringify_and_store_api_record(
                     else:
                         # If it's another type, simply convert to string
                         data = str(data)
-                except Exception as e:
+                except Exception as e:  # pylint: disable=broad-except
+                    LOGGER.error(
+                        generate_formatted_log(
+                            self.enterprise_configuration.channel_code(),
+                            self.enterprise_configuration.enterprise_customer.uuid,
+                            None,
+                            None,
+                            f"stringify_and_store_api_record: Unable to stringify data: {e}",
+                        )
+                    )
                     pass
             # Store stringified data in the database
             try:
-                self.IntegratedChannelAPIRequestLogs.store_api_call(
+                IntegratedChannelAPIRequestLogs = apps.get_model(
+                    "integrated_channel", "IntegratedChannelAPIRequestLogs"
+                )
+                IntegratedChannelAPIRequestLogs.store_api_call(
                     enterprise_customer=self.enterprise_configuration.enterprise_customer,
                     enterprise_customer_configuration_id=self.enterprise_configuration.id,
                     endpoint=url,
@@ -627,8 +637,16 @@ def stringify_and_store_api_record(
                     status_code=status_code,
                     response_body=response_body,
                 )
-            except Exception as e:
-                print(f"Failed to store data in the database: {e}")
+            except Exception as e:  # pylint: disable=broad-except
+                LOGGER.error(
+                    generate_formatted_log(
+                        self.enterprise_configuration.channel_code(),
+                        self.enterprise_configuration.enterprise_customer.uuid,
+                        None,
+                        None,
+                        f"stringify_and_store_api_record: Failed to store data in the database: {e}",
+                    )
+                )
 
     def _get(self, url, data=None):
         """

integrated_channels/cornerstone/client.py

@@ -13,7 +13,7 @@
 
 from integrated_channels.cornerstone.utils import get_or_create_key_pair
 from integrated_channels.integrated_channel.client import IntegratedChannelApiClient
-from integrated_channels.utils import generate_formatted_log, store_api_call
+from integrated_channels.utils import generate_formatted_log
 
 LOGGER = logging.getLogger(__name__)
 
@@ -88,6 +88,9 @@ def create_course_completion(self, user_id, payload):
         Raises:
             HTTPError: if we received a failure response code from Cornerstone
         """
+        IntegratedChannelAPIRequestLogs = apps.get_model(
+            "integrated_channel", "IntegratedChannelAPIRequestLogs"
+        )
         json_payload = json.loads(payload)
         callback_url = json_payload['data'].pop('callbackUrl')
         session_token = self.enterprise_configuration.session_token
@@ -115,7 +118,7 @@ def create_course_completion(self, user_id, payload):
             }
         )
         duration_seconds = time.time() - start_time
-        store_api_call(
+        IntegratedChannelAPIRequestLogs.store_api_call(
             enterprise_customer=self.enterprise_configuration.enterprise_customer,
             enterprise_customer_configuration_id=self.enterprise_configuration.id,
             endpoint=url,

integrated_channels/cornerstone/views.py

@@ -13,13 +13,13 @@
 from rest_framework.authentication import SessionAuthentication
 from rest_framework.response import Response
 
+from django.apps import apps
 from django.utils.http import parse_http_date_safe
 
 from enterprise.api.throttles import ServiceUserThrottle
 from enterprise.utils import get_enterprise_customer, get_enterprise_worker_user, get_oauth2authentication_class
 from integrated_channels.cornerstone.models import CornerstoneEnterpriseCustomerConfiguration
 from integrated_channels.integrated_channel.constants import ISO_8601_DATE_FORMAT
-from integrated_channels.utils import store_api_call
 
 logger = getLogger(__name__)
 
@@ -103,6 +103,9 @@ class CornerstoneCoursesListView(BaseViewSet):
     def get(self, request, *args, **kwargs):
         start_time = time.time()
         enterprise_customer_uuid = request.GET.get('ciid')
+        IntegratedChannelAPIRequestLogs = apps.get_model(
+            "integrated_channel", "IntegratedChannelAPIRequestLogs"
+        )
         if not enterprise_customer_uuid:
             return Response(
                 status=status.HTTP_400_BAD_REQUEST,
@@ -163,7 +166,7 @@ def get(self, request, *args, **kwargs):
         duration_seconds = time.time() - start_time
         headers_dict = dict(request.headers)
         headers_json = json.dumps(headers_dict)
-        store_api_call(
+        IntegratedChannelAPIRequestLogs.store_api_call(
             enterprise_customer=enterprise_customer,
             enterprise_customer_configuration_id=enterprise_config.id,
             endpoint=request.get_full_path(),

integrated_channels/degreed2/client.py

@@ -9,6 +9,7 @@
 from http import HTTPStatus
 
 import requests
+from edx_django_utils.cache import TieredCache, get_cache_key
 from six.moves.urllib.parse import urljoin
 
 from django.apps import apps
@@ -182,9 +183,17 @@ def delete_course_completion(self, user_id, payload):
 
     def fetch_degreed_course_id(self, external_id):
         """
-        Fetch the 'id' of a course from Degreed2, given the external-id as a search param
-        'external-id' is the edX course key
+        Fetch the 'id' of a course from cache first and if not found then send a request to Degreed2,
+        given the external-id as a search param 'external-id' is the edX course key.
         """
+        cache_key = get_cache_key(
+            resource='degreed2_course_id',
+            resource_id=external_id,
+        )
+        cached_course_id = TieredCache.get_cached_response(cache_key)
+        if cached_course_id.is_found:
+            LOGGER.info(self.make_log_msg(external_id, f'Found cached course id: {cached_course_id.value}'))
+            return cached_course_id.value
         # QueryDict converts + to space
         params = QueryDict(f"filter[external_id]={external_id.replace('+','%2B')}")
         course_search_url = f'{self.get_courses_url()}?{params.urlencode(safe="[]")}'
@@ -201,7 +210,11 @@ def fetch_degreed_course_id(self, external_id):
             )
         response_json = json.loads(response_body)
         if response_json['data']:
-            return response_json['data'][0]['id']
+            # cache the course id with a 1 day expiration
+            response_course_id = response_json['data'][0]['id']
+            expires_in = 60 * 60 * 24  # 1 day
+            TieredCache.set_all_tiers(cache_key, response_course_id, expires_in)
+            return response_course_id
         raise ClientError(
             f'Degreed2: Attempted to find degreed course id but failed, external id was {external_id}'
             f', Response from Degreed was {response_body}')

integrated_channels/integrated_channel/admin/__init__.py

@@ -89,13 +89,3 @@ class ApiResponseRecordAdmin(admin.ModelAdmin):
     )
 
     list_per_page = 1000
-
-
-@admin.register(IntegratedChannelAPIRequestLogs)
-class CornerstoneAPIRequestLogAdmin(admin.ModelAdmin):
-    """
-    Django admin model for IntegratedChannelAPIRequestLogs.
-    """
-
-    class Meta:
-        model = IntegratedChannelAPIRequestLogs

integrated_channels/integrated_channel/management/commands/remove_duplicate_learner_transmission_audit_records.py

@@ -0,0 +1,67 @@
+"""
+Transmits consenting enterprise learner data to the integrated channels.
+"""
+from logging import getLogger
+
+from django.apps import apps
+from django.contrib import auth
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.db.models import Max
+from django.utils.translation import gettext as _
+
+User = auth.get_user_model()
+LOGGER = getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Management command which removes the duplicated transmission audit records for integration channels
+    """
+    help = _('''
+    Transmit Enterprise learner course completion data for the given EnterpriseCustomer.
+    ''')
+
+    def handle(self, *args, **options):
+        """
+        Remove the duplicated transmission audit records for integration channels.
+        """
+        # Multiple transmission records were being saved against single enterprise_course_enrollment_id in case
+        # transmission fails against course and course run id. Job of this management command is to keep the latest
+        # record for enterprise_course_enrollment_id that doesn't start with "course-v1: and delete all other records."
+        channel_learner_audit_models = [
+            ('moodle', 'MoodleLearnerDataTransmissionAudit'),
+            ('blackboard', 'BlackboardLearnerDataTransmissionAudit'),
+            ('cornerstone', 'CornerstoneLearnerDataTransmissionAudit'),
+            ('canvas', 'CanvasLearnerAssessmentDataTransmissionAudit'),
+            ('degreed2', 'Degreed2LearnerDataTransmissionAudit'),
+            ('sap_success_factors', 'SapSuccessFactorsLearnerDataTransmissionAudit'),
+        ]
+        for app_label, model_name in channel_learner_audit_models:
+            model_class = apps.get_model(app_label=app_label, model_name=model_name)
+
+            latest_records_without_prefix = (
+                model_class.objects.exclude(course_id__startswith='course-v1:')
+                .values('enterprise_course_enrollment_id').annotate(most_recent_transmission_id=Max('id'))
+            )
+
+            LOGGER.info(
+                f'{app_label} channel has {latest_records_without_prefix.count()} records without prefix'
+            )
+
+            # Delete all duplicate records for each enterprise_course_enrollment_id
+            with transaction.atomic():
+                for entry in latest_records_without_prefix:
+                    enterprise_course_enrollment_id = entry['enterprise_course_enrollment_id']
+                    most_recent_transmission_id = entry['most_recent_transmission_id']
+
+                    # Delete all records except the latest one without "course-v1:"
+                    duplicate_records_to_delete = (
+                        model_class.objects
+                        .filter(enterprise_course_enrollment_id=enterprise_course_enrollment_id)
+                        .exclude(id=most_recent_transmission_id)
+                    )
+                    LOGGER.info(
+                        f'{app_label} channel - {duplicate_records_to_delete.count()} duplicate records are deleted'
+                    )
+                    duplicate_records_to_delete.delete()

integrated_channels/integrated_channel/migrations/0031_alter_integratedchannelapirequestlogs_options.py

@@ -1,6 +1,6 @@
 # Generated by Django 3.2.22 on 2024-01-25 09:25
 
-from django.db import migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -14,4 +14,23 @@ class Migration(migrations.Migration):
             name='integratedchannelapirequestlogs',
             options={'verbose_name_plural': 'Integrated channels API request logs'},
         ),
+        migrations.RemoveField(
+            model_name='integratedchannelapirequestlogs',
+            name='api_record',
+        ),
+        migrations.AddField(
+            model_name='integratedchannelapirequestlogs',
+            name='response_body',
+            field=models.TextField(blank=True, help_text='API call response body', null=True),
+        ),
+        migrations.AddField(
+            model_name='integratedchannelapirequestlogs',
+            name='status_code',
+            field=models.PositiveIntegerField(blank=True, help_text='API call response HTTP status code', null=True),
+        ),
+        migrations.AlterField(
+            model_name='integratedchannelapirequestlogs',
+            name='time_taken',
+            field=models.FloatField(),
+        ),
     ]