Hl 1269 versionseriesid (#2935)

* feat: add datetime when ahjo has downloaded a file

* feat: send any new records to ahjo before decision

* feat: quarter hourly cron job to  send new records

* fix: use update_or_create for pdf_summary

* feat: send only the application pdf on update

* chore: dump into json data only once

* fix: conflicting migration

* fix: use types and titles for decision in payload

* chore: newline in quarter_hourly_job

* feat: reformat record titles for AHJO

* fix: call with correct parameters

* chore: add missing migration file

* fix: comparison with in instead of ==

.env.benefit-backend.example

@@ -118,3 +118,4 @@ AHJO_TEST_USER_AD_USERNAME =
 ENABLE_CLAMAV=1
 CLAMAV_URL=http://localhost:8080/api/v1
 AHJO_REQUEST_TIMEOUT=60
+ENABLE_AHJO_AUTOMATION=0

backend/benefit/applications/api/v1/ahjo_integration_views.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import datetime, timezone
 
 from django.http import FileResponse
 from django.shortcuts import get_object_or_404
@@ -58,6 +59,8 @@ def get(self, request, *args, **kwargs):
             additional_information=f"attachment {attachment.attachment_file} \
 of type {attachment.attachment_type} was sent to AHJO!",
         )
+        attachment.downloaded_by_ahjo = datetime.now(timezone.utc)
+        attachment.save()
         return self._prepare_file_response(attachment)
 
     @staticmethod
@@ -151,9 +154,13 @@ def handle_success_callback(
                 ahjo_status = AhjoStatusEnum.DECISION_PROPOSAL_ACCEPTED
                 info = "Decision proposal was sent to Ahjo"
             elif request_type == AhjoRequestType.UPDATE_APPLICATION:
-                self._handle_update_records_success(application, callback_data)
+                self._handle_update_or_add_records_success(application, callback_data)
                 ahjo_status = AhjoStatusEnum.UPDATE_REQUEST_RECEIVED
                 info = f"Updated application records were sent to Ahjo with request id: {callback_data['requestId']}"
+            elif request_type == AhjoRequestType.ADD_RECORDS:
+                self._handle_update_or_add_records_success(application, callback_data)
+                ahjo_status = AhjoStatusEnum.NEW_RECORDS_RECEIVED
+                info = f"A attachments were sent as records to Ahjo with request id: {callback_data['requestId']}"
             else:
                 raise AhjoCallbackError(
                     f"Unknown request type {request_type} in the Ahjo callback"
@@ -183,7 +190,7 @@ def handle_failure_callback(
             status=status.HTTP_400_BAD_REQUEST,
         )
 
-    def _handle_update_records_success(
+    def _handle_update_or_add_records_success(
         self, application: Application, callback_data: dict
     ):
         cb_records = callback_data.get("records", [])
@@ -213,7 +220,9 @@ def _save_version_series_id(
             if the calculated sha256 hashes match."""
         attachment_map = {
             attachment.ahjo_hash_value: attachment
-            for attachment in application.attachments.all()
+            for attachment in application.attachments.filter(
+                ahjo_hash_value__isnull=False
+            )
         }
         for cb_record in cb_records:
             attachment = attachment_map.get(cb_record.get("hashValue"))

backend/benefit/applications/api/v1/application_views.py

@@ -257,6 +257,7 @@ def post_attachment(self, request, *args, **kwargs):
         )
         serializer.is_valid(raise_exception=True)
         serializer.save()
+
         return Response(serializer.data, status=status.HTTP_201_CREATED)
 
     def _get_simplified_queryset(self, request, context) -> QuerySet:

backend/benefit/applications/enums.py

@@ -175,6 +175,8 @@ class AhjoStatus(models.TextChoices):
     )
     DELETE_REQUEST_SENT = "delete_request_sent", _("Delete request sent")
     DELETE_REQUEST_RECEIVED = "delete_request_received", _("Delete request received")
+    NEW_RECORDS_REQUEST_SENT = "new_record_request_sent", _("New record request sent")
+    NEW_RECORDS_RECEIVED = "new_record_received", _("New record received by Ahjo")
 
 
 class ApplicationActions(models.TextChoices):
@@ -192,6 +194,7 @@ class AhjoRequestType(models.TextChoices):
     OPEN_CASE = "open_case", _("Open case in Ahjo")
     DELETE_APPLICATION = "delete_application", _("Delete application in Ahjo")
     UPDATE_APPLICATION = "update_application", _("Update application in Ahjo")
+    ADD_RECORDS = "add_records", _("Send new records to Ahjo")
     SEND_DECISION_PROPOSAL = "send_decision", _("Send decision to Ahjo")
 
 
@@ -222,6 +225,23 @@ class ApplicationReviewStep(models.TextChoices):
     STEP_3 = "step_3", _("Step 3 - review the decision")
 
 
+class AhjoRecordType(models.TextChoices):
+    APPLICATION = "hakemus", _("Application")
+    ATTACHMENT = "hakemuksen liite", _("Application attachment")
+    DECISION_PROPOSAL = "viranhaltijan päätös", _("Decision proposal")
+    SECRET_ATTACHMENT = "viranhaltijan päätöksen liite", _("Secret decision attachment")
+
+
+class AhjoRecordTitle(models.TextChoices):
+    APPLICATION = "Hakemus", _("Application title")
+    ATTACHMENT = "Hakemuksen liite", _("Application attachmen title")
+    DECISION_PROPOSAL = (
+        "Avustuksen myöntäminen, Työllisyyspalvelut, työllisyydenhoidon Helsinki-lisä vuonna 2024",
+        _("Decision proposal title"),
+    )
+    SECRET_ATTACHMENT = "Päätöksen liite", _("Secret decision attachment title")
+
+
 # Call gettext on some of the enums so that "makemessages" command can find them when used dynamically in templates
 _("granted")
 _("granted_aged")

backend/benefit/applications/jobs/quarter_hourly/quarter_hourly_ahjo_job.py

@@ -0,0 +1,11 @@
+from django.conf import settings
+from django.core.management import call_command
+from django_extensions.management.jobs import QuarterHourlyJob
+
+
+class Job(QuarterHourlyJob):
+    help = "Quarter hourly Ahjo integration jobs are executed here."
+
+    def execute(self):
+        if settings.ENABLE_AHJO_AUTOMATION:
+            call_command("send_new_records")

backend/benefit/applications/management/commands/send_new_records.py

@@ -0,0 +1,39 @@
+import logging
+import time
+
+from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import BaseCommand
+
+from applications.services.ahjo_authentication import AhjoToken
+from applications.services.ahjo_integration import (
+    get_token,
+    send_new_attachment_records_to_ahjo,
+)
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Send new records to Ahjo"
+
+    def handle(self, *args, **options):
+        try:
+            ahjo_auth_token = get_token()
+        except ImproperlyConfigured as e:
+            LOGGER.error(f"Failed to get auth token from Ahjo: {e}")
+            return
+
+        self.run_requests(ahjo_auth_token)
+
+    def run_requests(self, ahjo_auth_token: AhjoToken):
+        start_time = time.time()
+
+        self.stdout.write("Sending new records to Ahjo")
+
+        responses = send_new_attachment_records_to_ahjo(ahjo_auth_token)
+        self.stdout.write(f"Sent records for {len(responses)} applications to Ahjo")
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+        self.stdout.write(
+            f"Done! Sending new records for {len(responses)} applications {elapsed_time} seconds to run."
+        )

backend/benefit/applications/migrations/0065_downloaded_by_ahjo_timestamp.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.23 on 2024-04-12 10:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0064_decision_proposal_drafts'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='attachment',
+            name='downloaded_by_ahjo',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalattachment',
+            name='downloaded_by_ahjo',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

backend/benefit/applications/migrations/0066_alter_ahjostatus_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-04-17 13:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0065_downloaded_by_ahjo_timestamp'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ahjostatus',
+            name='status',
+            field=models.CharField(choices=[('submitted_but_not_sent_to_ahjo', 'Submitted but not sent to AHJO'), ('request_to_open_case_sent', 'Request to open the case sent to AHJO'), ('case_opened', 'Case opened in AHJO'), ('update_request_sent', 'Update request sent'), ('update_request_received', 'Update request received'), ('decision_proposal_sent', 'Decision proposal sent'), ('decision_proposal_accepted', 'Decision proposal accepted'), ('decision_proposal_rejected', 'Decision proposal rejected'), ('delete_request_sent', 'Delete request sent'), ('delete_request_received', 'Delete request received'), ('new_record_request_sent', 'New record request sent'), ('new_record_received', 'New record received by Ahjo')], default='submitted_but_not_sent_to_ahjo', max_length=64, verbose_name='status'),
+        ),
+    ]

backend/benefit/applications/models.py

@@ -3,7 +3,7 @@
 from dateutil.relativedelta import relativedelta
 from django.conf import settings
 from django.db import connection, models
-from django.db.models import JSONField, OuterRef, Subquery
+from django.db.models import JSONField, OuterRef, Prefetch, Subquery
 from django.db.models.constraints import UniqueConstraint
 from django.utils.translation import gettext_lazy as _
 from encrypted_fields.fields import EncryptedCharField, SearchField
@@ -121,6 +121,29 @@ def get_queryset(self):
         )
         return qs
 
+    def with_downloaded_attachments(self):
+        """
+        Returns applications with only those attachments that have
+        null in 'downloaded_by_ahjo' and where  the applications have a non-null ahjo_case_id,
+        which means that a case has been opened for them in AHJO.
+        """
+
+        qs = self.get_queryset().filter(ahjo_case_id__isnull=False)
+        attachments_queryset = Attachment.objects.filter(
+            downloaded_by_ahjo__isnull=True,
+            attachment_type__in=[
+                AttachmentType.EMPLOYMENT_CONTRACT,
+                AttachmentType.PAY_SUBSIDY_DECISION,
+                AttachmentType.COMMISSION_CONTRACT,
+                AttachmentType.EDUCATION_CONTRACT,
+                AttachmentType.HELSINKI_BENEFIT_VOUCHER,
+                AttachmentType.EMPLOYEE_CONSENT,
+                AttachmentType.OTHER_ATTACHMENT,
+            ],
+        )
+        attachments_prefetch = Prefetch("attachments", queryset=attachments_queryset)
+        return qs.prefetch_related(attachments_prefetch)
+
 
 class Application(UUIDModel, TimeStampedModel, DurationMixin):
     """
@@ -902,6 +925,8 @@ class Attachment(UUIDModel, TimeStampedModel):
 
     ahjo_hash_value = models.CharField(max_length=64, null=True, blank=True)
 
+    downloaded_by_ahjo = models.DateTimeField(null=True, blank=True)
+
     history = HistoricalRecords(
         table_name="bf_applications_attachment_history", cascade_delete_history=True
     )

backend/benefit/applications/services/ahjo_integration.py

@@ -26,6 +26,7 @@
 from applications.models import AhjoDecisionText, AhjoStatus, Application, Attachment
 from applications.services.ahjo_authentication import AhjoConnector, AhjoToken
 from applications.services.ahjo_payload import (
+    prepare_attachment_records_payload,
     prepare_decision_proposal_payload,
     prepare_open_case_payload,
     prepare_update_application_payload,
@@ -386,7 +387,7 @@ def export_application_batch(batch) -> bytes:
 def generate_application_attachment(
     application: Application, type: AttachmentType
 ) -> Attachment:
-    """Generate a xml decision of the given application and return it as an Attachment."""
+    """Generate and save an Attachment of the requested type for the given application"""
     if type == AttachmentType.PDF_SUMMARY:
         attachment_data = generate_application_summary_file(application)
         attachment_filename = (
@@ -413,12 +414,20 @@ def generate_application_attachment(
         raise ValueError(f"Invalid attachment type {type}")
 
     attachment_file = ContentFile(attachment_data, attachment_filename)
-    attachment = Attachment.objects.create(
-        application=application,
-        attachment_file=attachment_file,
-        content_type=content_type,
-        attachment_type=type,
-    )
+    if type == AttachmentType.PDF_SUMMARY:
+        # As there should only exist one pdf summary, update or create the attachment if it is one
+        attachment, _ = Attachment.objects.update_or_create(
+            application=application,
+            attachment_type=type,
+            defaults={"attachment_file": attachment_file, "content_type": content_type},
+        )
+    else:
+        attachment = Attachment.objects.create(
+            application=application,
+            attachment_file=attachment_file,
+            content_type=content_type,
+            attachment_type=type,
+        )
     return attachment
 
 
@@ -448,7 +457,7 @@ def prepare_headers(
 
 
 def get_application_for_ahjo(id: uuid.UUID) -> Optional[Application]:
-    """Get the first accepted application with attachment, calculation and company."""
+    """Get the application with calculation, company and employee."""
     application = Application.objects.select_related(
         "calculation", "company", "employee"
     ).get(pk=id)
@@ -511,25 +520,28 @@ def send_request_to_ahjo(
     timeout = settings.AHJO_REQUEST_TIMEOUT
     headers["Content-Type"] = "application/json"
 
+    data = json.dumps(data, default=str)
+
     url_base = f"{settings.AHJO_REST_API_URL}/cases"
 
     if request_type == AhjoRequestType.OPEN_CASE:
         method = "POST"
         api_url = url_base
-        data = json.dumps(data)
     elif request_type == AhjoRequestType.DELETE_APPLICATION:
         method = "DELETE"
         api_url = prepare_delete_url(url_base, application)
+        data = None
 
     elif request_type == AhjoRequestType.UPDATE_APPLICATION:
         method = "PUT"
-        data = json.dumps(data)
         api_url = f"{url_base}/{application.ahjo_case_id}/records"
 
-    elif request_type == AhjoRequestType.SEND_DECISION_PROPOSAL:
+    elif request_type in [
+        AhjoRequestType.SEND_DECISION_PROPOSAL,
+        AhjoRequestType.ADD_RECORDS,
+    ]:
         method = "POST"
         api_url = f"{url_base}/{application.ahjo_case_id}/records"
-        data = json.dumps(data)
 
     try:
         response = requests.request(
@@ -606,7 +618,12 @@ def delete_application_in_ahjo(application_id: uuid.UUID):
         LOGGER.error(f"Improperly configured: {e}")
 
 
-def update_application_in_ahjo(application: Application, ahjo_auth_token: str):
+def update_application_summary_record_in_ahjo(
+    application: Application, ahjo_auth_token: str
+):
+    """Update the application summary pdf in Ahjo.
+    Should be done just before the decision proposal is sent.
+    """
     try:
         headers = prepare_headers(
             ahjo_auth_token, application, AhjoRequestType.UPDATE_APPLICATION
@@ -615,7 +632,7 @@ def update_application_in_ahjo(application: Application, ahjo_auth_token: str):
         pdf_summary = generate_application_attachment(
             application, AttachmentType.PDF_SUMMARY
         )
-        data = prepare_update_application_payload(application, pdf_summary)
+        data = prepare_update_application_payload(pdf_summary, application)
 
         result = send_request_to_ahjo(
             AhjoRequestType.UPDATE_APPLICATION,
@@ -633,6 +650,38 @@ def update_application_in_ahjo(application: Application, ahjo_auth_token: str):
         LOGGER.error(f"Improperly configured: {e}")
 
 
+def send_new_attachment_records_to_ahjo(
+    token: AhjoToken,
+) -> List[Tuple[Application, str]]:
+    """Send any new attachments, that have been added after opening a case, to Ahjo."""
+    try:
+        applications = Application.objects.with_downloaded_attachments()
+        # TODO add a check for application status,
+        # so that only applications in the correct status have their attachments sent
+        responses = []
+        for application in applications:
+            attachments = application.attachments.all()
+
+            headers = prepare_headers(
+                token.access_token, application, AhjoRequestType.ADD_RECORDS
+            )
+
+            data = prepare_attachment_records_payload(attachments, application)
+
+            application, response_text = send_request_to_ahjo(
+                AhjoRequestType.ADD_RECORDS, headers, application, data
+            )
+            responses.append((application, response_text))
+            create_status_for_application(
+                application, AhjoStatusEnum.NEW_RECORDS_REQUEST_SENT
+            )
+        return responses
+    except ObjectDoesNotExist as e:
+        LOGGER.error(f"Object not found: {e}")
+    except ImproperlyConfigured as e:
+        LOGGER.error(f"Improperly configured: {e}")
+
+
 def send_decision_proposal_to_ahjo(application_id: uuid.UUID):
     """Open a case in Ahjo."""
     try: