feat(CRON): log file archival job

Author: niall-byrne

pi_portal/config.py

@@ -8,6 +8,7 @@
 )
 
 CRON_INTERVAL_DEAD_MAN_SWITCH = 60 * 5
+CRON_INTERVAL_LOGS_UPLOAD = 60 * 30
 CRON_INTERVAL_VIDEO_UPLOAD = 10
 
 FILE_BEAT_BINARY = os.getenv("PI_PORTAL_FILEBEAT_LOCATION", "/usr/bin/filebeat")

pi_portal/modules/cron/jobs/__init__.py

@@ -3,9 +3,10 @@
 from typing import List, Type
 
 from ..bases.job import CronJobBase
-from . import dead_man_switch, video_upload
+from . import dead_man_switch, log_upload, video_upload
 
 cron_jobs: List[Type[CronJobBase]] = [
     dead_man_switch.DeadManSwitchCronJob,
+    log_upload.LogFileUploadCronJob,
     video_upload.VideoUploadCronJob,
 ]

pi_portal/modules/cron/jobs/log_upload.py

@@ -0,0 +1,42 @@
+"""LogFileUploadCronJob class."""
+
+import logging
+import os
+from datetime import datetime
+
+from pi_portal import config
+from pi_portal.modules.configuration import state
+from ..bases import s3_upload_job
+
+
+class LogFileUploadCronJob(s3_upload_job.S3UploadCronJobBase):
+  """S3 upload cron for log file archival.
+
+  :param log: The logging instance for this cron job.
+  """
+
+  interval = config.CRON_INTERVAL_LOGS_UPLOAD
+  name = "Log File Upload"
+  path = config.PATH_QUEUE_LOG_UPLOAD
+
+  def __init__(self, log: logging.Logger) -> None:
+    running_state = state.State()
+    aws_config = running_state.user_config["ARCHIVAL"]["AWS"]
+    self.bucket_name = aws_config["AWS_S3_BUCKETS"]["LOGS"]
+    super().__init__(log)
+
+  def object_name(self, file_name: str) -> str:
+    """Extract a timestamp and logfile name.
+
+    :param file_name: The name of the file being processed.
+    :returns: The S3 object name that will be used.
+    """
+    base_name = os.path.basename(file_name)
+    parsed_file_name = base_name.split("_")
+    timestamp = datetime.fromisoformat(parsed_file_name[0])
+    log_name = "".join(parsed_file_name[1:])
+
+    return os.path.join(
+        timestamp.date().isoformat(),
+        f"{timestamp.timetz().isoformat()}_{log_name}"
+    )

pi_portal/modules/cron/jobs/tests/conftest.py

@@ -4,7 +4,7 @@
 
 import pytest
 from pi_portal.modules.configuration.tests.fixtures import mock_state
-from .. import dead_man_switch, video_upload
+from .. import dead_man_switch, log_upload, video_upload
 
 
 @pytest.fixture
@@ -14,6 +14,15 @@ def dead_man_switch_cron_job_instance(
   return dead_man_switch.DeadManSwitchCronJob(mocked_cron_logger)
 
 
+@pytest.fixture
+def log_upload_cron_job_instance(
+    mocked_cron_logger: logging.Logger
+) -> log_upload.LogFileUploadCronJob:
+  with mock_state.mock_state_creator():
+    instance = log_upload.LogFileUploadCronJob(mocked_cron_logger)
+  return instance
+
+
 @pytest.fixture
 def video_upload_cron_job_instance(
     mocked_cron_logger: logging.Logger

pi_portal/modules/cron/jobs/tests/dead_man_switch_test.py

@@ -13,7 +13,7 @@ class TestDeadManSwitchCronJob:
   def test__initialization__attrs(
       self,
       dead_man_switch_cron_job_instance: dead_man_switch.DeadManSwitchCronJob,
-      mocked_cron_logger: logging.Logger
+      mocked_cron_logger: logging.Logger,
   ) -> None:
     assert dead_man_switch_cron_job_instance.interval == \
            config.CRON_INTERVAL_DEAD_MAN_SWITCH

pi_portal/modules/cron/jobs/tests/log_upload_test.py

@@ -0,0 +1,84 @@
+"""Test the LogFileUploadCronJob class."""
+import logging
+import os
+from datetime import datetime, timezone
+
+from pi_portal import config
+from pi_portal.modules.configuration import state
+from pi_portal.modules.integrations.folder import queue
+from pi_portal.modules.integrations.s3 import client as s3_client
+from ...bases import s3_upload_job
+from .. import log_upload
+
+
+class TestLogFileUploadCronJob:
+  """Test the LogFileUploadCronJob class."""
+
+  def test__initialization__attrs(
+      self,
+      log_upload_cron_job_instance: log_upload.LogFileUploadCronJob,
+      mocked_cron_logger: logging.Logger,
+      mocked_state: state.State,
+  ) -> None:
+    aws_config = mocked_state.user_config["ARCHIVAL"]["AWS"]
+    assert log_upload_cron_job_instance.bucket_name == \
+           aws_config["AWS_S3_BUCKETS"]["LOGS"]
+    assert log_upload_cron_job_instance.interval == \
+           config.CRON_INTERVAL_LOGS_UPLOAD
+    assert log_upload_cron_job_instance.log == \
+           mocked_cron_logger
+    assert log_upload_cron_job_instance.name == \
+           "Log File Upload"
+    assert log_upload_cron_job_instance.path == \
+           config.PATH_QUEUE_LOG_UPLOAD
+
+  def test__initialization__inheritance(
+      self,
+      log_upload_cron_job_instance: log_upload.LogFileUploadCronJob,
+  ) -> None:
+    assert isinstance(
+        log_upload_cron_job_instance,
+        s3_upload_job.S3UploadCronJobBase,
+    )
+
+  def test__initialization__disk_queue(
+      self,
+      log_upload_cron_job_instance: log_upload.LogFileUploadCronJob,
+  ) -> None:
+    assert isinstance(
+        log_upload_cron_job_instance.disk_queue,
+        queue.DiskQueueIterator,
+    )
+    assert log_upload_cron_job_instance.disk_queue.path == \
+        log_upload_cron_job_instance.path
+
+  def test__initialization__s3_client(
+      self,
+      log_upload_cron_job_instance: log_upload.LogFileUploadCronJob,
+      mocked_state: state.State,
+  ) -> None:
+    aws_config = mocked_state.user_config["ARCHIVAL"]["AWS"]
+    assert isinstance(
+        log_upload_cron_job_instance.s3_client,
+        s3_client.S3BucketClient,
+    )
+    assert log_upload_cron_job_instance.s3_client.bucket_name == \
+        aws_config["AWS_S3_BUCKETS"]["LOGS"]
+
+  def test__object_name__correct_object_name(
+      self,
+      log_upload_cron_job_instance: log_upload.LogFileUploadCronJob,
+  ) -> None:
+    timestamp = datetime.now().replace(tzinfo=timezone.utc)
+    lof_file_name = "mock.log"
+    archival_name = os.path.join(
+        config.PATH_QUEUE_LOG_UPLOAD,
+        f"{timestamp.isoformat()}_{lof_file_name}",
+    )
+
+    object_name = log_upload_cron_job_instance.object_name(archival_name)
+
+    assert object_name == os.path.join(
+        timestamp.date().isoformat(),
+        f"{timestamp.timetz().isoformat()}_{lof_file_name}",
+    )