Add stacks related to dispatch Lambda

config/develop/namespaced/lambda-dispatch-role.yaml

@@ -0,0 +1,13 @@
+template:
+  path: lambda-dispatch-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch-role"
+dependencies:
+  - develop/namespaced/sqs-input-to-dispatch.yaml
+  - develop/namespaced/sns-dispatch.yaml
+  - develop/s3-cloudformation-bucket.yaml
+parameters:
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/lambda-dispatch.yaml

@@ -0,0 +1,15 @@
+template:
+  type: sam
+  path: src/lambda_function/dispatch/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - develop/namespaced/lambda-dispatch-role.yaml
+  - develop/namespaced/sqs-input-to-dispatch.yaml
+  - develop/s3-cloudformation-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch-role::RoleArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  DispatchSnsArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags: {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sns-dispatch-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch-policy"
+dependencies:
+  - develop/namespaced/lambda-dispatch.yaml
+  - develop/namespaced/sns-dispatch.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+  LambdaSourceArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch::DispatchFunctionArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sns-dispatch.yaml

@@ -0,0 +1,5 @@
+template:
+  path: sns-topic.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sns-input-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-input-policy"
+dependencies:
+  - develop/s3-input-bucket.yaml
+  - develop/namespaced/sns-input.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
+  S3SourceBucketArn: !stack_output_external "recover-dev-input-bucket::BucketArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sns-input.yaml

@@ -1,9 +1,5 @@
 template:
   path: sns-topic.yaml
-parameters:
-  S3SourceBucketArn: !stack_output_external recover-dev-input-bucket::BucketArn
-dependencies:
-  - develop/s3-input-bucket.yaml
 stack_name: "{{ stack_group_config.namespace }}-sns-input"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/sqs-input-to-dispatch.yaml

@@ -7,6 +7,6 @@ parameters:
   SNSTopicSubscription: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
 dependencies:
   - develop/namespaced/sns-input.yaml
-stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-dispatch'
+stack_name: "{{ stack_group_config.namespace }}-sqs-input-to-dispatch"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/lambda-dispatch-role.yaml

@@ -0,0 +1,13 @@
+template:
+  path: lambda-dispatch-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch-role"
+dependencies:
+  - prod/namespaced/sqs-input-to-dispatch.yaml
+  - prod/namespaced/sns-dispatch.yaml
+  - prod/s3-cloudformation-bucket.yaml
+parameters:
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  SNSTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/lambda-dispatch.yaml

@@ -0,0 +1,15 @@
+template:
+  type: sam
+  path: src/lambda_function/dispatch/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - prod/namespaced/lambda-dispatch-role.yaml
+  - prod/namespaced/sqs-input-to-dispatch.yaml
+  - prod/s3-cloudformation-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-dispatch"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch-role::RoleArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-dispatch::PrimaryQueueArn"
+  DispatchSnsArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+stack_tags: {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sns-dispatch-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch-policy"
+dependencies:
+  - prod/namespaced/lambda-dispatch.yaml
+  - prod/namespaced/sns-dispatch.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-dispatch::SnsTopicArn"
+  LambdaSourceArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-dispatch::DispatchFunctionArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sns-dispatch.yaml

@@ -0,0 +1,5 @@
+template:
+  path: sns-topic.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-dispatch"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sns-input-policy.yaml

@@ -0,0 +1,11 @@
+template:
+  path: sns-topic-policy.yaml
+stack_name: "{{ stack_group_config.namespace }}-sns-input-policy"
+dependencies:
+  - prod/s3-input-bucket.yaml
+  - prod/namespaced/sns-input.yaml
+parameters:
+  SnsTopicArn: !stack_output_external "{{ stack_group_config.namespace }}-sns-input::SnsTopicArn"
+  S3SourceBucketArn: !stack_output_external "recover-input-bucket::BucketArn"
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/prod/namespaced/sns-input.yaml

@@ -1,9 +1,5 @@
 template:
   path: sns-topic.yaml
-parameters:
-  S3SourceBucketArn: !stack_output_external recover-input-bucket::BucketArn
-dependencies:
-  - prod/s3-input-bucket.yaml
 stack_name: "{{ stack_group_config.namespace }}-sns-input"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}

src/lambda_function/dispatch/README.md

@@ -1,7 +1,7 @@
 # Dispatch Lambda
 
 The dispatch Lambda polls the input-to-dispatch SQS queue and publishes to the dispatch SNS topic.
-Its purpose is to inspect the each export and dispatch each file as a separate job to be consumed
+Its purpose is to inspect an export and dispatch each file as a separate job to eventually be consumed
 by the dispatch-to-raw Lambda.
 
 ## Development
@@ -18,8 +18,7 @@ To use the SAM CLI, you need the following tools.
 You may need the following for local testing.
 * [Python 3 installed](https://www.python.org/downloads/)
 
-You will also need to configure your AWS credentials,
-if you have not already done so.
+You will also need to configure your AWS credentials, if you have not already done so.
 
 ## Creating a local build
 
@@ -31,21 +30,7 @@ cd src/lambda_function/dispatch/
 sam build
 ```
 
-## Test events
+## Tests
 
-### Creating/modifying test events
-
-
-### Invoking test events
-
-To invoke the lambda with the test event:
-
-```bash
-cd src/lambda_function/dispatch
-sam local invoke
-```
-
-## Launching Lambda stack in AWS
-
-There are two stacks relevant to this Lambda: `dispatch-lambda` and `dispatch-lambda-role`.
+Tests are available in `tests/test_dispatch_lambda.py`.
 

src/lambda_function/dispatch/template.yaml

@@ -25,7 +25,7 @@ Parameters:
   LambdaPythonVersion:
     Type: String
     Description: Python version to use for this lambda function
-    Default: "3.11"
+    Default: "3.9"
 
   LambdaBatchSize:
     Type: Number

templates/lambda-dispatch-role.yaml

@@ -0,0 +1,78 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Transform: AWS::Serverless-2016-10-31
+
+Description: >
+  An IAM Role for the dispatch lambda
+
+Parameters:
+  SQSQueueArn:
+    Type: String
+    Description: ARN of the SQS queue for lambda to poll messages from.
+
+  S3SourceBucketName:
+    Type: String
+    Description: Name of the S3 bucket where exports are deposited.
+
+  SNSTopicArn:
+    Type: String
+    Description: ARN of the SNS topic which dispatched jobs will be published to.
+
+Resources:
+  DispatchRole:
+      Type: AWS::IAM::Role
+      Properties:
+        AssumeRolePolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+              - lambda.amazonaws.com
+            Action:
+            - sts:AssumeRole
+        ManagedPolicyArns:
+          - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+        Policies:
+        - PolicyName: PollSQSQueue
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - sqs:DeleteMessage
+              - sqs:GetQueueAttributes
+              - sqs:ReceiveMessage
+              Resource:
+              - !Ref SQSQueueArn
+        - PolicyName: ReadS3
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - s3:Get*
+              - s3:List*
+              Resource:
+              - !Sub arn:aws:s3:::${S3SourceBucketName}
+              - !Sub arn:aws:s3:::${S3SourceBucketName}/*
+        - PolicyName: PublishToSNS
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - sns:Publish
+                Resource:
+                  - !Ref SNSTopicArn
+
+Outputs:
+  RoleName:
+    Value: !Ref DispatchRole
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleName'
+
+  RoleArn:
+    Value: !GetAtt DispatchRole.Arn
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleArn'

templates/sns-topic-policy.yaml

@@ -0,0 +1,73 @@
+AWSTemplateFormatVersion: "2010-09-09"
+
+Description: >
+  Allows an SNS Topic to be used with S3 event notifications and/or Lambda triggers.
+
+Parameters:
+
+  SnsTopicArn:
+    Type: String
+    Description: The ARN of the SNS topic to apply this policy to.
+
+  S3SourceBucketArn:
+    Type: String
+    Default: ""
+    Description: >
+      ARN of an S3 bucket where source data are stored. Specifying this
+      parameter enables this SNS topic to have S3 event notifications published to it.
+
+  LambdaSourceArn:
+    Type: String
+    Default: ""
+    Description: ARN of the Lambda function which can trigger this SNS topic. Specifying this
+      parameter enables this SNS topic to have a Lambda publish to it.
+
+Conditions:
+
+  HasS3SourceBucketArn:
+    !Not [!Equals [!Ref S3SourceBucketArn, ""]]
+
+  HasLambdaSourceArn:
+    !Not [!Equals [!Ref LambdaSourceArn, ""]]
+
+Resources:
+
+  SnsTopicPolicyForS3:
+    Type: AWS::SNS::TopicPolicy
+    Condition: HasS3SourceBucketArn
+    Properties:
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: Input S3 bucket event notification to SNS
+            Effect: Allow
+            Principal:
+              Service: s3.amazonaws.com
+            Action:
+              - sns:Publish
+            Resource: !Ref SnsTopicArn
+            Condition:
+              ArnLike:
+                "aws:SourceArn": !Ref S3SourceBucketArn
+      Topics:
+        - !Ref SnsTopicArn
+
+  SnsTopicPolicyForLambda:
+    Type: AWS::SNS::TopicPolicy
+    Condition: HasLambdaSourceArn
+    Properties:
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: Lambda trigger to SNS
+            Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action:
+              - sns:Publish
+            Resource: !Ref SnsTopicArn
+            Condition:
+              ArnLike:
+                "aws:SourceArn": !Ref LambdaSourceArn
+      Topics:
+        - !Ref SnsTopicArn

templates/sns-topic.yaml

@@ -1,13 +1,7 @@
 AWSTemplateFormatVersion: "2010-09-09"
 
 Description: >
-  Creates a Standard SNS queue which can be used with S3 event notifications.
-
-Parameters:
-
-  S3SourceBucketArn:
-    Type: String
-    Description: Arn of the S3 bucket where source data are stored.
+  Creates a Standard SNS topic.
 
 Resources:
 
@@ -17,26 +11,6 @@ Resources:
       TopicName: !Sub "${AWS::StackName}-Topic"
       FifoTopic: false
 
-  SnsTopicPolicy:
-    Type: AWS::SNS::TopicPolicy
-    Properties:
-      PolicyDocument:
-        Version: "2012-10-17"
-        Statement:
-          - Sid: Input S3 bucket event notification to SNS
-            Effect: Allow
-            Principal:
-              Service: s3.amazonaws.com
-              AWS: !Sub '${AWS::AccountId}'
-            Action:
-            - sns:Publish
-            Resource: !Ref SnsTopic
-            Condition:
-              ArnLike:
-                "aws:SourceArn": !Ref S3SourceBucketArn
-      Topics:
-        - !Ref SnsTopic
-
 Outputs:
 
   SnsTopicArn: