raw lambda initial commit

config/develop/namespaced/lambda-raw-role.yaml

@@ -0,0 +1,13 @@
+template:
+  path: lambda-raw-role.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw-role"
+dependencies:
+  - develop/namespaced/sqs-dispatch-to-raw.yaml
+  - develop/s3-cloudformation-bucket.yaml
+  - develop/s3-raw-bucket.yaml
+parameters:
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-dispatch-to-raw::PrimaryQueueArn"
+  S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
+  S3RawBucket: {{ stack_group_config.raw_bucket_name }}
+stack_tags:
+  {{ stack_group_config.default_stack_tags }}

config/develop/namespaced/lambda-raw.yaml

@@ -0,0 +1,16 @@
+template:
+  type: sam
+  path: src/lambda_function/raw/template.yaml
+  artifact_bucket_name: {{ stack_group_config.template_bucket_name }}
+  artifact_prefix: "{{ stack_group_config.namespace }}/src/lambda"
+dependencies:
+  - develop/namespaced/lambda-raw-role.yaml
+  - develop/namespaced/sqs-dispatch-to-raw.yaml
+  - develop/s3-cloudformation-bucket.yaml
+  - develop/s3-raw-bucket.yaml
+stack_name: "{{ stack_group_config.namespace }}-lambda-raw"
+parameters:
+  RoleArn: !stack_output_external "{{ stack_group_config.namespace }}-lambda-raw-role::RoleArn"
+  SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-dispatch-to-raw::PrimaryQueueArn"
+  S3RawBucket: {{ stack_group_config.raw_bucket_name }}
+stack_tags: {{ stack_group_config.default_stack_tags }}

src/lambda_function/raw/README.md

@@ -0,0 +1,34 @@
+# Raw Lambda
+
+The raw Lambda polls the dispatch-to-raw SQS queue and uploads an object to the raw S3 bucket.
+Its purpose is to decompress a single file from an export, recompress that file, and store it to S3.
+
+## Development
+
+The Serverless Application Model Command Line Interface (SAM CLI) is an
+extension of the AWS CLI that adds functionality for building and testing
+Lambda applications.
+
+To use the SAM CLI, you need the following tools.
+
+* SAM CLI - [Install the SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html)
+* Docker - [Install Docker community edition](https://hub.docker.com/search/?type=edition&offering=community)
+
+You may need the following for local testing.
+* [Python 3 installed](https://www.python.org/downloads/)
+
+You will also need to configure your AWS credentials, if you have not already done so.
+
+## Creating a local build
+
+Use the SAM CLI to build and test your lambda locally.
+Build your application with the `sam build` command.
+
+```bash
+cd src/lambda_function/raw/
+sam build
+```
+
+## Tests
+
+Tests are available in `tests/test_raw_lambda.py`.

src/lambda_function/raw/app.py

@@ -0,0 +1,8 @@
+import json
+
+import boto3
+
+
+def lambda_handler(event, context):
+    s3_client = boto3.client("s3")
+    return {"statusCode": 200, "body": json.dumps("Hello from Lambda!")}

src/lambda_function/raw/template.yaml

@@ -0,0 +1,70 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Transform: AWS::Serverless-2016-10-31
+
+Description: >
+  SAM Template for the raw Lambda. The raw Lambda polls the dispatch-to-raw SQS
+  queue and uploads an object to the raw S3 bucket. Its purpose is to decompress
+  a single file from an export, recompress that file, and store it to S3.
+
+Parameters:
+
+  RoleArn:
+    Type: String
+    Description: ARN of the raw Lambda role.
+
+  SQSQueueArn:
+    Type: String
+    Description: ARN of the dispatch-to-raw SQS queue.
+
+  S3RawBucket:
+    Type: String
+    Description: Name of the Raw S3 bucket.
+
+  LambdaPythonVersion:
+    Type: String
+    Description: Python version to use for this lambda function
+    Default: "3.11"
+
+  LambdaBatchSize:
+    Type: Number
+    Default: 1
+    Description: >-
+      The maximum number of messages in an SQS event that Lambda will process in a batch
+
+  LambdaMaximumBatchingWindowInSeconds:
+    Type: Number
+    Default: 300
+    Description: >-
+      The maximum amount of time in seconds Lambda will batch messages before polling
+      the SQS queue and processing them
+
+Resources:
+  RawFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      PackageType: Zip
+      CodeUri: ./
+      Handler: app.lambda_handler
+      Runtime: !Sub "python${LambdaPythonVersion}"
+      Role: !Ref RoleArn
+      MemorySize: 1024
+      EphemeralStorage:
+        Size: 2048
+      Timeout: 900
+      Environment:
+        Variables:
+          RAW_S3_BUCKET: !Ref S3RawBucket
+      Events:
+        SQSEvent:
+          Type: SQS
+          Properties:
+            BatchSize: !Ref LambdaBatchSize
+            Queue: !Ref SQSQueueArn
+
+Outputs:
+  RawFunctionArn:
+    Description: Arn of the raw Lambda.
+    Value: !GetAtt DispatchFunction.Arn
+    Export:
+      Name: !Sub "${AWS::Region}-${AWS::StackName}-DispatchFunctionArn"

templates/lambda-raw-role.yaml

@@ -0,0 +1,81 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Transform: AWS::Serverless-2016-10-31
+
+Description: >
+  An IAM Role for the raw Lambda
+
+Parameters:
+  SQSQueueArn:
+    Type: String
+    Description: ARN of the SQS queue for lambda to poll messages from.
+
+  S3SourceBucketName:
+    Type: String
+    Description: Name of the S3 bucket where exports are deposited.
+
+  S3TargetBucketName:
+    Type: String
+    Description: Name of the S3 bucket where compressed JSON are written to.
+
+Resources:
+  RawRole:
+      Type: AWS::IAM::Role
+      Properties:
+        AssumeRolePolicyDocument:
+          Version: '2012-10-17'
+          Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+              - lambda.amazonaws.com
+            Action:
+            - sts:AssumeRole
+        ManagedPolicyArns:
+          - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+        Policies:
+        - PolicyName: PollSQSQueue
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - sqs:DeleteMessage
+              - sqs:GetQueueAttributes
+              - sqs:ReceiveMessage
+              Resource:
+              - !Ref SQSQueueArn
+        - PolicyName: ReadS3
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - s3:Get*
+              - s3:List*
+              Resource:
+              - !Sub arn:aws:s3:::${S3SourceBucketName}
+              - !Sub arn:aws:s3:::${S3SourceBucketName}/*
+        - PolicyName: WriteS3
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+            - Effect: Allow
+              Action:
+              - "s3:PutObject"
+              - "s3:PutObjectAcl"
+              - "s3:GetBucketLocation"
+              Resource:
+              - !Sub arn:aws:s3:::${S3TargetBucketName}
+              - !Sub arn:aws:s3:::${S3TargetBucketName}/*
+
+Outputs:
+  RoleName:
+    Value: !Ref RawRole
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleName'
+
+  RoleArn:
+    Value: !GetAtt RawRole.Arn
+    Export:
+      Name: !Sub '${AWS::Region}-${AWS::StackName}-RoleArn'