diff --git a/config/develop/namespaced/s3-event-config-lambda.yaml b/config/develop/namespaced/s3-event-config-lambda.yaml index b450a5a9..83fc6f07 100644 --- a/config/develop/namespaced/s3-event-config-lambda.yaml +++ b/config/develop/namespaced/s3-event-config-lambda.yaml @@ -6,12 +6,12 @@ template: dependencies: - develop/namespaced/s3-event-config-lambda-role.yaml - develop/namespaced/s3-to-glue-lambda.yaml - - develop/namespaced/sqs-queue.yaml + - develop/namespaced/sqs-input-to-intermediate.yaml stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig' stack_tags: {{ stack_group_config.default_stack_tags }} parameters: Namespace: {{ stack_group_config.namespace }} - S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn" + S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn" S3ToGlueDestinationType: "Queue" S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn" S3SourceBucketName: {{ stack_group_config.input_bucket_name }} diff --git a/config/develop/namespaced/s3-to-glue-lambda-role.yaml b/config/develop/namespaced/s3-to-glue-lambda-role.yaml index 9395b730..c055bc1f 100644 --- a/config/develop/namespaced/s3-to-glue-lambda-role.yaml +++ b/config/develop/namespaced/s3-to-glue-lambda-role.yaml @@ -2,8 +2,8 @@ template: path: s3-to-glue-lambda-role.yaml stack_name: '{{ stack_group_config.namespace }}-s3-to-glue-lambda-role' dependencies: - - develop/namespaced/sqs-queue.yaml + - develop/namespaced/sqs-input-to-intermediate.yaml parameters: - SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn" + SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn" stack_tags: {{ stack_group_config.default_stack_tags }} diff --git a/config/develop/namespaced/s3-to-glue-lambda.yaml b/config/develop/namespaced/s3-to-glue-lambda.yaml index 0444b9c6..e41c1cfe 100644 --- a/config/develop/namespaced/s3-to-glue-lambda.yaml +++ b/config/develop/namespaced/s3-to-glue-lambda.yaml @@ -6,10 +6,11 @@ template: dependencies: - develop/namespaced/s3-to-glue-lambda-role.yaml - develop/namespaced/glue-workflow.yaml -stack_name: '{{ stack_group_config.namespace }}-lambda-S3ToGlue' + - develop/namespaced/sqs-input-to-intermediate.yaml +stack_name: '{{ stack_group_config.namespace }}-lambda-input-to-intermediate' stack_tags: {{ stack_group_config.default_stack_tags }} parameters: - SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn" + SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn" S3ToGlueRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-to-glue-lambda-role::RoleArn" S3ToJsonWorkflowName: !stack_output_external "{{ stack_group_config.namespace }}-glue-workflow::S3ToJsonWorkflowName" LambdaBatchSize: '10' diff --git a/config/develop/namespaced/sqs-queue.yaml b/config/develop/namespaced/sqs-input-to-intermediate.yaml similarity index 56% rename from config/develop/namespaced/sqs-queue.yaml rename to config/develop/namespaced/sqs-input-to-intermediate.yaml index 88f5e0de..3e195f29 100644 --- a/config/develop/namespaced/sqs-queue.yaml +++ b/config/develop/namespaced/sqs-input-to-intermediate.yaml @@ -1,12 +1,12 @@ template: path: sqs-queue.yaml parameters: - MessageRetentionPeriod: '86400' - ReceiveMessageWaitTimeSeconds: '20' - VisibilityTimeout: '120' + MessageRetentionPeriod: "86400" + ReceiveMessageWaitTimeSeconds: "20" + VisibilityTimeout: "120" S3SourceBucketArn: !stack_output_external recover-dev-input-bucket::BucketArn dependencies: - develop/s3-input-bucket.yaml -stack_name: '{{ stack_group_config.namespace }}-sqs-S3ToLambda' +stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-intermediate' stack_tags: {{ stack_group_config.default_stack_tags }} diff --git a/config/develop/namespaced/sqs-input-to-raw.yaml b/config/develop/namespaced/sqs-input-to-raw.yaml new file mode 100644 index 00000000..0ef7adbf --- /dev/null +++ b/config/develop/namespaced/sqs-input-to-raw.yaml @@ -0,0 +1,12 @@ +template: + path: sqs-queue.yaml +parameters: + MessageRetentionPeriod: "1209600" + ReceiveMessageWaitTimeSeconds: "20" + VisibilityTimeout: "120" + S3SourceBucketArn: !stack_output_external recover-dev-raw-bucket::BucketArn +dependencies: + - develop/s3-raw-bucket.yaml +stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-raw' +stack_tags: + {{ stack_group_config.default_stack_tags }} diff --git a/config/prod/namespaced/s3-event-config-lambda.yaml b/config/prod/namespaced/s3-event-config-lambda.yaml index e7ebcaba..08741f14 100644 --- a/config/prod/namespaced/s3-event-config-lambda.yaml +++ b/config/prod/namespaced/s3-event-config-lambda.yaml @@ -6,12 +6,12 @@ template: dependencies: - prod/namespaced/s3-event-config-lambda-role.yaml - prod/namespaced/s3-to-glue-lambda.yaml - - prod/namespaced/sqs-queue.yaml + - prod/namespaced/sqs-input-to-intermediate.yaml stack_name: '{{ stack_group_config.namespace }}-lambda-S3EventConfig' stack_tags: {{ stack_group_config.default_stack_tags }} parameters: Namespace: {{ stack_group_config.namespace }} - S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn" + S3ToGlueDestinationArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn" S3ToGlueDestinationType: "Queue" S3EventConfigRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-event-config-lambda-role::RoleArn" S3SourceBucketName: {{ stack_group_config.input_bucket_name }} diff --git a/config/prod/namespaced/s3-to-glue-lambda-role.yaml b/config/prod/namespaced/s3-to-glue-lambda-role.yaml index 4ed883dd..5f319445 100644 --- a/config/prod/namespaced/s3-to-glue-lambda-role.yaml +++ b/config/prod/namespaced/s3-to-glue-lambda-role.yaml @@ -2,8 +2,8 @@ template: path: s3-to-glue-lambda-role.yaml stack_name: '{{ stack_group_config.namespace }}-s3-to-glue-lambda-role' dependencies: - - prod/namespaced/sqs-queue.yaml + - prod/namespaced/sqs-input-to-intermediate.yaml parameters: - SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn" + SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn" stack_tags: {{ stack_group_config.default_stack_tags }} diff --git a/config/prod/namespaced/s3-to-glue-lambda.yaml b/config/prod/namespaced/s3-to-glue-lambda.yaml index 9cfad844..ee5898a3 100644 --- a/config/prod/namespaced/s3-to-glue-lambda.yaml +++ b/config/prod/namespaced/s3-to-glue-lambda.yaml @@ -6,10 +6,11 @@ template: dependencies: - prod/namespaced/s3-to-glue-lambda-role.yaml - prod/namespaced/glue-workflow.yaml + - prod/namespaced/sqs-input-to-intermediate.yaml stack_name: '{{ stack_group_config.namespace }}-lambda-S3ToGlue' stack_tags: {{ stack_group_config.default_stack_tags }} parameters: - SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-S3ToLambda::PrimaryQueueArn" + SQSQueueArn: !stack_output_external "{{ stack_group_config.namespace }}-sqs-input-to-intermediate::PrimaryQueueArn" S3ToGlueRoleArn: !stack_output_external "{{ stack_group_config.namespace }}-s3-to-glue-lambda-role::RoleArn" S3ToJsonWorkflowName: !stack_output_external "{{ stack_group_config.namespace }}-glue-workflow::S3ToJsonWorkflowName" LambdaBatchSize: '10' diff --git a/config/prod/namespaced/sqs-queue.yaml b/config/prod/namespaced/sqs-input-to-intermediate.yaml similarity index 81% rename from config/prod/namespaced/sqs-queue.yaml rename to config/prod/namespaced/sqs-input-to-intermediate.yaml index 77010459..ff87c2a8 100644 --- a/config/prod/namespaced/sqs-queue.yaml +++ b/config/prod/namespaced/sqs-input-to-intermediate.yaml @@ -7,6 +7,6 @@ parameters: S3SourceBucketArn: !stack_output_external recover-input-bucket::BucketArn dependencies: - prod/s3-input-bucket.yaml -stack_name: '{{ stack_group_config.namespace }}-sqs-S3ToLambda' +stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-intermediate' stack_tags: {{ stack_group_config.default_stack_tags }} diff --git a/config/prod/namespaced/sqs-input-to-raw.yaml b/config/prod/namespaced/sqs-input-to-raw.yaml new file mode 100644 index 00000000..7fb1e94a --- /dev/null +++ b/config/prod/namespaced/sqs-input-to-raw.yaml @@ -0,0 +1,12 @@ +template: + path: sqs-queue.yaml +parameters: + MessageRetentionPeriod: "1209600" + ReceiveMessageWaitTimeSeconds: "20" + VisibilityTimeout: "120" + S3SourceBucketArn: !stack_output_external recover-raw-bucket::BucketArn +dependencies: + - prod/s3-raw-bucket.yaml +stack_name: '{{ stack_group_config.namespace }}-sqs-input-to-raw' +stack_tags: + {{ stack_group_config.default_stack_tags }}