add prod cf configs, add perm for glue role for shareable artifacts b…

…ucket
Sage-Bionetworks · Sep 6, 2024 · ee6a812 · ee6a812
1 parent 84f5985
commit ee6a812
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 0 deletions.
diff --git a/config/develop/glue-job-role.yaml b/config/develop/glue-job-role.yaml
@@ -6,5 +6,6 @@ parameters:
   S3IntermediateBucketName: {{ stack_group_config.intermediate_bucket_name }}
   S3ParquetBucketName: {{ stack_group_config.processed_data_bucket_name }}
   S3ArtifactBucketName: {{ stack_group_config.template_bucket_name }}
+  S3ShareableArtifactBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
diff --git a/config/prod/namespaced/glue-workflow.yaml b/config/prod/namespaced/glue-workflow.yaml
@@ -6,6 +6,7 @@ dependencies:
   - prod/namespaced/glue-job-S3ToJsonS3.yaml
   - prod/namespaced/glue-job-JSONToParquet.yaml
   - prod/namespaced/glue-job-compare-parquet.yaml
+  - prod/namespaced/glue-job-run-great-expectations-on-parquet.yaml
   - prod/glue-job-role.yaml
   - prod/s3-cloudformation-bucket.yaml
 parameters:
@@ -19,6 +20,8 @@ parameters:
   CompareParquetMainNamespace: "main"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
   CloudformationBucketName: {{ stack_group_config.template_bucket_name }}
+  ShareableArtifactsBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
+  ExpectationSuiteKeyPrefix: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:

diff --git a/templates/glue-job-role.yaml b/templates/glue-job-role.yaml
@@ -20,6 +20,9 @@ Parameters:
     Type: String
     Description: Name of the S3 bucket where cloudformation templates and tests are stored.
 
+  S3ShareableArtifactBucketName:
+    Type: String
+    Description: Name of the S3 bucket where the shareable artifacts (like great expectations reports) are stored.
 
 Resources:
 
@@ -65,6 +68,8 @@ Resources:
             - !Sub arn:aws:s3:::${S3IntermediateBucketName}/*
             - !Sub arn:aws:s3:::${S3ParquetBucketName}
             - !Sub arn:aws:s3:::${S3ParquetBucketName}/*
+            - !Sub arn:aws:s3:::${S3ShareableArtifactBucketName}
+            - !Sub arn:aws:s3:::${S3ShareableArtifactBucketName}/*
       - PolicyName: ReadS3
         PolicyDocument:
           Version: '2012-10-17'