correct formatting, refactor triggers

Sage-Bionetworks · Sep 5, 2024 · 6ae0e24 · 6ae0e24
1 parent 9d1a0d5
commit 6ae0e24
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 14 deletions.
diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml
@@ -285,7 +285,7 @@ jobs:
         run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV
 
       - name: "Deploy sceptre stacks to dev"
-        run: pipenv run sceptre --var "namespace=${{ env.NAMESPACE }}" launch develop --yes
+        run: pipenv run sceptre --debug --var "namespace=${{ env.NAMESPACE }}" launch develop --yes
 
       - name: Delete preexisting S3 event notification for this namespace
         uses: gagoar/invoke-aws-lambda@v3

diff --git a/config/develop/namespaced/glue-workflow.yaml b/config/develop/namespaced/glue-workflow.yaml
@@ -19,8 +19,8 @@ parameters:
   CompareParquetMainNamespace: "main"
   S3SourceBucketName: {{ stack_group_config.input_bucket_name }}
   CloudformationBucketName: {{ stack_group_config.template_bucket_name }}
-  ShareableArtifactsBucket: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
-  ExpectationSuiteKeyPrefix:  !stack_output_external '{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json'
+  ShareableArtifactsBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }}
+  ExpectationSuiteKeyPrefix: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json"
 stack_tags:
   {{ stack_group_config.default_stack_tags }}
 sceptre_user_data:

diff --git a/templates/glue-workflow.j2 b/templates/glue-workflow.j2
@@ -84,14 +84,11 @@ Parameters:
 
   ShareableArtifactsBucketName:
     Type: String
-    Description: >-
-        The name of the bucket where shareable artifacts are stored.
+    Description: The name of the bucket where shareable artifacts are stored.
 
-  ExpectationSuiteKeyPrefix
+  ExpectationSuiteKeyPrefix:
     Type: String
-    Description: >-
-        The s3 key prefix of the expectation suite.
-
+    Description: The s3 key prefix of the expectation suite.
 
 Conditions:
   IsMainNamespace: !Equals [!Ref Namespace, "main"]
@@ -281,11 +278,10 @@ Resources:
       StartOnCreation: true
       WorkflowName: !Ref JsonToParquetWorkflow
 
-  JsontoParquetCompleteTrigger:
+  CompareParquetTrigger:
     Type: AWS::Glue::Trigger
-    Condition: IsDevelopmentNamespace
     Properties:
-      Name: !Sub "${Namespace}-JsontoParquetCompleteTrigger"
+      Name: !Sub "${Namespace}-CompareParquetTrigger"
       Actions:
         {% for dataset in datasets %}
         - JobName: !Sub ${Namespace}-{{ dataset["stackname_prefix"] }}-CompareParquetJob
@@ -297,7 +293,28 @@ Resources:
             "--cfn-bucket": !Ref CloudformationBucketName
             "--parquet-bucket": !Ref ParquetBucketName
             "--additional-python-modules": "datacompy~=0.8 flask~=2.0 flask-cors~=3.0"
-        - JobName: !Sub ${Namespace}-{{ dataset["stackname_prefix"] }}-GreatExpectationsJob
+        {% endfor %}
+      Description: This trigger runs the compare parquet jobs after completion of all JSON to Parquet jobs
+      Type: CONDITIONAL
+      Predicate:
+        Conditions:
+          {% for dataset in datasets if "Garmin" in dataset["data_type"] %}
+          - JobName: !Sub "${Namespace}-{{ dataset["stackname_prefix"] }}-Job"
+            State: SUCCEEDED
+            LogicalOperator: EQUALS
+          {% endfor %}
+        Logical: AND
+      StartOnCreation: true
+      WorkflowName: !Ref JsonToParquetWorkflow
+
+  GreatExpectationsParquetTrigger:
+    Type: AWS::Glue::Trigger
+    Condition: IsDevelopmentNamespace
+    Properties:
+      Name: !Sub "${Namespace}-GreatExpectationsParquetTrigger"
+      Actions:
+        {% for dataset in datasets %}
+        - JobName: !Sub ${Namespace}-{{ dataset["stackname_prefix"] }}-GreatExpectationsParquetJob
           Arguments:
             "--data-type": {{ "{}".format(dataset["table_name"]) }}
             "--namespace": !Ref Namespace
@@ -307,7 +324,7 @@ Resources:
             "--expectation-suite-key-prefix": !Sub "${Namespace}/src/glue/resources/data_values_expectations.json"
             "--additional-python-modules": "great_expectations~=0.17"
         {% endfor %}
-      Description: This trigger runs after completion of all JSON to Parquet jobs
+      Description:  This trigger runs the great expectation parquet jobs after completion of all JSON to Parquet jobs
       Type: CONDITIONAL
       Predicate:
         Conditions: