diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index b2e87eb..1f7363c 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -285,7 +285,7 @@ jobs: run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV - name: "Deploy sceptre stacks to dev" - run: pipenv run sceptre --var "namespace=${{ env.NAMESPACE }}" launch develop --yes + run: pipenv run sceptre --debug --var "namespace=${{ env.NAMESPACE }}" launch develop --yes - name: Delete preexisting S3 event notification for this namespace uses: gagoar/invoke-aws-lambda@v3 diff --git a/config/develop/namespaced/glue-workflow.yaml b/config/develop/namespaced/glue-workflow.yaml index 151ee1a..1c29783 100644 --- a/config/develop/namespaced/glue-workflow.yaml +++ b/config/develop/namespaced/glue-workflow.yaml @@ -19,8 +19,8 @@ parameters: CompareParquetMainNamespace: "main" S3SourceBucketName: {{ stack_group_config.input_bucket_name }} CloudformationBucketName: {{ stack_group_config.template_bucket_name }} - ShareableArtifactsBucket: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }} - ExpectationSuiteKeyPrefix: !stack_output_external '{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json' + ShareableArtifactsBucketName: {{ stack_group_config.shareable_artifacts_vpn_bucket_name }} + ExpectationSuiteKeyPrefix: "{{ stack_group_config.namespace }}/src/glue/resources/data_values_expectations.json" stack_tags: {{ stack_group_config.default_stack_tags }} sceptre_user_data: diff --git a/templates/glue-workflow.j2 b/templates/glue-workflow.j2 index 6bd5e8c..f138793 100644 --- a/templates/glue-workflow.j2 +++ b/templates/glue-workflow.j2 @@ -84,14 +84,11 @@ Parameters: ShareableArtifactsBucketName: Type: String - Description: >- - The name of the bucket where shareable artifacts are stored. + Description: The name of the bucket where shareable artifacts are stored. - ExpectationSuiteKeyPrefix + ExpectationSuiteKeyPrefix: Type: String - Description: >- - The s3 key prefix of the expectation suite. - + Description: The s3 key prefix of the expectation suite. Conditions: IsMainNamespace: !Equals [!Ref Namespace, "main"] @@ -281,11 +278,10 @@ Resources: StartOnCreation: true WorkflowName: !Ref JsonToParquetWorkflow - JsontoParquetCompleteTrigger: + CompareParquetTrigger: Type: AWS::Glue::Trigger - Condition: IsDevelopmentNamespace Properties: - Name: !Sub "${Namespace}-JsontoParquetCompleteTrigger" + Name: !Sub "${Namespace}-CompareParquetTrigger" Actions: {% for dataset in datasets %} - JobName: !Sub ${Namespace}-{{ dataset["stackname_prefix"] }}-CompareParquetJob @@ -297,7 +293,28 @@ Resources: "--cfn-bucket": !Ref CloudformationBucketName "--parquet-bucket": !Ref ParquetBucketName "--additional-python-modules": "datacompy~=0.8 flask~=2.0 flask-cors~=3.0" - - JobName: !Sub ${Namespace}-{{ dataset["stackname_prefix"] }}-GreatExpectationsJob + {% endfor %} + Description: This trigger runs the compare parquet jobs after completion of all JSON to Parquet jobs + Type: CONDITIONAL + Predicate: + Conditions: + {% for dataset in datasets if "Garmin" in dataset["data_type"] %} + - JobName: !Sub "${Namespace}-{{ dataset["stackname_prefix"] }}-Job" + State: SUCCEEDED + LogicalOperator: EQUALS + {% endfor %} + Logical: AND + StartOnCreation: true + WorkflowName: !Ref JsonToParquetWorkflow + + GreatExpectationsParquetTrigger: + Type: AWS::Glue::Trigger + Condition: IsDevelopmentNamespace + Properties: + Name: !Sub "${Namespace}-GreatExpectationsParquetTrigger" + Actions: + {% for dataset in datasets %} + - JobName: !Sub ${Namespace}-{{ dataset["stackname_prefix"] }}-GreatExpectationsParquetJob Arguments: "--data-type": {{ "{}".format(dataset["table_name"]) }} "--namespace": !Ref Namespace @@ -307,7 +324,7 @@ Resources: "--expectation-suite-key-prefix": !Sub "${Namespace}/src/glue/resources/data_values_expectations.json" "--additional-python-modules": "great_expectations~=0.17" {% endfor %} - Description: This trigger runs after completion of all JSON to Parquet jobs + Description: This trigger runs the great expectation parquet jobs after completion of all JSON to Parquet jobs Type: CONDITIONAL Predicate: Conditions: