combine tables into one job (#806)

LBHackney-IT · Jul 21, 2022 · af073d2 · af073d2
1 parent 239e695
commit af073d2
Showing 1 changed file with 9 additions and 17 deletions.
diff --git a/terraform/core/39-housing-interim-finance-db-ingestion.tf b/terraform/core/39-housing-interim-finance-db-ingestion.tf
@@ -14,25 +14,18 @@ module "housing_interim_finance_database_ingestion" {
 }
 
 locals {
-  table_filter_expressions_housing_interim_finance = local.is_live_environment ? {
-    ma-tenancy-agreement       = "^sow2b_dbo_matenancyagreement$",
-    uh-account-recovery-action = "^sow2b_dbo_uharaction$",
-    ma-property                = "^sow2b_dbo_maproperty$",
-    uh-mini-transaction        = "^sow2b_dbo_uhminitransaction$",
-    ss-mini-transaction        = "^sow2b_dbo_ssminitransaction$"
-  } : {}
+  table_filter_expressions_housing_interim_finance = local.is_live_environment ? "(^sow2b_dbo_matenancyagreement$|^sow2b_dbo_uharaction$|^sow2b_dbo_maproperty$|^sow2b_dbo_uhminitransaction$|^sow2b_dbo_ssminitransaction$)" : ""
 }
 
 resource "aws_glue_trigger" "housing_interim_finance_filter_ingestion_tables" {
-  tags     = module.tags.values
-  for_each = local.table_filter_expressions_housing_interim_finance
-  name     = "${local.short_identifier_prefix}housing-interim-finance-table-${each.key}"
-  type     = "CONDITIONAL"
+  tags = module.tags.values
+  name = "${local.short_identifier_prefix}housing-interim-finance-tables"
+  type = "CONDITIONAL"
 
   actions {
-    job_name = module.ingest_housing_interim_finance_database_to_housing_raw_zone[each.key].job_name
+    job_name = module.ingest_housing_interim_finance_database_to_housing_raw_zone.job_name
     arguments = {
-      "--table_filter_expression" = each.value
+      "--table_filter_expression" = local.table_filter_expressions_housing_interim_finance
     }
   }
 
@@ -45,14 +38,13 @@ resource "aws_glue_trigger" "housing_interim_finance_filter_ingestion_tables" {
 }
 
 module "ingest_housing_interim_finance_database_to_housing_raw_zone" {
-  for_each = local.table_filter_expressions_housing_interim_finance
-  tags     = module.tags.values
+  tags = module.tags.values
 
   source = "../modules/aws-glue-job"
 
   department = module.department_housing
 
-  job_name                   = "${local.short_identifier_prefix}Housing Interim Finance Database Ingestion-${each.key}"
+  job_name                   = "${local.short_identifier_prefix}Housing Interim Finance Database Ingestion"
   script_s3_object_key       = aws_s3_bucket_object.ingest_database_tables_via_jdbc_connection.key
   environment                = var.environment
   pydeequ_zip_key            = aws_s3_bucket_object.pydeequ.key
@@ -65,7 +57,7 @@ module "ingest_housing_interim_finance_database_to_housing_raw_zone" {
     "--source_data_database"        = module.housing_interim_finance_database_ingestion[0].ingestion_database_name
     "--s3_ingestion_bucket_target"  = "s3://${module.raw_zone.bucket_id}/housing/"
     "--s3_ingestion_details_target" = "s3://${module.raw_zone.bucket_id}/housing/ingestion-details/"
-    "--table_filter_expression"     = each.value
+    "--table_filter_expression"     = local.table_filter_expressions_housing_interim_finance
   }
   crawler_details = {
     database_name      = module.department_housing.raw_zone_catalog_database_name