From 919e2c71942c84f9fdd6563159fd1f508b542aab Mon Sep 17 00:00:00 2001 From: Hannah Pho Date: Mon, 3 Feb 2025 15:26:24 -0500 Subject: [PATCH 1/4] Make `REDIS_HOST` and `REDIS_PORT` available to both services and data management --- .../modules/locals.tf | 28 +++++++++++++------ .../modules/main.tf | 13 +++------ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/deploy/terraform-custom-datacommons/modules/locals.tf b/deploy/terraform-custom-datacommons/modules/locals.tf index ea528e2a00..cc24ca2030 100644 --- a/deploy/terraform-custom-datacommons/modules/locals.tf +++ b/deploy/terraform-custom-datacommons/modules/locals.tf @@ -18,15 +18,15 @@ locals { # Data Commons Data Bucket gcs_data_bucket_name = var.gcs_data_bucket_name != "" ? var.gcs_data_bucket_name : "${var.namespace}-datacommons-data-${var.project_id}" # VPC Connector CIDR block - vpc_connector_cidr = cidrsubnet(var.vpc_base_cidr_block, 4, 0) # Generates the first /28 subnet from the /24 block + vpc_connector_cidr = cidrsubnet(var.vpc_base_cidr_block, 4, 0) # Generates the first /28 subnet from the /24 block # Use var.maps_api_key if set, otherwise use generated Maps API key maps_api_key = var.maps_api_key != null ? var.maps_api_key : google_apikeys_key.maps_api_key.key_string # Use var.billing_project_id if set, otherwise use project_id for billing billing_project_id = var.billing_project_id != null ? var.billing_project_id : var.project_id - - # Data Commons API hostname + + # Data Commons API hostname dc_api_hostname = "api.datacommons.org" # Data Commons API protocol @@ -34,7 +34,11 @@ locals { # Data Commons API root URL dc_api_root = "${local.dc_api_protocol}://${local.dc_api_hostname}" - + + # Optionally-configured Redis instance + redis_instance = var.enable_redis ? google_redis_instance.redis_instance[0] : null + + # Shared environment variables used by the Data Commons web service and the Data # Commons data loading job cloud_run_shared_env_variables = [ @@ -61,6 +65,14 @@ locals { { name = "FORCE_RESTART" value = "${timestamp()}" + }, + { + name = "REDIS_HOST" + value = try(local.redis_instance.host, "") + }, + { + name = "REDIS_PORT" + value = try(local.redis_instance.port, "") } ] @@ -68,16 +80,16 @@ locals { # web service and the Data Commons data loading job cloud_run_shared_env_variable_secrets = [ { - name = "DC_API_KEY" + name = "DC_API_KEY" value_source = { secret_key_ref = { - secret = google_secret_manager_secret.dc_api_key.secret_id - version = "latest" + secret = google_secret_manager_secret.dc_api_key.secret_id + version = "latest" } } }, { - name = "DB_PASS" + name = "DB_PASS" value_source = { secret_key_ref = { secret = google_secret_manager_secret.mysql_password.secret_id diff --git a/deploy/terraform-custom-datacommons/modules/main.tf b/deploy/terraform-custom-datacommons/modules/main.tf index 57d946c1a1..7b781ee634 100644 --- a/deploy/terraform-custom-datacommons/modules/main.tf +++ b/deploy/terraform-custom-datacommons/modules/main.tf @@ -220,7 +220,7 @@ resource "google_cloud_run_v2_service" "dc_web_service" { secret_key_ref { secret = env.value.value_source.secret_key_ref.secret version = env.value.value_source.secret_key_ref.version - } + } } } } @@ -255,11 +255,6 @@ resource "google_cloud_run_v2_service" "dc_web_service" { value = "true" } - env { - name = "REDIS_HOST" - value = var.enable_redis ? google_redis_instance.redis_instance[0].host : "" - } - env { name = "MAPS_API_KEY" value_source { @@ -338,7 +333,7 @@ resource "google_cloud_run_v2_job" "dc_data_job" { location = var.region deletion_protection = false - template { + template { template { containers { image = var.dc_data_job_image @@ -353,7 +348,7 @@ resource "google_cloud_run_v2_job" "dc_data_job" { # Shared environment variables dynamic "env" { for_each = local.cloud_run_shared_env_variables - content { + content { name = env.value.name value = env.value.value } @@ -368,7 +363,7 @@ resource "google_cloud_run_v2_job" "dc_data_job" { secret_key_ref { secret = env.value.value_source.secret_key_ref.secret version = env.value.value_source.secret_key_ref.version - } + } } } } From 82def52a02174cc8a94d07231067ef7d16541f14 Mon Sep 17 00:00:00 2001 From: Hannah Pho Date: Mon, 3 Feb 2025 16:40:26 -0500 Subject: [PATCH 2/4] Use local var for redis outputs --- deploy/terraform-custom-datacommons/modules/outputs.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/terraform-custom-datacommons/modules/outputs.tf b/deploy/terraform-custom-datacommons/modules/outputs.tf index ccc3ebd54d..6e07788181 100644 --- a/deploy/terraform-custom-datacommons/modules/outputs.tf +++ b/deploy/terraform-custom-datacommons/modules/outputs.tf @@ -14,12 +14,12 @@ output "redis_instance_host" { description = "The hostname or IP address of the Redis instance" - value = var.enable_redis ? google_redis_instance.redis_instance[0].host : "" + value = try(local.redis_instance.host, "") } output "redis_instance_port" { description = "The port number the Redis instance is listening on" - value = var.enable_redis ? google_redis_instance.redis_instance[0].port : null + value = try(local.redis_instance.port, "") } output "mysql_instance_connection_name" { @@ -68,4 +68,4 @@ output "maps_api_key" { description = "Maps API key" value = local.maps_api_key sensitive = true -} \ No newline at end of file +} From 19027c43ab4ef405b92552740152940ddbc83875 Mon Sep 17 00:00:00 2001 From: Hannah Pho Date: Mon, 3 Feb 2025 16:40:41 -0500 Subject: [PATCH 3/4] latest -> stable for data image --- deploy/terraform-custom-datacommons/modules/variables.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/deploy/terraform-custom-datacommons/modules/variables.tf b/deploy/terraform-custom-datacommons/modules/variables.tf index e3ca593a2e..b7b623236a 100644 --- a/deploy/terraform-custom-datacommons/modules/variables.tf +++ b/deploy/terraform-custom-datacommons/modules/variables.tf @@ -33,7 +33,7 @@ variable "dc_api_key" { # Optional variables -# Optional: If blank, will generate a Maps API key. +# Optional: If blank, will generate a Maps API key. variable "maps_api_key" { description = "Google Maps API Key" type = string @@ -182,11 +182,10 @@ variable "make_dc_web_service_public" { } # Data Commons Cloud Run job variables -# TODO: Change to stable after the next release variable "dc_data_job_image" { description = "The container image for the data job" type = string - default = "gcr.io/datcom-ci/datacommons-data:latest" + default = "gcr.io/datcom-ci/datacommons-data:stable" } variable "dc_data_job_cpu" { @@ -262,4 +261,4 @@ variable "redis_replica_count" { description = "Redis reserved IP range" type = number default = 1 -} \ No newline at end of file +} From a1947f826fd25f0938e8b37d6683ae7246022e65 Mon Sep 17 00:00:00 2001 From: Hannah Pho Date: Tue, 4 Feb 2025 13:09:58 -0500 Subject: [PATCH 4/4] VPC for data management job --- .../modules/locals.tf | 2 - .../modules/main.tf | 82 ++++++++++--------- .../modules/variables.tf | 6 -- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/deploy/terraform-custom-datacommons/modules/locals.tf b/deploy/terraform-custom-datacommons/modules/locals.tf index cc24ca2030..f26cdc540a 100644 --- a/deploy/terraform-custom-datacommons/modules/locals.tf +++ b/deploy/terraform-custom-datacommons/modules/locals.tf @@ -17,8 +17,6 @@ locals { # Data Commons Data Bucket gcs_data_bucket_name = var.gcs_data_bucket_name != "" ? var.gcs_data_bucket_name : "${var.namespace}-datacommons-data-${var.project_id}" - # VPC Connector CIDR block - vpc_connector_cidr = cidrsubnet(var.vpc_base_cidr_block, 4, 0) # Generates the first /28 subnet from the /24 block # Use var.maps_api_key if set, otherwise use generated Maps API key maps_api_key = var.maps_api_key != null ? var.maps_api_key : google_apikeys_key.maps_api_key.key_string diff --git a/deploy/terraform-custom-datacommons/modules/main.tf b/deploy/terraform-custom-datacommons/modules/main.tf index 7b781ee634..547156ed2e 100644 --- a/deploy/terraform-custom-datacommons/modules/main.tf +++ b/deploy/terraform-custom-datacommons/modules/main.tf @@ -15,10 +15,10 @@ # Custom Data Commons terraform resources provider "google" { - project = var.project_id - region = var.region + project = var.project_id + region = var.region user_project_override = var.user_project_override - billing_project = local.billing_project_id + billing_project = local.billing_project_id } # Reference the default VPC network @@ -34,15 +34,15 @@ data "google_compute_subnetwork" "default_subnet" { # Create redis instance resource "google_redis_instance" "redis_instance" { - count = var.enable_redis ? 1 : 0 - name = "${var.namespace}-${var.redis_instance_name}" - tier = var.redis_tier - memory_size_gb = var.redis_memory_size_gb - region = var.region - location_id = var.redis_location_id + count = var.enable_redis ? 1 : 0 + name = "${var.namespace}-${var.redis_instance_name}" + tier = var.redis_tier + memory_size_gb = var.redis_memory_size_gb + region = var.region + location_id = var.redis_location_id alternative_location_id = var.redis_alternative_location_id - authorized_network = data.google_compute_network.default.self_link - replica_count = var.redis_replica_count + authorized_network = data.google_compute_network.default.self_link + replica_count = var.redis_replica_count } # Create MySQL instance @@ -93,9 +93,9 @@ resource "google_secret_manager_secret_version" "mysql_password_version" { } resource "google_sql_database" "mysql_db" { - name = var.mysql_database_name - instance = google_sql_database_instance.mysql_instance.name - charset = "utf8mb4" + name = var.mysql_database_name + instance = google_sql_database_instance.mysql_instance.name + charset = "utf8mb4" collation = "utf8mb4_unicode_ci" } @@ -108,23 +108,23 @@ resource "google_sql_user" "mysql_user" { # Data commons storage bucket resource "google_storage_bucket" "gcs_data_bucket" { - name = local.gcs_data_bucket_name - location = var.gcs_data_bucket_location + name = local.gcs_data_bucket_name + location = var.gcs_data_bucket_location uniform_bucket_level_access = true } # Input 'folder' for the data loading job. resource "google_storage_bucket_object" "gcs_data_bucket_input_folder" { - name = "${var.gcs_data_bucket_input_folder}/" - content = "Input folder" - bucket = "${google_storage_bucket.gcs_data_bucket.name}" + name = "${var.gcs_data_bucket_input_folder}/" + content = "Input folder" + bucket = google_storage_bucket.gcs_data_bucket.name } # Output 'folder' for the data loading job. resource "google_storage_bucket_object" "gcs_data_bucket_output_folder" { - name = "${var.gcs_data_bucket_output_folder}/" - content = "Output folder" - bucket = "${google_storage_bucket.gcs_data_bucket.name}" + name = "${var.gcs_data_bucket_output_folder}/" + content = "Output folder" + bucket = google_storage_bucket.gcs_data_bucket.name } # Generate a random suffix to append to api keys. @@ -181,8 +181,8 @@ resource "google_secret_manager_secret_version" "dc_api_key_version" { # Data Commons Cloud Run Service resource "google_cloud_run_v2_service" "dc_web_service" { - name = "${var.namespace}-datacommons-web-service" - location = var.region + name = "${var.namespace}-datacommons-web-service" + location = var.region deletion_protection = false template { @@ -215,10 +215,10 @@ resource "google_cloud_run_v2_service" "dc_web_service" { dynamic "env" { for_each = local.cloud_run_shared_env_variable_secrets content { - name = env.value.name + name = env.value.name value_source { secret_key_ref { - secret = env.value.value_source.secret_key_ref.secret + secret = env.value.value_source.secret_key_ref.secret version = env.value.value_source.secret_key_ref.version } } @@ -226,7 +226,7 @@ resource "google_cloud_run_v2_service" "dc_web_service" { } env { - name = "GOOGLE_ANALYTICS_TAG_ID" + name = "GOOGLE_ANALYTICS_TAG_ID" value = var.google_analytics_tag_id != null ? var.google_analytics_tag_id : "" } @@ -256,11 +256,11 @@ resource "google_cloud_run_v2_service" "dc_web_service" { } env { - name = "MAPS_API_KEY" + name = "MAPS_API_KEY" value_source { secret_key_ref { - secret = google_secret_manager_secret.maps_api_key.secret_id - version = "latest" + secret = google_secret_manager_secret.maps_api_key.secret_id + version = "latest" } } } @@ -292,8 +292,8 @@ resource "google_cloud_run_v2_service" "dc_web_service" { vpc_access { network_interfaces { - network = data.google_compute_network.default.id - subnetwork = data.google_compute_subnetwork.default_subnet.name + network = data.google_compute_network.default.id + subnetwork = data.google_compute_subnetwork.default_subnet.name } egress = "PRIVATE_RANGES_ONLY" } @@ -329,8 +329,8 @@ resource "google_cloud_run_service_iam_member" "dc_web_service_invoker" { # Data Commons data loading job resource "google_cloud_run_v2_job" "dc_data_job" { - name = "${var.namespace}-datacommons-data-job" - location = var.region + name = "${var.namespace}-datacommons-data-job" + location = var.region deletion_protection = false template { @@ -358,10 +358,10 @@ resource "google_cloud_run_v2_job" "dc_data_job" { dynamic "env" { for_each = local.cloud_run_shared_env_variable_secrets content { - name = env.value.name + name = env.value.name value_source { secret_key_ref { - secret = env.value.value_source.secret_key_ref.secret + secret = env.value.value_source.secret_key_ref.secret version = env.value.value_source.secret_key_ref.version } } @@ -373,8 +373,16 @@ resource "google_cloud_run_v2_job" "dc_data_job" { value = "gs://${local.gcs_data_bucket_name}/${var.gcs_data_bucket_input_folder}" } } + vpc_access { + network_interfaces { + network = data.google_compute_network.default.id + subnetwork = data.google_compute_subnetwork.default_subnet.name + } + egress = "PRIVATE_RANGES_ONLY" + } + execution_environment = "EXECUTION_ENVIRONMENT_GEN2" - service_account = google_service_account.datacommons_service_account.email + service_account = google_service_account.datacommons_service_account.email } } diff --git a/deploy/terraform-custom-datacommons/modules/variables.tf b/deploy/terraform-custom-datacommons/modules/variables.tf index b7b623236a..d915903899 100644 --- a/deploy/terraform-custom-datacommons/modules/variables.tf +++ b/deploy/terraform-custom-datacommons/modules/variables.tf @@ -214,12 +214,6 @@ variable "vpc_network_subnet_name" { default = "default" } -variable "vpc_base_cidr_block" { - description = "Base CIDR block to be subdivided for VPC connectors" - type = string - default = "10.8.0.0/24" -} - # Data Commons Cloud Redis Memorystore instance variables variable "enable_redis" {