Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Custom DC] [Terraform] Allow data management job to connect to Redis #4925

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+74 −68
Open

[Custom DC] [Terraform] Allow data management job to connect to Redis #4925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
919e2c7
Make `REDIS_HOST` and `REDIS_PORT` available to both services and dat…
hqpho Feb 3, 2025
82def52
Use local var for redis outputs
hqpho Feb 3, 2025
19027c4
latest -> stable for data image
hqpho Feb 3, 2025
a1947f8
VPC for data management job
hqpho Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions deploy/terraform-custom-datacommons/modules/locals.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,26 @@
locals {
# Data Commons Data Bucket
gcs_data_bucket_name = var.gcs_data_bucket_name != "" ? var.gcs_data_bucket_name : "${var.namespace}-datacommons-data-${var.project_id}"
# VPC Connector CIDR block
vpc_connector_cidr = cidrsubnet(var.vpc_base_cidr_block, 4, 0) # Generates the first /28 subnet from the /24 block

# Use var.maps_api_key if set, otherwise use generated Maps API key
maps_api_key = var.maps_api_key != null ? var.maps_api_key : google_apikeys_key.maps_api_key.key_string

# Use var.billing_project_id if set, otherwise use project_id for billing
billing_project_id = var.billing_project_id != null ? var.billing_project_id : var.project_id
# Data Commons API hostname

# Data Commons API hostname
dc_api_hostname = "api.datacommons.org"

# Data Commons API protocol
dc_api_protocol = "https"

# Data Commons API root URL
dc_api_root = "${local.dc_api_protocol}://${local.dc_api_hostname}"


# Optionally-configured Redis instance
redis_instance = var.enable_redis ? google_redis_instance.redis_instance[0] : null


# Shared environment variables used by the Data Commons web service and the Data
# Commons data loading job
cloud_run_shared_env_variables = [
Expand All @@ -61,23 +63,31 @@ locals {
{
name = "FORCE_RESTART"
value = "${timestamp()}"
},
{
name = "REDIS_HOST"
value = try(local.redis_instance.host, "")
},
{
name = "REDIS_PORT"
value = try(local.redis_instance.port, "")
}
]

# Shared environment variables containing secret refs used by the Data Commons
# web service and the Data Commons data loading job
cloud_run_shared_env_variable_secrets = [
{
name = "DC_API_KEY"
name = "DC_API_KEY"
value_source = {
secret_key_ref = {
secret = google_secret_manager_secret.dc_api_key.secret_id
version = "latest"
secret = google_secret_manager_secret.dc_api_key.secret_id
version = "latest"
}
}
},
{
name = "DB_PASS"
name = "DB_PASS"
value_source = {
secret_key_ref = {
secret = google_secret_manager_secret.mysql_password.secret_id
Expand Down
95 changes: 49 additions & 46 deletions deploy/terraform-custom-datacommons/modules/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
# Custom Data Commons terraform resources

provider "google" {
project = var.project_id
region = var.region
project = var.project_id
region = var.region
user_project_override = var.user_project_override
billing_project = local.billing_project_id
billing_project = local.billing_project_id
}

# Reference the default VPC network
Expand All @@ -34,15 +34,15 @@ data "google_compute_subnetwork" "default_subnet" {

# Create redis instance
resource "google_redis_instance" "redis_instance" {
count = var.enable_redis ? 1 : 0
name = "${var.namespace}-${var.redis_instance_name}"
tier = var.redis_tier
memory_size_gb = var.redis_memory_size_gb
region = var.region
location_id = var.redis_location_id
count = var.enable_redis ? 1 : 0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the formatting. did you use a code formatting tool, or do these manually?

name = "${var.namespace}-${var.redis_instance_name}"
tier = var.redis_tier
memory_size_gb = var.redis_memory_size_gb
region = var.region
location_id = var.redis_location_id
alternative_location_id = var.redis_alternative_location_id
authorized_network = data.google_compute_network.default.self_link
replica_count = var.redis_replica_count
authorized_network = data.google_compute_network.default.self_link
replica_count = var.redis_replica_count
}

# Create MySQL instance
Expand Down Expand Up @@ -93,9 +93,9 @@ resource "google_secret_manager_secret_version" "mysql_password_version" {
}

resource "google_sql_database" "mysql_db" {
name = var.mysql_database_name
instance = google_sql_database_instance.mysql_instance.name
charset = "utf8mb4"
name = var.mysql_database_name
instance = google_sql_database_instance.mysql_instance.name
charset = "utf8mb4"
collation = "utf8mb4_unicode_ci"
}

Expand All @@ -108,23 +108,23 @@ resource "google_sql_user" "mysql_user" {

# Data commons storage bucket
resource "google_storage_bucket" "gcs_data_bucket" {
name = local.gcs_data_bucket_name
location = var.gcs_data_bucket_location
name = local.gcs_data_bucket_name
location = var.gcs_data_bucket_location
uniform_bucket_level_access = true
}

# Input 'folder' for the data loading job.
resource "google_storage_bucket_object" "gcs_data_bucket_input_folder" {
name = "${var.gcs_data_bucket_input_folder}/"
content = "Input folder"
bucket = "${google_storage_bucket.gcs_data_bucket.name}"
name = "${var.gcs_data_bucket_input_folder}/"
content = "Input folder"
bucket = google_storage_bucket.gcs_data_bucket.name
}

# Output 'folder' for the data loading job.
resource "google_storage_bucket_object" "gcs_data_bucket_output_folder" {
name = "${var.gcs_data_bucket_output_folder}/"
content = "Output folder"
bucket = "${google_storage_bucket.gcs_data_bucket.name}"
name = "${var.gcs_data_bucket_output_folder}/"
content = "Output folder"
bucket = google_storage_bucket.gcs_data_bucket.name
}

# Generate a random suffix to append to api keys.
Expand Down Expand Up @@ -181,8 +181,8 @@ resource "google_secret_manager_secret_version" "dc_api_key_version" {

# Data Commons Cloud Run Service
resource "google_cloud_run_v2_service" "dc_web_service" {
name = "${var.namespace}-datacommons-web-service"
location = var.region
name = "${var.namespace}-datacommons-web-service"
location = var.region
deletion_protection = false

template {
Expand Down Expand Up @@ -215,18 +215,18 @@ resource "google_cloud_run_v2_service" "dc_web_service" {
dynamic "env" {
for_each = local.cloud_run_shared_env_variable_secrets
content {
name = env.value.name
name = env.value.name
value_source {
secret_key_ref {
secret = env.value.value_source.secret_key_ref.secret
secret = env.value.value_source.secret_key_ref.secret
version = env.value.value_source.secret_key_ref.version
}
}
}
}
}

env {
name = "GOOGLE_ANALYTICS_TAG_ID"
name = "GOOGLE_ANALYTICS_TAG_ID"
value = var.google_analytics_tag_id != null ? var.google_analytics_tag_id : ""
}

Expand Down Expand Up @@ -256,16 +256,11 @@ resource "google_cloud_run_v2_service" "dc_web_service" {
}

env {
name = "REDIS_HOST"
value = var.enable_redis ? google_redis_instance.redis_instance[0].host : ""
}

env {
name = "MAPS_API_KEY"
name = "MAPS_API_KEY"
value_source {
secret_key_ref {
secret = google_secret_manager_secret.maps_api_key.secret_id
version = "latest"
secret = google_secret_manager_secret.maps_api_key.secret_id
version = "latest"
}
}
}
Expand Down Expand Up @@ -297,8 +292,8 @@ resource "google_cloud_run_v2_service" "dc_web_service" {

vpc_access {
network_interfaces {
network = data.google_compute_network.default.id
subnetwork = data.google_compute_subnetwork.default_subnet.name
network = data.google_compute_network.default.id
subnetwork = data.google_compute_subnetwork.default_subnet.name
}
egress = "PRIVATE_RANGES_ONLY"
}
Expand Down Expand Up @@ -334,11 +329,11 @@ resource "google_cloud_run_service_iam_member" "dc_web_service_invoker" {

# Data Commons data loading job
resource "google_cloud_run_v2_job" "dc_data_job" {
name = "${var.namespace}-datacommons-data-job"
location = var.region
name = "${var.namespace}-datacommons-data-job"
location = var.region
deletion_protection = false

template {
template {
template {
containers {
image = var.dc_data_job_image
Expand All @@ -353,7 +348,7 @@ resource "google_cloud_run_v2_job" "dc_data_job" {
# Shared environment variables
dynamic "env" {
for_each = local.cloud_run_shared_env_variables
content {
content {
name = env.value.name
value = env.value.value
}
Expand All @@ -363,12 +358,12 @@ resource "google_cloud_run_v2_job" "dc_data_job" {
dynamic "env" {
for_each = local.cloud_run_shared_env_variable_secrets
content {
name = env.value.name
name = env.value.name
value_source {
secret_key_ref {
secret = env.value.value_source.secret_key_ref.secret
secret = env.value.value_source.secret_key_ref.secret
version = env.value.value_source.secret_key_ref.version
}
}
}
}
}
Expand All @@ -378,8 +373,16 @@ resource "google_cloud_run_v2_job" "dc_data_job" {
value = "gs://${local.gcs_data_bucket_name}/${var.gcs_data_bucket_input_folder}"
}
}
vpc_access {
network_interfaces {
network = data.google_compute_network.default.id
subnetwork = data.google_compute_subnetwork.default_subnet.name
}
egress = "PRIVATE_RANGES_ONLY"
}

execution_environment = "EXECUTION_ENVIRONMENT_GEN2"
service_account = google_service_account.datacommons_service_account.email
service_account = google_service_account.datacommons_service_account.email
}
}

Expand Down
6 changes: 3 additions & 3 deletions deploy/terraform-custom-datacommons/modules/outputs.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@

output "redis_instance_host" {
description = "The hostname or IP address of the Redis instance"
value = var.enable_redis ? google_redis_instance.redis_instance[0].host : ""
value = try(local.redis_instance.host, "")
}

output "redis_instance_port" {
description = "The port number the Redis instance is listening on"
value = var.enable_redis ? google_redis_instance.redis_instance[0].port : null
value = try(local.redis_instance.port, "")
}

output "mysql_instance_connection_name" {
Expand Down Expand Up @@ -68,4 +68,4 @@ output "maps_api_key" {
description = "Maps API key"
value = local.maps_api_key
sensitive = true
}
}
13 changes: 3 additions & 10 deletions deploy/terraform-custom-datacommons/modules/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ variable "dc_api_key" {

# Optional variables

# Optional: If blank, will generate a Maps API key.
# Optional: If blank, will generate a Maps API key.
variable "maps_api_key" {
description = "Google Maps API Key"
type = string
Expand Down Expand Up @@ -182,11 +182,10 @@ variable "make_dc_web_service_public" {
}

# Data Commons Cloud Run job variables
# TODO: Change to stable after the next release
variable "dc_data_job_image" {
description = "The container image for the data job"
type = string
default = "gcr.io/datcom-ci/datacommons-data:latest"
default = "gcr.io/datcom-ci/datacommons-data:stable"
}

variable "dc_data_job_cpu" {
Expand Down Expand Up @@ -215,12 +214,6 @@ variable "vpc_network_subnet_name" {
default = "default"
}

variable "vpc_base_cidr_block" {
description = "Base CIDR block to be subdivided for VPC connectors"
type = string
default = "10.8.0.0/24"
}

# Data Commons Cloud Redis Memorystore instance variables

variable "enable_redis" {
Expand Down Expand Up @@ -262,4 +255,4 @@ variable "redis_replica_count" {
description = "Redis reserved IP range"
type = number
default = 1
}
}