Skip to content

Commit

Permalink
Allow-departmental-ecs-role-access-the-etl-bucket (#2115)
Browse files Browse the repository at this point in the history 
* add departmental ecs role access the etl bucket

* pass to main

* allow departmental ecs role to access teh etl bucket
  • Loading branch information
@Tian-2017
Tian-2017 authored Feb 9, 2025
1 parent b923169 commit c204262
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 3 deletions.
37 changes: 36 additions & 1 deletion terraform/core/05-departments.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ module "department_housing_repairs" {
sso_instance_arn = local.sso_instance_arn
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_parking" {
Expand Down Expand Up @@ -69,6 +71,8 @@ module "department_parking" {
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-parking@hackney.gov.uk"
departmental_airflow_user = true
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_finance" {
Expand Down Expand Up @@ -99,6 +103,8 @@ module "department_finance" {
sso_instance_arn = local.sso_instance_arn
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_data_and_insight" {
Expand Down Expand Up @@ -130,6 +136,8 @@ module "department_data_and_insight" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-datainsight@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_env_enforcement" {
Expand Down Expand Up @@ -160,6 +168,8 @@ module "department_env_enforcement" {
sso_instance_arn = local.sso_instance_arn
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_planning" {
Expand Down Expand Up @@ -191,6 +201,8 @@ module "department_planning" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-planning@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_unrestricted" {
Expand Down Expand Up @@ -222,6 +234,8 @@ module "department_unrestricted" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
departmental_airflow_user = true
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_sandbox" {
Expand Down Expand Up @@ -253,6 +267,8 @@ module "department_sandbox" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-sandbox@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_benefits_and_housing_needs" {
Expand Down Expand Up @@ -284,6 +300,8 @@ module "department_benefits_and_housing_needs" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-benefits-housing-needs@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_revenues" {
Expand Down Expand Up @@ -315,6 +333,9 @@ module "department_revenues" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-revenues@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn

}

module "department_environmental_services" {
Expand Down Expand Up @@ -347,6 +368,8 @@ module "department_environmental_services" {
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-environmental-services@hackney.gov.uk"
departmental_airflow_user = true
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_housing" {
Expand Down Expand Up @@ -379,6 +402,8 @@ module "department_housing" {
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-housing@hackney.gov.uk"
departmental_airflow_user = true
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_children_and_education" {
Expand Down Expand Up @@ -410,6 +435,8 @@ module "department_children_and_education" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-children-and-family-services@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_customer_services" {
Expand Down Expand Up @@ -441,6 +468,8 @@ module "department_customer_services" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-customer-services@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_hr_and_od" {
Expand Down Expand Up @@ -472,6 +501,8 @@ module "department_hr_and_od" {
identity_store_id = local.identity_store_id
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-hr-and-od@hackney.gov.uk"
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_streetscene" {
Expand Down Expand Up @@ -504,6 +535,8 @@ module "department_streetscene" {
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-streetscene@hackney.gov.uk"
departmental_airflow_user = true
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}

module "department_children_family_services" {
Expand Down Expand Up @@ -536,4 +569,6 @@ module "department_children_family_services" {
google_group_admin_display_name = local.google_group_admin_display_name
google_group_display_name = "saml-aws-data-platform-collaborator-cfs@hackney.gov.uk"
departmental_airflow_user = true
}
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
mwaa_key_arn = aws_kms_key.mwaa_key.arn
}
8 changes: 8 additions & 0 deletions terraform/core/99-outputs.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,11 @@ output "identity_store_id" {
output "arn" {
value = local.sso_instance_arn
}

output "mwaa_etl_scripts_bucket_arn" {
value = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
}

output "mwaa_key_arn" {
value = aws_kms_key.mwaa_key.arn
}
8 changes: 8 additions & 0 deletions terraform/modules/department/01-inputs-required.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,11 @@ variable "redshift_port" {
description = "Port that the redshift cluster is running on"
type = number
}

variable "mwaa_etl_scripts_bucket_arn" {
type = string
}

variable "mwaa_key_arn" {
type = string
}
9 changes: 7 additions & 2 deletions terraform/modules/department/50-aws-iam-policies.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,8 @@ data "aws_iam_policy_document" "s3_department_access" {
var.athena_storage_bucket.kms_key_arn,
var.glue_scripts_bucket.kms_key_arn,
var.spark_ui_output_storage_bucket.kms_key_arn,
var.glue_temp_storage_bucket.kms_key_arn
var.glue_temp_storage_bucket.kms_key_arn,
var.mwaa_key_arn
]
}

Expand Down Expand Up @@ -216,7 +217,11 @@ data "aws_iam_policy_document" "s3_department_access" {
var.glue_temp_storage_bucket.bucket_arn,

var.spark_ui_output_storage_bucket.bucket_arn,
"${var.spark_ui_output_storage_bucket.bucket_arn}/${local.department_identifier}/*"
"${var.spark_ui_output_storage_bucket.bucket_arn}/${local.department_identifier}/*",

var.mwaa_etl_scripts_bucket_arn,
"${var.mwaa_etl_scripts_bucket_arn}/${local.department_identifier}/*",
"${var.mwaa_etl_scripts_bucket_arn}/${local.department_identifier}/unrestricted/*",
]
}

Expand Down

0 comments on commit c204262

Please sign in to comment.