fix: CDI-3817 - Add support for overriding volume storage path on buc…

…ket (#692)

* feat: add availability zone selection to all dbx cluster compute policies

* fix: allow overriding volume path in bucket

* syntax fix

* drop bucket name specification from storage_location

* move storage credential creation logic gate

* adjust storage credential name

* one more name change

* add read only flag

databricks-s3-volume/iam.tf

@@ -5,7 +5,7 @@ data "aws_caller_identity" "current" {
 }
 
 data "aws_iam_policy_document" "dbx_unity_aws_role_assume_role" {
-  count = var.create_catalog ? 1 : 0
+  count = local.create_storage_credential ? 1 : 0
 
   statement {
     principals {
@@ -37,7 +37,7 @@ data "aws_iam_policy_document" "dbx_unity_aws_role_assume_role" {
 }
 
 resource "aws_iam_role" "dbx_unity_aws_role" {
-  count = var.create_catalog ? 1 : 0
+  count = local.create_storage_credential ? 1 : 0
 
   name               = local.unity_aws_role_name
   path               = local.path
@@ -46,11 +46,7 @@ resource "aws_iam_role" "dbx_unity_aws_role" {
 
 ### Policy document to access default volume bucket and assume role
 data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" {
-  count = var.create_catalog ? 1 : 0
-
-  depends_on = [
-    module.databricks_bucket
-  ]
+  count = local.create_storage_credential ? 1 : 0
 
   statement {
     sid    = "dbxSCBucketAccess"
@@ -90,13 +86,13 @@ data "aws_iam_policy_document" "volume_bucket_dbx_unity_access" {
 }
 
 resource "aws_iam_policy" "dbx_unity_access_policy" {
-  count = var.create_catalog ? 1 : 0
+  count = local.create_storage_credential ? 1 : 0
 
   policy = data.aws_iam_policy_document.volume_bucket_dbx_unity_access[0].json
 }
 
 resource "aws_iam_role_policy_attachment" "dbx_unity_aws_access" {
-  count = var.create_catalog ? 1 : 0
+  count = local.create_storage_credential ? 1 : 0
 
   policy_arn = aws_iam_policy.dbx_unity_access_policy[0].arn
   role       = aws_iam_role.dbx_unity_aws_role[0].name

databricks-s3-volume/main.tf

@@ -14,6 +14,12 @@ locals {
   bucket_name            = var.volume_bucket != null ? var.volume_bucket : (
     var.override_bucket_name != null ? var.override_bucket_name : replace(var.catalog_name, "_", "-") # buckets don't work with underscores
   )
+
+  create_storage_credential = var.create_catalog ? true : (var.create_storage_credential ? true : false)
+
+  # Allow overriding the storage location in case of an existing bucket
+  storage_location = var.override_storage_location != null ? var.override_storage_location : "s3://${local.bucket_name}/${local.schema_name}/${local.volume_name}"
+
 }
 
 ### Databricks storage credential - allows workspace to access an external location.
@@ -29,11 +35,12 @@ resource "databricks_storage_credential" "volume" {
     module.databricks_bucket
   ]
 
-  name = local.catalog_name
+  name = var.create_catalog ? local.catalog_name : local.volume_name
   aws_iam_role {
     role_arn = aws_iam_role.dbx_unity_aws_role[0].arn
   }
   comment = "Managed by Terraform - access for ${var.catalog_name}"
+  read_only       = var.read_only_volume
 }
 
 # upstream external location sometimes takes a moment to register
@@ -47,10 +54,11 @@ resource "databricks_external_location" "volume" {
   count = var.create_storage_credential ? 1 : 0
   depends_on      = [time_sleep.wait_30_seconds]
 
-  name            = local.catalog_name
+  name            = var.create_catalog ? local.catalog_name : local.volume_name
   url             = "s3://${local.bucket_name}"
   credential_name = databricks_storage_credential.volume[0].name
   comment         = "Managed by Terraform - access for ${var.catalog_name}"
+  read_only       = var.read_only_volume
 }
 
 # New catalog, schema, and volume
@@ -87,7 +95,7 @@ resource "databricks_volume" "volume" {
   catalog_name     = local.catalog_name
   schema_name      = local.schema_name
   volume_type      = "EXTERNAL"
-  storage_location = "s3://${local.bucket_name}/${local.schema_name}/${local.volume_name}"
+  storage_location = local.storage_location
   owner            = var.owner
   comment          = "This volume is managed by Terraform - ${var.volume_comment}"
 }

databricks-s3-volume/variables.tf

@@ -136,6 +136,18 @@ variable "override_bucket_name" {
   default     = null
 }
 
+variable "override_storage_location" {
+  description = "(Optional) Prefix to use for the storage location in case of an existing bucket (e.g. 's3://bucket' or 's3://bucket/prefix')"
+  type        = string
+  default     = null
+}
+
+variable "read_only_volume" {
+  description = "(Optional) Flag to set volume as read-only"
+  type        = bool
+  default     = false
+}
+
 variable "tags" {
   description = "REQUIRED: Tags to include for this environment."
   type = object({