materialize-databricks: quote schema and volume identifier

estuary · Feb 29, 2024 · c0a503a · c0a503a
1 parent 7e1970a
commit c0a503a
Show file tree

Hide file tree

Showing 5 changed files with 125 additions and 125 deletions.
diff --git a/materialize-databricks/driver.go b/materialize-databricks/driver.go
@@ -216,7 +216,7 @@ func newTransactor(
 	}
 
 	// Create volume for storing staged files
-	if _, err := d.store.conn.ExecContext(ctx, fmt.Sprintf("CREATE VOLUME IF NOT EXISTS %s.%s;", cfg.SchemaName, volumeName)); err != nil {
+	if _, err := d.store.conn.ExecContext(ctx, fmt.Sprintf("CREATE VOLUME IF NOT EXISTS `%s`.`%s`;", cfg.SchemaName, volumeName)); err != nil {
 		return nil, fmt.Errorf("Exec(CREATE VOLUME IF NOT EXISTS %s;): %w", volumeName, err)
 	}
 

diff --git a/tests/materialize/materialize-databricks/cleanup.sh b/tests/materialize/materialize-databricks/cleanup.sh
@@ -9,12 +9,12 @@ function dropTable() {
 }
 
 # Remove materialized tables.
-dropTable "\`$DATABRICKS_CATALOG\`.default.simple"
-dropTable "\`$DATABRICKS_CATALOG\`.default.duplicate_keys_standard"
-dropTable "\`$DATABRICKS_CATALOG\`.default.duplicate_keys_delta"
-dropTable "\`$DATABRICKS_CATALOG\`.default.duplicate_keys_delta_exclude_flow_doc"
-dropTable "\`$DATABRICKS_CATALOG\`.default.multiple_types"
-dropTable "\`$DATABRICKS_CATALOG\`.default.formatted_strings"
+dropTable "\`$DATABRICKS_CATALOG\`.\`some-schema\`.simple"
+dropTable "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_standard"
+dropTable "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_delta"
+dropTable "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_delta_exclude_flow_doc"
+dropTable "\`$DATABRICKS_CATALOG\`.\`some-schema\`.multiple_types"
+dropTable "\`$DATABRICKS_CATALOG\`.\`some-schema\`.formatted_strings"
 
-yes | dbsqlcli -e "delete from \`$DATABRICKS_CATALOG\`.\`default\`.flow_materializations_v2 where MATERIALIZATION='tests/materialize-databricks/materialize';"
+yes | dbsqlcli -e "delete from \`$DATABRICKS_CATALOG\`.\`some-schema\`.flow_materializations_v2 where MATERIALIZATION='tests/materialize-databricks/materialize';"
 
diff --git a/tests/materialize/materialize-databricks/fetch.sh b/tests/materialize/materialize-databricks/fetch.sh
@@ -8,14 +8,14 @@ function exportToJsonl() {
 	dbsqlcli -e "SELECT * FROM $1;" | mlr --icsv --ojsonl cat | jq -c "{ table: \"$1\", row: . }"
 }
 
-exportToJsonl "\`$DATABRICKS_CATALOG\`.default.simple"
-exportToJsonl "\`$DATABRICKS_CATALOG\`.default.duplicate_keys_standard"
-exportToJsonl "\`$DATABRICKS_CATALOG\`.default.duplicate_keys_delta"
-exportToJsonl "\`$DATABRICKS_CATALOG\`.default.duplicate_keys_delta_exclude_flow_doc"
-exportToJsonl "\`$DATABRICKS_CATALOG\`.default.multiple_types"
+exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.simple"
+exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_standard"
+exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_delta"
+exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.duplicate_keys_delta_exclude_flow_doc"
+exportToJsonl "\`$DATABRICKS_CATALOG\`.\`some-schema\`.multiple_types"
 
 # due to a bug in the cli, we can't use this at the moment,
 # see https://github.com/databricks/databricks-sql-cli/issues/51
 # exportToJsonl "\`$DATABRICKS_CATALOG\`.default.formatted_strings"
-dbsqlcli -e "SELECT id, datetime::string, date::string, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document FROM main.default.formatted_strings;" | \
-	mlr --icsv --ojsonl cat | jq -c "del(.flow_document) | { table: \"\`$DATABRICKS_CATALOG\`.default.formatted_strings\", row: . }"
+dbsqlcli -e "SELECT id, datetime::string, date::string, flow_published_at, int_and_str, int_str, num_and_str, num_str, time, flow_document FROM main.\`some-schema\`.formatted_strings;" | \
+	mlr --icsv --ojsonl cat | jq -c "del(.flow_document) | { table: \"\`$DATABRICKS_CATALOG\`.\`some-schema\`.formatted_strings\", row: . }"
diff --git a/tests/materialize/materialize-databricks/setup.sh b/tests/materialize/materialize-databricks/setup.sh
@@ -17,7 +17,7 @@ config_json_template='{
    "address": "$DATABRICKS_HOST_NAME",
    "http_path": "$DATABRICKS_HTTP_PATH",
    "catalog_name": "$DATABRICKS_CATALOG",
-	 "schema_name": "default",
+	 "schema_name": "some-schema",
 	 "advanced": { "updateDelay": "0s" },
    "credentials": {
 			"auth_type": "PAT",