Merge branch 'main' into faeture/update-schema-check-validation

.github/workflows/build_cluster_http_path.py

@@ -1,13 +1,19 @@
 import os
+import re
 
-workspace_id = os.getenv("DBT_DATABRICKS_HOST_NAME")[4:18]
+workspace_re = re.compile(r"^.*-(\d+)\..*$")
+hostname = os.getenv("DBT_DATABRICKS_HOST_NAME", "")
+matches = workspace_re.match(hostname)
+if matches:
+    workspace_id = matches.group(1)
+    print(workspace_id)
 cluster_id = os.getenv("TEST_PECO_CLUSTER_ID")
 uc_cluster_id = os.getenv("TEST_PECO_UC_CLUSTER_ID")
 http_path = f"sql/protocolv1/o/{workspace_id}/{cluster_id}"
 uc_http_path = f"sql/protocolv1/o/{workspace_id}/{uc_cluster_id}"
 
 # https://stackoverflow.com/a/72225291/5093960
-env_file = os.getenv("GITHUB_ENV")
+env_file = os.getenv("GITHUB_ENV", "")
 with open(env_file, "a") as myfile:
     myfile.write(f"DBT_DATABRICKS_CLUSTER_HTTP_PATH={http_path}\n")
     myfile.write(f"DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH={uc_http_path}\n")

CHANGELOG.md

@@ -1,3 +1,21 @@
+## dbt-databricks 1.7.4 (TBD)
+
+### Fixes
+
+- Fix for issue where long-running python models led to invalid session errors ([544](https://github.com/databricks/dbt-databricks/pull/544))
+- Allow schema to be specified in testing (thanks @case-k-git!) ([538](https://github.com/databricks/dbt-databricks/pull/538))
+
+## dbt-databricks 1.7.3 (Dec 12, 2023)
+
+### Fixes
+
+- Fix for issue where we were invoking create schema or not exists when the schema already exists (leading to permission issue) ([529](https://github.com/databricks/dbt-databricks/pull/529))
+- Fix for issue where we never reused connections ([517](https://github.com/databricks/dbt-databricks/pull/517))
+
+### Under the Hood
+
+- Refactor macro tests to be more usable ([524](https://github.com/databricks/dbt-databricks/pull/524))
+
 ## dbt-databricks 1.7.2 (Nov 30, 2023)
 
 ### Features

dbt/adapters/databricks/__version__.py

@@ -1 +1 @@
-version: str = "1.7.2"
+version: str = "1.7.3"

dbt/adapters/databricks/connections.py

@@ -1070,6 +1070,12 @@ def _cleanup_idle_connections(self) -> None:
                         self.close(conn)
                         conn.handle = LazyHandle(self._open2)
 
+    def get_thread_connection(self) -> Connection:
+        if USE_LONG_SESSIONS:
+            self._cleanup_idle_connections()
+
+        return super().get_thread_connection()
+
     def add_query(
         self,
         sql: str,

dbt/include/databricks/macros/catalog.sql → dbt/include/databricks/macros/adapters/catalog.sql

@@ -1,24 +1,3 @@
-{% macro current_catalog() -%}
-  {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}
-{% endmacro %}
-
-{% macro databricks__current_catalog() -%}
-  {% call statement('current_catalog', fetch_result=True) %}
-      select current_catalog()
-  {% endcall %}
-  {% do return(load_result('current_catalog').table) %}
-{% endmacro %}
-
-{% macro use_catalog(catalog) -%}
-  {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}
-{% endmacro %}
-
-{% macro databricks__use_catalog(catalog) -%}
-  {% call statement() %}
-    use catalog {{ adapter.quote(catalog) }}
-  {% endcall %}
-{% endmacro %}
-
 {% macro get_catalog(information_schema, schemas) -%}
   {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}
 {% endmacro %}

dbt/include/databricks/macros/adapters/databricks_catalog.sql

@@ -0,0 +1,20 @@
+{% macro current_catalog() -%}
+  {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}
+{% endmacro %}
+
+{% macro databricks__current_catalog() -%}
+  {% call statement('current_catalog', fetch_result=True) %}
+      select current_catalog()
+  {% endcall %}
+  {% do return(load_result('current_catalog').table) %}
+{% endmacro %}
+
+{% macro use_catalog(catalog) -%}
+  {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}
+{% endmacro %}
+
+{% macro databricks__use_catalog(catalog) -%}
+  {% call statement() %}
+    use catalog {{ adapter.quote(catalog) }}
+  {% endcall %}
+{% endmacro %}

dbt/include/databricks/macros/adapters/metadata.sql

@@ -0,0 +1,72 @@
+{% macro databricks__list_relations_without_caching(schema_relation) %}
+  {{ return(adapter.get_relations_without_caching(schema_relation)) }}
+{% endmacro %}
+
+{% macro show_table_extended(schema_relation) %}
+  {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}
+{% endmacro %}
+
+{% macro databricks__show_table_extended(schema_relation) %}
+  {% call statement('show_table_extended', fetch_result=True) -%}
+    show table extended in {{ schema_relation.without_identifier() }} like '{{ schema_relation.identifier }}'
+  {% endcall %}
+
+  {% do return(load_result('show_table_extended').table) %}
+{% endmacro %}
+
+{% macro show_tables(relation) %}
+  {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}
+{% endmacro %}
+
+{% macro databricks__show_tables(relation) %}
+  {% call statement('show_tables', fetch_result=True) -%}
+    show tables in {{ relation }}
+  {% endcall %}
+
+  {% do return(load_result('show_tables').table) %}
+{% endmacro %}
+
+{% macro show_views(relation) %}
+  {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}
+{% endmacro %}
+
+{% macro databricks__show_views(relation) %}
+  {% call statement('show_views', fetch_result=True) -%}
+    show views in {{ relation }}
+  {% endcall %}
+
+  {% do return(load_result('show_views').table) %}
+{% endmacro %}
+
+{% macro databricks__get_relation_last_modified(information_schema, relations) -%}
+
+  {%- call statement('last_modified', fetch_result=True) -%}
+    {% if information_schema.is_hive_metastore %}
+        {%- for relation in relations -%}
+            select '{{ relation.schema }}' as schema,
+                    '{{ relation.identifier }}' as identifier,
+                    max(timestamp) as last_modified,
+                    {{ current_timestamp() }} as snapshotted_at
+            from (describe history {{ relation.schema }}.{{ relation.identifier }})
+            {% if not loop.last %}
+            union all
+            {% endif %}
+        {%- endfor -%}
+    {% else %}
+        select table_schema as schema,
+               table_name as identifier,
+               last_altered as last_modified,
+               {{ current_timestamp() }} as snapshotted_at
+        from {{ information_schema }}.tables
+        where (
+          {%- for relation in relations -%}
+            (table_schema = '{{ relation.schema }}' and
+             table_name = '{{ relation.identifier }}'){%- if not loop.last %} or {% endif -%}
+          {%- endfor -%}
+        )
+    {% endif %}
+  {%- endcall -%}
+
+  {{ return(load_result('last_modified')) }}
+
+{% endmacro %}

dbt/include/databricks/macros/adapters/relation.sql

@@ -0,0 +1,33 @@
+{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}
+    {% set tmp_identifier = base_relation.identifier ~ suffix %}
+    {%- if as_table -%}
+        {% set tmp_relation = api.Relation.create(
+            identifier=tmp_identifier,
+            schema=base_relation.schema,
+            database=base_relation.database,
+            type='table') %}
+    {%- else -%}
+        {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}
+    {%- endif -%}
+    {% do return(tmp_relation) %}
+{% endmacro %}
+
+{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}
+  {%- set target_relation = adapter.get_relation(
+            database=database,
+            schema=schema,
+            identifier=identifier,
+            needs_information=needs_information) %}
+
+  {% if target_relation %}
+    {% do return([true, target_relation]) %}
+  {% endif %}
+
+  {%- set new_relation = api.Relation.create(
+      database=database,
+      schema=schema,
+      identifier=identifier,
+      type=type
+  ) -%}
+  {% do return([false, new_relation]) %}
+{% endmacro %}

dbt/include/databricks/macros/get_custom_name/get_custom_database.sql

@@ -0,0 +1,13 @@
+{# 
+    This is identical to the implementation in dbt-core.
+    We need to override because dbt-spark overrides to something we don't like.
+#}
+
+{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}
+    {%- set default_database = target.database -%}
+    {%- if custom_database_name is none -%}
+        {{ return(default_database) }}
+    {%- else -%}
+        {{ return(custom_database_name) }}
+    {%- endif -%}
+{%- endmacro %}