diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eac8689..b476424c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ -## dbt-databricks 1.8.7 (TBD) +## dbt-databricks 1.8.7 (October 10, 2024) + +### Features + +- Add config for generating unique tmp table names for enabling parralel replace-where (thanks @huangxingyi-git!) ([811](https://github.com/databricks/dbt-databricks/pull/811)) ### Fixes - Stop setting cluster by to None. If you want to drop liquid clustering, you will need to full-refresh ([806]https://github.com/databricks/dbt-databricks/pull/806) +- Don't define table properties on snapshot staging views (thanks @jelmerk!) ([820](https://github.com/databricks/dbt-databricks/pull/820)) ## dbt-databricks 1.8.6 (September 18, 2024) diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py index 8fe27739..192c2fde 100644 --- a/dbt/adapters/databricks/__version__.py +++ b/dbt/adapters/databricks/__version__.py @@ -1 +1 @@ -version: str = "1.8.6" +version: str = "1.8.7" diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 46cb11dd..a2184014 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING from typing import TypeVar from typing import Union +from uuid import uuid4 from dbt.adapters.base import AdapterConfig from dbt.adapters.base import PythonJobHelper @@ -103,6 +104,7 @@ class DatabricksConfig(AdapterConfig): databricks_tags: Optional[Dict[str, str]] = None tblproperties: Optional[Dict[str, str]] = None zorder: Optional[Union[List[str], str]] = None + unique_tmp_table_suffix: bool = False def check_not_found_error(errmsg: str) -> bool: @@ -685,6 +687,10 @@ def get_config_from_model(self, model: RelationConfig) -> DatabricksRelationConf f"Materialization {model.config.materialized} is not supported." ) + @available + def generate_unique_temporary_table_suffix(self, suffix_initial: str = "__dbt_tmp") -> str: + return f"{suffix_initial}_{str(uuid4())}" + @dataclass(frozen=True) class RelationAPIBase(ABC, Generic[DatabricksRelationConfig]): diff --git a/dbt/include/databricks/macros/materializations/incremental/incremental.sql b/dbt/include/databricks/macros/materializations/incremental/incremental.sql index faa59b0a..6096be4a 100644 --- a/dbt/include/databricks/macros/materializations/incremental/incremental.sql +++ b/dbt/include/databricks/macros/materializations/incremental/incremental.sql @@ -5,6 +5,7 @@ {%- set grant_config = config.get('grants') -%} {%- set tblproperties = config.get('tblproperties') -%} {%- set tags = config.get('databricks_tags') -%} + {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%} {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%} {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%} @@ -18,7 +19,11 @@ {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} {%- set target_relation = this.incorporate(type='table') -%} {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%} - + {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%} + {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%} + {%- else -%} + {%- set temp_relation_suffix = '__dbt_tmp' -%} + {%- endif -%} {#-- Set Overwrite Mode to STATIC for initial replace --#} {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%} @@ -69,7 +74,7 @@ {%- set _existing_config = adapter.get_relation_config(existing_relation) -%} {%- set model_config = adapter.get_config_from_model(config.model) -%} {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%} - {%- set temp_relation = databricks__make_temp_relation(target_relation, as_table=language != 'sql') -%} + {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%} {%- call statement('create_temp_relation', language=language) -%} {{ create_table_as(True, temp_relation, compiled_code, language) }} {%- endcall -%} diff --git a/dbt/include/databricks/macros/materializations/snapshot.sql b/dbt/include/databricks/macros/materializations/snapshot.sql index c1b8055c..3d1236a1 100644 --- a/dbt/include/databricks/macros/materializations/snapshot.sql +++ b/dbt/include/databricks/macros/materializations/snapshot.sql @@ -10,7 +10,9 @@ {# needs to be a non-temp view so that its columns can be ascertained via `describe` #} {% call statement('build_snapshot_staging_relation') %} - {{ create_view_as(tmp_relation, select) }} + create or replace view {{ tmp_relation }} + as + {{ select }} {% endcall %} {% do return(tmp_relation) %}