From 717188b19e0e956360a5958be60a4d4c3cf297d1 Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:32:41 -0800 Subject: [PATCH 01/11] Updating CHANGELOG for 1.7.x (#502) From a92bede8f7bf8b2985420d0ff924bf90b9a44f99 Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Fri, 27 Oct 2023 15:44:20 -0700 Subject: [PATCH 02/11] beginning to reorg the functional tests --- tests/functional/adapter/aliases/fixtures.py | 10 ++ .../adapter/aliases/test_aliases.py | 40 ++++++++ tests/functional/adapter/basic/test_base.py | 5 + .../test_docs_generate.py} | 98 ++----------------- tests/functional/adapter/basic/test_empty.py | 5 + .../adapter/basic/test_ephemeral.py | 5 + .../adapter/basic/test_generic_tests.py | 5 + .../adapter/basic/test_incremental.py | 9 ++ .../adapter/basic/test_singular_tests.py | 5 + .../basic/test_singular_tests_ephemeral.py | 5 + .../adapter/basic/test_snapshot_check_cols.py | 5 + .../adapter/basic/test_snapshot_timestamp.py | 5 + tests/functional/adapter/basic/typing.py | 28 ++++++ .../adapter/ephemeral/test_ephemeral_multi.py | 13 +++ tests/functional/adapter/test_ephemeral.py | 13 --- 15 files changed, 149 insertions(+), 102 deletions(-) create mode 100644 tests/functional/adapter/aliases/fixtures.py create mode 100644 tests/functional/adapter/aliases/test_aliases.py create mode 100644 tests/functional/adapter/basic/test_base.py rename tests/functional/adapter/{test_basic.py => basic/test_docs_generate.py} (64%) create mode 100644 tests/functional/adapter/basic/test_empty.py create mode 100644 tests/functional/adapter/basic/test_ephemeral.py create mode 100644 tests/functional/adapter/basic/test_generic_tests.py create mode 100644 tests/functional/adapter/basic/test_incremental.py create mode 100644 tests/functional/adapter/basic/test_singular_tests.py create mode 100644 tests/functional/adapter/basic/test_singular_tests_ephemeral.py create mode 100644 tests/functional/adapter/basic/test_snapshot_check_cols.py create mode 100644 tests/functional/adapter/basic/test_snapshot_timestamp.py create mode 100644 tests/functional/adapter/basic/typing.py create mode 100644 tests/functional/adapter/ephemeral/test_ephemeral_multi.py delete mode 100644 tests/functional/adapter/test_ephemeral.py diff --git a/tests/functional/adapter/aliases/fixtures.py b/tests/functional/adapter/aliases/fixtures.py new file mode 100644 index 000000000..2f7803a69 --- /dev/null +++ b/tests/functional/adapter/aliases/fixtures.py @@ -0,0 +1,10 @@ +# macros # +MACROS__CAST_SQL = """ +{% macro string_literal(s) -%} + {{ adapter.dispatch('string_literal', macro_namespace='test')(s) }} +{%- endmacro %} + +{% macro databricks__string_literal(s) %} + cast('{{ s }}' as STRING) +{% endmacro %} +""" diff --git a/tests/functional/adapter/aliases/test_aliases.py b/tests/functional/adapter/aliases/test_aliases.py new file mode 100644 index 000000000..9b63192bb --- /dev/null +++ b/tests/functional/adapter/aliases/test_aliases.py @@ -0,0 +1,40 @@ +from dbt.tests.adapter.aliases.test_aliases import ( + BaseAliases, + BaseAliasErrors, + BaseSameAliasDifferentSchemas, + BaseSameAliasDifferentDatabases, +) +from dbt.tests.adapter.aliases import fixtures as dbt_fixtures +import pytest + +from tests.functional.adapter.aliases import fixtures as databricks_fixtures + + +macro_override = { + "cast.sql": databricks_fixtures.MACROS__CAST_SQL, + "expect_value.sql": dbt_fixtures.MACROS__EXPECT_VALUE_SQL, +} + + +class TestDatabricksAliases(BaseAliases): + @pytest.fixture(scope="class") + def macros(self): + return macro_override + + +class TestDatabricksAliasErrors(BaseAliasErrors): + @pytest.fixture(scope="class") + def macros(self): + return macro_override + + +class TestDatabricksSameAliasDifferentSchemas(BaseSameAliasDifferentSchemas): + @pytest.fixture(scope="class") + def macros(self): + return macro_override + + +class TestDatabricksSameAliasDifferentDatabases(BaseSameAliasDifferentDatabases): + @pytest.fixture(scope="class") + def macros(self): + return macro_override diff --git a/tests/functional/adapter/basic/test_base.py b/tests/functional/adapter/basic/test_base.py new file mode 100644 index 000000000..dc11a5f81 --- /dev/null +++ b/tests/functional/adapter/basic/test_base.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations + + +class TestDatabricksSimpleMaterializations(BaseSimpleMaterializations): + pass diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/basic/test_docs_generate.py similarity index 64% rename from tests/functional/adapter/test_basic.py rename to tests/functional/adapter/basic/test_docs_generate.py index 7fc27dce9..263b3ad19 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/basic/test_docs_generate.py @@ -1,69 +1,16 @@ -import pytest - -from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog -from dbt.tests.util import AnyInteger, AnyString - -from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations -from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests -from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral -from dbt.tests.adapter.basic.test_empty import BaseEmpty -from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral -from dbt.tests.adapter.basic.test_incremental import BaseIncremental, BaseIncrementalNotSchemaChange -from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests -from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols -from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp -from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod +from dbt.tests.adapter.basic import expected_catalog from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences +from dbt.tests.util import AnyString +import pytest -class TestSimpleMaterializationsDatabricks(BaseSimpleMaterializations): - pass - - -class TestSingularTestsDatabricks(BaseSingularTests): - pass - - -class TestSingularTestsEphemeralDatabricks(BaseSingularTestsEphemeral): - pass - - -class TestEmptyDatabricks(BaseEmpty): - pass - - -class TestEphemeralDatabricks(BaseEphemeral): - pass - - -class TestIncrementalDatabricks(BaseIncremental): - pass - - -class TestIncrementalNotSchemaChangeDatabricks(BaseIncrementalNotSchemaChange): - pass - - -class TestGenericTestsDatabricks(BaseGenericTests): - pass - - -class TestSnapshotCheckColsDatabricks(BaseSnapshotCheckCols): - pass - - -class TestSnapshotTimestampDatabricks(BaseSnapshotTimestamp): - pass - - -class TestBaseAdapterMethodDatabricks(BaseAdapterMethod): - pass +from tests.functional.adapter.basic.typing import AnyLongType, StatsLikeDict -class TestDocsGenerateDatabricks(BaseDocsGenerate): +class TestDatabricksDocsGenerate(BaseDocsGenerate): @pytest.fixture(scope="class") def expected_catalog(self, project): - return base_expected_catalog( + return expected_catalog.base_expected_catalog( project, role=AnyString(), id_type=AnyLongType(), @@ -71,11 +18,11 @@ def expected_catalog(self, project): time_type="timestamp", view_type="view", table_type="table", - model_stats=_StatsLikeDict(), + model_stats=StatsLikeDict(), ) -class TestDocsGenReferencesDatabricks(BaseDocsGenReferences): +class TestDatabricksDocsGenReferences(BaseDocsGenReferences): @pytest.fixture(scope="class") def expected_catalog(self, project): return self.expected_references_catalog( @@ -87,7 +34,7 @@ def expected_catalog(self, project): bigint_type=AnyLongType(), view_type="view", table_type="table", - model_stats=_StatsLikeDict(), + model_stats=StatsLikeDict(), ) # Temporary until upstream fixes to allow 0-based indexing @@ -214,30 +161,3 @@ def expected_references_catalog( }, }, } - - -class _StatsLikeDict: - """Any stats-like dict. Use this in assert calls""" - - def __eq__(self, other): - return ( - isinstance(other, dict) - and "has_stats" in other - and ( - other["has_stats"] - == { - "id": "has_stats", - "label": "Has Stats?", - "value": AnyInteger(), - "description": "Indicates whether there are statistics for this table", - "include": False, - } - ) - ) - - -class AnyLongType: - """Allows bigint and long to be treated equivalently""" - - def __eq__(self, other): - return isinstance(other, str) and other in ("bigint", "long") diff --git a/tests/functional/adapter/basic/test_empty.py b/tests/functional/adapter/basic/test_empty.py new file mode 100644 index 000000000..daa5300e9 --- /dev/null +++ b/tests/functional/adapter/basic/test_empty.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_empty import BaseEmpty + + +class TestDatabricksEmpty(BaseEmpty): + pass diff --git a/tests/functional/adapter/basic/test_ephemeral.py b/tests/functional/adapter/basic/test_ephemeral.py new file mode 100644 index 000000000..59f77dc51 --- /dev/null +++ b/tests/functional/adapter/basic/test_ephemeral.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral + + +class TestDatabricksEphemeral(BaseEphemeral): + pass diff --git a/tests/functional/adapter/basic/test_generic_tests.py b/tests/functional/adapter/basic/test_generic_tests.py new file mode 100644 index 000000000..ed6949ccc --- /dev/null +++ b/tests/functional/adapter/basic/test_generic_tests.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests + + +class TestDatabricksGenericTests(BaseGenericTests): + pass diff --git a/tests/functional/adapter/basic/test_incremental.py b/tests/functional/adapter/basic/test_incremental.py new file mode 100644 index 000000000..a5bfdc26a --- /dev/null +++ b/tests/functional/adapter/basic/test_incremental.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.basic.test_incremental import BaseIncremental, BaseIncrementalNotSchemaChange + + +class TestDatabricksIncremental(BaseIncremental): + pass + + +class TestDatabricksIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): + pass diff --git a/tests/functional/adapter/basic/test_singular_tests.py b/tests/functional/adapter/basic/test_singular_tests.py new file mode 100644 index 000000000..937a3fb0c --- /dev/null +++ b/tests/functional/adapter/basic/test_singular_tests.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests + + +class TestDatabricksSingularTests(BaseSingularTests): + pass diff --git a/tests/functional/adapter/basic/test_singular_tests_ephemeral.py b/tests/functional/adapter/basic/test_singular_tests_ephemeral.py new file mode 100644 index 000000000..f26651ff7 --- /dev/null +++ b/tests/functional/adapter/basic/test_singular_tests_ephemeral.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral + + +class TestDatabricksSingularTestsEphemeral(BaseSingularTestsEphemeral): + pass diff --git a/tests/functional/adapter/basic/test_snapshot_check_cols.py b/tests/functional/adapter/basic/test_snapshot_check_cols.py new file mode 100644 index 000000000..e15198bf9 --- /dev/null +++ b/tests/functional/adapter/basic/test_snapshot_check_cols.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols + + +class TestDatabricksSnapshotCheckCols(BaseSnapshotCheckCols): + pass diff --git a/tests/functional/adapter/basic/test_snapshot_timestamp.py b/tests/functional/adapter/basic/test_snapshot_timestamp.py new file mode 100644 index 000000000..2b48324d8 --- /dev/null +++ b/tests/functional/adapter/basic/test_snapshot_timestamp.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp + + +class TestDatabricksSnapshotTimestamp(BaseSnapshotTimestamp): + pass diff --git a/tests/functional/adapter/basic/typing.py b/tests/functional/adapter/basic/typing.py new file mode 100644 index 000000000..ff5b2e30a --- /dev/null +++ b/tests/functional/adapter/basic/typing.py @@ -0,0 +1,28 @@ +from dbt.tests.util import AnyInteger + + +class StatsLikeDict: + """Any stats-like dict. Use this in assert calls""" + + def __eq__(self, other): + return ( + isinstance(other, dict) + and "has_stats" in other + and ( + other["has_stats"] + == { + "id": "has_stats", + "label": "Has Stats?", + "value": AnyInteger(), + "description": "Indicates whether there are statistics for this table", + "include": False, + } + ) + ) + + +class AnyLongType: + """Allows bigint and long to be treated equivalently""" + + def __eq__(self, other): + return isinstance(other, str) and other in ("bigint", "long") diff --git a/tests/functional/adapter/ephemeral/test_ephemeral_multi.py b/tests/functional/adapter/ephemeral/test_ephemeral_multi.py new file mode 100644 index 000000000..698ad1686 --- /dev/null +++ b/tests/functional/adapter/ephemeral/test_ephemeral_multi.py @@ -0,0 +1,13 @@ +from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti +from dbt.tests import util + + +class TestDatabricksEphemeralMulti(BaseEphemeralMulti): + def test_ephemeral_multi(self, project): + util.run_dbt(["seed"]) + results = util.run_dbt(["run"]) + assert len(results) == 3 + + util.check_relations_equal(project.adapter, ["seed", "dependent"]) + util.check_relations_equal(project.adapter, ["seed", "double_dependent"]) + util.check_relations_equal(project.adapter, ["seed", "super_dependent"]) diff --git a/tests/functional/adapter/test_ephemeral.py b/tests/functional/adapter/test_ephemeral.py deleted file mode 100644 index 781fb8161..000000000 --- a/tests/functional/adapter/test_ephemeral.py +++ /dev/null @@ -1,13 +0,0 @@ -from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti -from dbt.tests.util import run_dbt, check_relations_equal - - -class TestEphemeralMultiDatabricks(BaseEphemeralMulti): - def test_ephemeral_multi(self, project): - run_dbt(["seed"]) - results = run_dbt(["run"]) - assert len(results) == 3 - - check_relations_equal(project.adapter, ["seed", "dependent"]) - check_relations_equal(project.adapter, ["seed", "double_dependent"]) - check_relations_equal(project.adapter, ["seed", "super_dependent"]) From 9bdf3fb39f96bb40d685f029978fc4f8bbdf0a59 Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Fri, 10 Nov 2023 10:38:00 -0800 Subject: [PATCH 03/11] Beginning to reorg the functional tests (#492) From aa4fab43195948b9cd17b1043cc3fcf7a39351c5 Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Fri, 10 Nov 2023 13:54:17 -0800 Subject: [PATCH 04/11] does this work? --- dbt/adapters/databricks/impl.py | 72 +++++++++++++++++------ dbt/include/databricks/macros/catalog.sql | 29 +++++++-- 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 4d8990c44..78373ddf1 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -22,7 +22,7 @@ from dbt.adapters.base import AdapterConfig, PythonJobHelper from dbt.adapters.base.impl import catch_as_completed from dbt.adapters.base.meta import available -from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.base.relation import BaseRelation, InformationSchema from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability from dbt.adapters.spark.impl import ( SparkAdapter, @@ -59,7 +59,7 @@ CURRENT_CATALOG_MACRO_NAME = "current_catalog" USE_CATALOG_MACRO_NAME = "use_catalog" - +GET_CATALOG_MACRO_NAME = "get_catalog" SHOW_TABLE_EXTENDED_MACRO_NAME = "show_table_extended" SHOW_TABLES_MACRO_NAME = "show_tables" SHOW_VIEWS_MACRO_NAME = "show_views" @@ -449,16 +449,54 @@ def parse_columns_from_information( # type: ignore[override] columns.append(column) return columns - def get_catalog( - self, manifest: Manifest, selected_nodes: Optional[Set[Any]] = None - ) -> Tuple[Table, List[Exception]]: - if selected_nodes: - relations: Set[BaseRelation] = { - self.Relation.create_from(self.config, n) for n in selected_nodes - } - else: - relations = set(self._get_catalog_relations(manifest)) - return self.get_catalog_by_relations(manifest, relations) + def get_catalog(self, manifest: Manifest) -> Tuple[Table, List[Exception]]: + schema_map = self._get_catalog_schemas(manifest) + + with executor(self.config) as tpe: + futures: List[Future[Table]] = [] + for info, schemas in schema_map.items(): + if is_hive_metastore(info.database): + for schema in schemas: + futures.append( + tpe.submit_connected( + self, + "hive_metastore", + self._get_hive_catalog, + schema, + "*", + ) + ) + else: + name = ".".join([str(info.database), "information_schema"]) + fut = tpe.submit_connected( + self, + name, + self._get_one_unity_catalog, + info, + schemas, + manifest, + ) + futures.append(fut) + catalogs, exceptions = catch_as_completed(futures) + return catalogs, exceptions + + def _get_one_unity_catalog( + self, info: InformationSchema, schemas: Set[str], manifest: Manifest + ) -> Table: + kwargs = { + "information_schema": info, + "schemas": schemas, + } + table = self.execute_macro( + GET_CATALOG_MACRO_NAME, + kwargs=kwargs, + # pass in the full manifest, so we get any local project + # overrides + manifest=manifest, + ) + + results = self._catalog_filter_table(table, manifest) # type: ignore[arg-type] + return results def get_catalog_by_relations( self, manifest: Manifest, relations: Set[BaseRelation] @@ -475,13 +513,14 @@ def get_catalog_by_relations( schema_map[relation.schema].append(relation) for schema, schema_relations in schema_map.items(): + table_names = extract_identifiers(schema_relations) futures.append( tpe.submit_connected( self, "hive_metastore", self._get_hive_catalog, schema, - schema_relations, + get_identifier_list_string(table_names), ) ) else: @@ -502,16 +541,15 @@ def get_catalog_by_relations( def _get_hive_catalog( self, schema: str, - relations: Set[BaseRelation], + identifier: str, ) -> Table: - table_names = extract_identifiers(relations) columns: List[Dict[str, Any]] = [] - if len(table_names) > 0: + if identifier: schema_relation = self.Relation.create( database="hive_metastore", schema=schema, - identifier=get_identifier_list_string(table_names), + identifier=get_identifier_list_string(identifier), quote_policy=self.config.quoting, ) for relation, information in self._list_relations_with_information(schema_relation): diff --git a/dbt/include/databricks/macros/catalog.sql b/dbt/include/databricks/macros/catalog.sql index 0749bca95..144122466 100644 --- a/dbt/include/databricks/macros/catalog.sql +++ b/dbt/include/databricks/macros/catalog.sql @@ -19,6 +19,27 @@ {% endcall %} {% endmacro %} +{% macro get_catalog(information_schema, schemas) -%} + {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }} +{% endmacro %} + +{% macro databricks__get_catalog(information_schema, schemas) -%} + + {% set query %} + with tables as ( + {{ databricks__get_catalog_tables_sql(information_schema) }} + {{ databricks__get_catalog_schemas_where_clause_sql(schemas) }} + ), + columns as ( + {{ databricks__get_catalog_columns_sql(information_schema) }} + {{ databricks__get_catalog_schemas_where_clause_sql(schemas) }} + ) + {{ databricks__get_catalog_results_sql() }} + {%- endset -%} + + {{ return(run_query(query)) }} +{%- endmacro %} + {% macro databricks__get_catalog_relations(information_schema, relations) -%} {% set query %} @@ -72,7 +93,7 @@ {% macro databricks__get_catalog_schemas_where_clause_sql(schemas) -%} where ({%- for schema in schemas -%} - upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + table_schema = lower('{{ schema }}'){%- if not loop.last %} or {% endif -%} {%- endfor -%}) {%- endmacro %} @@ -82,12 +103,12 @@ {%- for relation in relations -%} {% if relation.schema and relation.identifier %} ( - upper(table_schema) = upper('{{ relation.schema }}') - and upper(table_name) = upper('{{ relation.identifier }}') + table_schema = lower('{{ relation.schema }}') + and table_name = lower('{{ relation.identifier }}') ) {% elif relation.schema %} ( - upper(table_schema) = upper('{{ relation.schema }}') + table_schema = lower('{{ relation.schema }}') ) {% else %} {% do exceptions.raise_compiler_error( From 3bba7dc8c00caa82475764c59c01adb0f0cd375f Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Fri, 10 Nov 2023 15:21:18 -0800 Subject: [PATCH 05/11] fix a bug --- dbt/adapters/databricks/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 78373ddf1..3f3438ea4 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -549,7 +549,7 @@ def _get_hive_catalog( schema_relation = self.Relation.create( database="hive_metastore", schema=schema, - identifier=get_identifier_list_string(identifier), + identifier=identifier, quote_policy=self.config.quoting, ) for relation, information in self._list_relations_with_information(schema_relation): From 7abec82cc006d102694137e24183b37ca11d842f Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Fri, 10 Nov 2023 15:26:40 -0800 Subject: [PATCH 06/11] fix linting --- dbt/adapters/databricks/impl.py | 2 +- dbt/adapters/databricks/relation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/databricks/impl.py b/dbt/adapters/databricks/impl.py index 3f3438ea4..e4affbde6 100644 --- a/dbt/adapters/databricks/impl.py +++ b/dbt/adapters/databricks/impl.py @@ -449,7 +449,7 @@ def parse_columns_from_information( # type: ignore[override] columns.append(column) return columns - def get_catalog(self, manifest: Manifest) -> Tuple[Table, List[Exception]]: + def get_catalog(self, manifest: Manifest) -> Tuple[Table, List[Exception]]: # type: ignore schema_map = self._get_catalog_schemas(manifest) with executor(self.config) as tpe: diff --git a/dbt/adapters/databricks/relation.py b/dbt/adapters/databricks/relation.py index 29dd60634..07046f98b 100644 --- a/dbt/adapters/databricks/relation.py +++ b/dbt/adapters/databricks/relation.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Any, Dict, Optional, Set, Type +from typing import Any, Dict, Iterable, Optional, Set, Type from dbt.contracts.relation import ( ComponentName, ) @@ -141,5 +141,5 @@ def is_hive_metastore(database: Optional[str]) -> bool: return database is None or database.lower() == "hive_metastore" -def extract_identifiers(relations: Set[BaseRelation]) -> Set[str]: +def extract_identifiers(relations: Iterable[BaseRelation]) -> Set[str]: return {r.identifier for r in relations if r.identifier is not None} From befdce393c309519e39c6935a384e9e6547bfb77 Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:58:25 -0800 Subject: [PATCH 07/11] Another attempt to improve metadata gathering performance (#503) From fdc79ca6b779a077bec726782a72e988ce6535d3 Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Fri, 10 Nov 2023 15:59:10 -0800 Subject: [PATCH 08/11] forgot to update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d9dc41f..c1eac5526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## dbt-databricks 1.7.x (TBD) +### Under the Hood + +- Another attempt to improve catalog gathering performance ([503](https://github.com/databricks/dbt-databricks/pull/503)) + ## dbt-databricks 1.7.0 (November 9, 2023) ### Features From 7802292fb9f5a90ad62c5ff6ea578f41f9450e8e Mon Sep 17 00:00:00 2001 From: Ben Cassell <98852248+benc-db@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:00:25 -0800 Subject: [PATCH 09/11] Update Changelog (#504) From 50af43b81dea23c651cee28aa61424058bc6b2f5 Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Mon, 13 Nov 2023 14:25:17 -0800 Subject: [PATCH 10/11] up version number --- CHANGELOG.md | 2 +- dbt/adapters/databricks/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1eac5526..01db76bf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## dbt-databricks 1.7.x (TBD) +## dbt-databricks 1.7.1 (Nov 13, 2023) ### Under the Hood diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py index c97ea9909..59bc4f8db 100644 --- a/dbt/adapters/databricks/__version__.py +++ b/dbt/adapters/databricks/__version__.py @@ -1 +1 @@ -version: str = "1.7.0" +version: str = "1.7.1" From d17257be43e663874fec6f5b5fc3f56e04ff0198 Mon Sep 17 00:00:00 2001 From: Ben Cassell Date: Thu, 16 Nov 2023 09:36:00 -0800 Subject: [PATCH 11/11] changelof --- CHANGELOG.md | 8 +++++--- dbt/adapters/databricks/__version__.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2904ddfa..76fb4d9c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,10 @@ -## dbt-databricks 1.7.1 (Nov 13, 2023) +## dbt-databricks 1.7.x (TBD) -### Under the Hood +### Features -- Another attempt to improve catalog gathering performance ([503](https://github.com/databricks/dbt-databricks/pull/503)) +- Adding capability to specify compute on a per model basis ([488](https://github.com/databricks/dbt-databricks/pull/488)) + +## dbt-databricks 1.7.1 (Nov 13, 2023) ### Under the Hood diff --git a/dbt/adapters/databricks/__version__.py b/dbt/adapters/databricks/__version__.py index 59bc4f8db..b9be5424c 100644 --- a/dbt/adapters/databricks/__version__.py +++ b/dbt/adapters/databricks/__version__.py @@ -1 +1 @@ -version: str = "1.7.1" +version: str = "1.7.2b1"