Skip to content

Commit

Permalink
More warehouse-per-model test updates
Browse files Browse the repository at this point in the history
Updated functional tests to add a model using the default compute alongside models using alternate compute.
Updated the error message for when a named compute does not exist or does not specify http_path.
Clarified the logic in _get_http_path()
Signed-off-by: Raymond Cypher <raymond.cypher@databricks.com>
  • Loading branch information
rcypher-databricks committed Nov 18, 2023
1 parent 015235d commit 1e3b37d
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 21 deletions.
25 changes: 14 additions & 11 deletions dbt/adapters/databricks/connections.py
Original file line number Diff line number Diff line change
Expand Up @@ -1122,26 +1122,29 @@ def _get_compute_name(node: Optional[ResultNode]) -> Optional[str]:

def _get_http_path(node: Optional[ResultNode], creds: DatabricksCredentials) -> Optional[str]:
thread_id = (os.getpid(), get_ident())
# Get the http path of the compute resource specified in the node's config.
# If none is specified return the default path from creds.
compute_name = _get_compute_name(node)
if not node or not compute_name:
if node:
logger.debug(
f"On thread {thread_id}: {node.relation_name} using default compute resource."
)
else:
logger.debug(f"Thread {thread_id}: using default compute resource.")

# If there is no node we return the http_path for the default compute.
if not node:
logger.debug(f"Thread {thread_id}: using default compute resource.")
return creds.http_path

# Get the name of the compute resource specified in the node's config.
# If none is specified return the http_path for the default compute.
compute_name = _get_compute_name(node)
if not compute_name:
logger.debug(f"On thread {thread_id}: {node.relation_name} using default compute resource.")
return creds.http_path

# Get the http_path for the named compute.
http_path = None
if creds.compute:
http_path = creds.compute.get(compute_name, {}).get("http_path", None)

# no http_path for the named compute resource is an error condition
if not http_path:
raise dbt.exceptions.DbtRuntimeError(
f"Compute resource {compute_name} does not exist, relation: {node.relation_name}"
f"Compute resource {compute_name} does not exist or "
f"does not specify http_path, relation: {node.relation_name}"
)

logger.debug(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,9 @@ def test_wpm(self, project):
_, log = util.run_dbt_and_capture(["--debug", "seed"])
assert "`source` using compute resource 'alternate_warehouse2'" in log

_, log = util.run_dbt_and_capture(["--debug", "run", "--select", "target"])
_, log = util.run_dbt_and_capture(["--debug", "run", "--select", "target", "target3"])
assert "`target` using compute resource 'alternate_warehouse'" in log
assert "`target3` using default compute resource" in log

_, log = util.run_dbt_and_capture(["--debug", "snapshot"])
assert "`target_snap` using compute resource 'alternate_warehouse3'" in log
Expand Down
22 changes: 13 additions & 9 deletions tests/unit/test_compute_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
class TestDatabricksConnectionHTTPPath(unittest.TestCase):
"""Test the various cases for determining a specified warehouse."""

errMsg = (
"Compute resource foo does not exist or does not specify http_path, " "relation: a_relation"
)

def test_get_http_path_model(self):
default_path = "my_http_path"
creds = connections.DatabricksCredentials(http_path=default_path)
Expand Down Expand Up @@ -42,21 +46,21 @@ def test_get_http_path_model(self):
node.config._extra["databricks_compute"] = "foo"
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

creds.compute = {}
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

creds.compute = {"foo": {}}
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

Expand Down Expand Up @@ -99,21 +103,21 @@ def test_get_http_path_seed(self):
node.config._extra["databricks_compute"] = "foo"
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

creds.compute = {}
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

creds.compute = {"foo": {}}
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

Expand Down Expand Up @@ -155,21 +159,21 @@ def test_get_http_path_snapshot(self):
node.config._extra["databricks_compute"] = "foo"
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

creds.compute = {}
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

creds.compute = {"foo": {}}
with self.assertRaisesRegex(
dbt.exceptions.DbtRuntimeError,
"Compute resource foo does not exist, relation: a_relation",
self.errMsg,
):
connections._get_http_path(node, creds)

Expand Down

0 comments on commit 1e3b37d

Please sign in to comment.