Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
- Apply `tblproperties` to `metric_view` models at create time, not only on a later alter/replace run ([#1530](https://github.com/databricks/dbt-databricks/pull/1530) closes [#1527](https://github.com/databricks/dbt-databricks/issues/1527))
- Only emit `INSERT ... BY NAME` in the `replace_where`/`microbatch` strategies on DBR 18.0+ (and SQL warehouses), since older clusters reject the `BY NAME ... REPLACE WHERE` combination with a parse error ([1539](https://github.com/databricks/dbt-databricks/pull/1539) closes [#1532](https://github.com/databricks/dbt-databricks/issues/1532))
- Fix materialized views always rebuilding because Databricks-internal `tblproperties` were read as configuration drift; the diff now compares only the configured properties ([#1350](https://github.com/databricks/dbt-databricks/pull/1350) closes [#1314](https://github.com/databricks/dbt-databricks/issues/1314)).
- Stop metric views with `view_update_via_alter` from re-issuing a redundant `ALTER VIEW ... AS` on every run ([#1546](https://github.com/databricks/dbt-databricks/pull/1546))

### Under the Hood

Expand Down
21 changes: 16 additions & 5 deletions dbt/adapters/databricks/relation_configs/metric_view.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from typing import ClassVar, Optional

import yaml
from dbt.adapters.contracts.relation import RelationConfig
from dbt.adapters.relation_configs.config_base import RelationResults
from dbt_common.exceptions import DbtRuntimeError
Expand All @@ -19,12 +20,22 @@ class MetricViewQueryConfig(DatabricksComponentConfig):
query: str

def get_diff(self, other: "MetricViewQueryConfig") -> Optional["MetricViewQueryConfig"]:
# Normalize whitespace for comparison
self_normalized = " ".join(self.query.split())
other_normalized = " ".join(other.query.split())
if self_normalized != other_normalized:
# Databricks re-renders the stored definition (requoting `source`, rewriting
# flow-style lists to block style), so compare the parsed YAML structure, not the
# text -- otherwise an unchanged model reports a phantom change every run.
# BaseLoader keeps scalars as strings so genuine value changes still register.
# Fall back to a whitespace-normalized text comparison if either definition is
# not valid YAML.
try:
if yaml.load(self.query, Loader=yaml.BaseLoader) == yaml.load(
other.query, Loader=yaml.BaseLoader
):
return None
return self
return None
except yaml.YAMLError:
self_normalized = " ".join(self.query.split())
other_normalized = " ".join(other.query.split())
return self if self_normalized != other_normalized else None


class MetricViewQueryProcessor(DatabricksComponentProcessor[MetricViewQueryConfig]):
Expand Down
18 changes: 18 additions & 0 deletions tests/functional/adapter/metric_views/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,24 @@
expr: count(1)
"""

# version 1.1 is required for `synonyms`; the flow-style list and double-quoted
# `source` are both re-rendered by Databricks on read-back (block list, single quotes).
metric_view_with_synonyms = """
{{ config(materialized='metric_view') }}

version: 1.1
source: "{{ ref('source_orders') }}"
dimensions:
- name: status
expr: status
synonyms: [state, order_state]
measures:
- name: total_orders
expr: count(1)
- name: total_revenue
expr: sum(revenue)
"""

metric_view_with_tblproperties = """
{{
config(
Expand Down
54 changes: 54 additions & 0 deletions tests/functional/adapter/metric_views/test_metric_view_noop.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
"""An unchanged metric view with `view_update_via_alter` must be a no-op on re-run.

Databricks re-renders the stored metric-view definition (requoting `source`
double->single, rewriting flow-style lists to block style), so a text comparison of
the model YAML against the stored View Text always registered a change and dbt
re-issued `ALTER VIEW ... AS` on every run. The definition is compared structurally,
so an unchanged re-run leaves the view untouched.
"""

import pytest
from dbt.tests.util import run_dbt

from tests.functional.adapter.metric_views.fixtures import (
metric_view_with_synonyms,
source_table,
)


def _last_altered(project, name):
return project.run_sql(
f"SELECT last_altered FROM {project.database}.information_schema.tables "
f"WHERE table_schema = '{project.test_schema}' AND table_name = '{name}'",
fetch="all",
)[0][0]


@pytest.mark.skip_profile("databricks_cluster")
class TestMetricViewNoopOnRerun:
@pytest.fixture(scope="class")
def project_config_update(self):
return {
"flags": {"use_materialization_v2": True},
"models": {"+view_update_via_alter": True},
}

@pytest.fixture(scope="class")
def models(self):
return {
"source_orders.sql": source_table,
"noop_metrics.sql": metric_view_with_synonyms,
}

def test_unchanged_metric_view_is_noop(self, project):
run_dbt(["run"])
before = _last_altered(project, "noop_metrics")

results = run_dbt(["run", "--models", "noop_metrics"])
assert len(results) == 1
assert results[0].status == "success"

after = _last_altered(project, "noop_metrics")
assert after == before, (
f"metric view was re-altered on an unchanged re-run: {before} -> {after}"
)
42 changes: 42 additions & 0 deletions tests/unit/relation_configs/test_metric_view_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,48 @@ def test_get_diff__different_whitespace_content(self):
config2 = MetricViewQueryConfig(query="version: 0.1 source: other_table")
assert config1.get_diff(config2) is config1

def test_get_diff__source_quote_normalization(self):
# Databricks stores `source` single-quoted; the model emits it double-quoted.
# The two are semantically identical, so no change should be detected.
desired = MetricViewQueryConfig(query='version: 1.1\nsource: "`c`.`s`.`t`"')
existing = MetricViewQueryConfig(query="version: 1.1\nsource: '`c`.`s`.`t`'")
assert desired.get_diff(existing) is None

def test_get_diff__flow_vs_block_list(self):
# Databricks rewrites flow-style lists (e.g. synonyms) to block style on read-back.
desired = MetricViewQueryConfig(
query="version: 1.1\ndimensions:\n - name: s\n expr: s\n synonyms: [a, b]"
)
existing = MetricViewQueryConfig(
query=(
"version: 1.1\ndimensions:\n - name: s\n expr: s\n"
" synonyms:\n - a\n - b"
)
)
assert desired.get_diff(existing) is None

def test_get_diff__semantic_difference_detected(self):
desired = MetricViewQueryConfig(query="version: 1.1\nsource: t\nfilter: a = 1")
existing = MetricViewQueryConfig(query="version: 1.1\nsource: t\nfilter: a = 2")
assert desired.get_diff(existing) is desired

def test_get_diff__unparseable_falls_back_to_whitespace(self):
# Malformed YAML still compares via whitespace normalization (no crash).
desired = MetricViewQueryConfig(query="a: [1, 2")
existing = MetricViewQueryConfig(query="a: [1, 2")
assert desired.get_diff(existing) is None

def test_get_diff__reserved_word_scalars_not_coerced(self):
# `yes`/`no` and `true`/`false` are distinct synonym values; YAML implicit typing
# would collapse them to the same booleans, so a real change must still be detected.
desired = MetricViewQueryConfig(
query="version: 1.1\ndimensions:\n - name: s\n expr: s\n synonyms: [yes, no]"
)
existing = MetricViewQueryConfig(
query="version: 1.1\ndimensions:\n - name: s\n expr: s\n synonyms: [true, false]"
)
assert desired.get_diff(existing) is desired


class TestMetricViewQueryProcessor:
def test_from_relation_results__with_dollar_delimiters(self):
Expand Down
Loading