Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix refresh_strategy = auto semantics for dynamic tables #1268

Merged
merged 4 commits into from
Dec 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20241209-131530.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: AUTO should no longer lead to rebuilds of dynamic tables.
time: 2024-12-09T13:15:30.554566-08:00
custom:
Author: versusfacit
Issue: "1267"
6 changes: 5 additions & 1 deletion dbt/adapters/snowflake/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from dbt_common.events.functions import fire_event, warn_or_error

from dbt.adapters.snowflake.relation_configs import (
RefreshMode,
SnowflakeCatalogConfigChange,
SnowflakeDynamicTableConfig,
SnowflakeDynamicTableConfigChangeset,
Expand Down Expand Up @@ -109,7 +110,10 @@ def dynamic_table_config_changeset(
)
)

if new_dynamic_table.refresh_mode != existing_dynamic_table.refresh_mode:
if (
new_dynamic_table.refresh_mode != RefreshMode.AUTO
and new_dynamic_table.refresh_mode != existing_dynamic_table.refresh_mode
):
config_change_collection.refresh_mode = SnowflakeDynamicTableRefreshModeConfigChange(
action=RelationConfigChangeAction.create,
context=new_dynamic_table.refresh_mode,
Expand Down
1 change: 1 addition & 0 deletions dbt/adapters/snowflake/relation_configs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
SnowflakeCatalogConfigChange,
)
from dbt.adapters.snowflake.relation_configs.dynamic_table import (
RefreshMode,
SnowflakeDynamicTableConfig,
SnowflakeDynamicTableConfigChangeset,
SnowflakeDynamicTableRefreshModeConfigChange,
Expand Down
20 changes: 20 additions & 0 deletions tests/functional/relation_tests/dynamic_table_tests/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,26 @@
"""


EXPLICIT_AUTO_DYNAMIC_TABLE = """
{{ config(
materialized='dynamic_table',
snowflake_warehouse='DBT_TESTING',
target_lag='2 minutes',
refresh_mode='AUTO',
) }}
select * from {{ ref('my_seed') }}
"""

IMPLICIT_AUTO_DYNAMIC_TABLE = """
{{ config(
materialized='dynamic_table',
snowflake_warehouse='DBT_TESTING',
target_lag='2 minutes',
) }}
select * from {{ ref('my_seed') }}
"""


DYNAMIC_TABLE_DOWNSTREAM = """
{{ config(
materialized='dynamic_table',
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import pytest

from dbt.tests.util import run_dbt
from dbt.tests.util import assert_message_in_logs, run_dbt, run_dbt_and_capture

from tests.functional.relation_tests.dynamic_table_tests import models
from tests.functional.utils import query_relation_type
Expand Down Expand Up @@ -46,3 +46,44 @@ class TestBasicIcebergOn(TestBasic):
@pytest.fixture(scope="class")
def project_config_update(self):
return {"flags": {"enable_iceberg_materializations": True}}


class TestAutoConfigDoesntFullRefresh:
"""
AUTO refresh_strategy will be compared accurately with both INCREMENTAL and FULL.
https://github.com/dbt-labs/dbt-snowflake/issues/1267
"""

DT_NAME = "my_dynamic_table"

@pytest.fixture(scope="class", autouse=True)
def seeds(self):
return {"my_seed.csv": models.SEED}

@pytest.fixture(scope="class", autouse=True)
def models(self):
yield {
f"explicit_{self.DT_NAME}.sql": models.EXPLICIT_AUTO_DYNAMIC_TABLE,
f"implicit_{self.DT_NAME}.sql": models.IMPLICIT_AUTO_DYNAMIC_TABLE,
}

@pytest.mark.parametrize("test_dt", [f"explicit_{DT_NAME}", f"implicit_{DT_NAME}"])
def test_auto_config_doesnt_full_refresh(self, project, test_dt):
model_qualified_name = f"{project.database}.{project.test_schema}.{test_dt}"

run_dbt(["seed"])
_, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"])
assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs)
assert_message_in_logs("refresh_mode = AUTO", logs)

_, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"])

assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs, False)
assert_message_in_logs(
f"create or replace dynamic table {model_qualified_name}", logs, False
)
assert_message_in_logs("refresh_mode = AUTO", logs, False)
assert_message_in_logs(
f"No configuration changes were identified on: `{model_qualified_name}`. Continuing.",
logs,
)
Loading