From 0b24a5a2d311ffec5f996ca28532076a637aa6b3 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:05:38 -0800 Subject: [PATCH 1/5] update libpq-dev dependency to 13.18-0+deb11u1 (#1262) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 17315b12d..16060db61 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update \ build-essential=12.9 \ ca-certificates=20210119 \ git=1:2.30.2-1+deb11u2 \ - libpq-dev=13.14-0+deb11u1 \ + libpq-dev=13.18-0+deb11u1 \ make=4.3-4.1 \ openssh-client=1:8.4p1-5+deb11u3 \ software-properties-common=0.96.20.2-2.1 \ From 86cf6e61b904a501d562d56d9fa1caea1c3efd7d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 5 Dec 2024 15:37:57 -0500 Subject: [PATCH 2/5] support MicrobatchConcurrency capability (#1259) --- .changes/unreleased/Features-20241202-095136.yaml | 6 ++++++ dbt/adapters/snowflake/impl.py | 1 + setup.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20241202-095136.yaml diff --git a/.changes/unreleased/Features-20241202-095136.yaml b/.changes/unreleased/Features-20241202-095136.yaml new file mode 100644 index 000000000..973866a6d --- /dev/null +++ b/.changes/unreleased/Features-20241202-095136.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support MicrobatchConcurrency +time: 2024-12-02T09:51:36.606097-05:00 +custom: + Author: michelleark + Issue: "1260" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index ac0d903db..10ad2a8a1 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -77,6 +77,7 @@ class SnowflakeAdapter(SQLAdapter): Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), Capability.GetCatalogForSingleRelation: CapabilitySupport(support=Support.Full), + Capability.MicrobatchConcurrency: CapabilitySupport(support=Support.Full), } ) diff --git a/setup.py b/setup.py index c0716341d..b237182ce 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ def _plugin_version() -> str: include_package_data=True, install_requires=[ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters>=1.10.4,<2.0", "snowflake-connector-python[secure-local-storage]~=3.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", From 457c361a8453d7e4c09ce9033f872a651646fd6c Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 5 Dec 2024 16:42:45 -0500 Subject: [PATCH 3/5] [Refactor] use new model.batch context vars (#1258) --- .changes/unreleased/Under the Hood-20241205-144036.yaml | 6 ++++++ dbt/include/snowflake/macros/materializations/merge.sql | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20241205-144036.yaml diff --git a/.changes/unreleased/Under the Hood-20241205-144036.yaml b/.changes/unreleased/Under the Hood-20241205-144036.yaml new file mode 100644 index 000000000..aedcb4ce1 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241205-144036.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Use new `batch` context variables over `node.config.__dbt_internal` ones +time: 2024-12-05T14:40:36.373637-05:00 +custom: + Author: michelleark + Issue: "1263" diff --git a/dbt/include/snowflake/macros/materializations/merge.sql b/dbt/include/snowflake/macros/materializations/merge.sql index 716a325c1..7515f6010 100644 --- a/dbt/include/snowflake/macros/materializations/merge.sql +++ b/dbt/include/snowflake/macros/materializations/merge.sql @@ -57,10 +57,10 @@ {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%} {#-- Add additional incremental_predicates to filter for batch --#} - {% if model.config.get("__dbt_internal_microbatch_event_time_start") -%} + {% if model.batch and model.batch.event_time_start -%} {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "')") %} {% endif %} - {% if model.config.__dbt_internal_microbatch_event_time_end -%} + {% if model.batch and model.batch.event_time_end -%} {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < to_timestamp_tz('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "')") %} {% endif %} {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} From b3eeb083ce379ee84468cd485e58ea10b353350a Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 9 Dec 2024 21:44:30 +0000 Subject: [PATCH 4/5] Fix refresh_strategy = auto semantics for dynamic tables (#1268) * Add failing test. * Parametrize cases * Change comparison logic for when to rebuild if DT has auto refresh strategy (implicit or explicit). * Add changelog --------- Co-authored-by: VersusFacit --- .../unreleased/Fixes-20241209-131530.yaml | 6 +++ dbt/adapters/snowflake/relation.py | 6 ++- .../snowflake/relation_configs/__init__.py | 1 + .../dynamic_table_tests/models.py | 20 +++++++++ .../dynamic_table_tests/test_basic.py | 43 ++++++++++++++++++- 5 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241209-131530.yaml diff --git a/.changes/unreleased/Fixes-20241209-131530.yaml b/.changes/unreleased/Fixes-20241209-131530.yaml new file mode 100644 index 000000000..b3196a3b0 --- /dev/null +++ b/.changes/unreleased/Fixes-20241209-131530.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: AUTO should no longer lead to rebuilds of dynamic tables. +time: 2024-12-09T13:15:30.554566-08:00 +custom: + Author: versusfacit + Issue: "1267" diff --git a/dbt/adapters/snowflake/relation.py b/dbt/adapters/snowflake/relation.py index b6924b9b3..54db21924 100644 --- a/dbt/adapters/snowflake/relation.py +++ b/dbt/adapters/snowflake/relation.py @@ -17,6 +17,7 @@ from dbt_common.events.functions import fire_event, warn_or_error from dbt.adapters.snowflake.relation_configs import ( + RefreshMode, SnowflakeCatalogConfigChange, SnowflakeDynamicTableConfig, SnowflakeDynamicTableConfigChangeset, @@ -109,7 +110,10 @@ def dynamic_table_config_changeset( ) ) - if new_dynamic_table.refresh_mode != existing_dynamic_table.refresh_mode: + if ( + new_dynamic_table.refresh_mode != RefreshMode.AUTO + and new_dynamic_table.refresh_mode != existing_dynamic_table.refresh_mode + ): config_change_collection.refresh_mode = SnowflakeDynamicTableRefreshModeConfigChange( action=RelationConfigChangeAction.create, context=new_dynamic_table.refresh_mode, diff --git a/dbt/adapters/snowflake/relation_configs/__init__.py b/dbt/adapters/snowflake/relation_configs/__init__.py index fec9d8a54..67f7644d2 100644 --- a/dbt/adapters/snowflake/relation_configs/__init__.py +++ b/dbt/adapters/snowflake/relation_configs/__init__.py @@ -3,6 +3,7 @@ SnowflakeCatalogConfigChange, ) from dbt.adapters.snowflake.relation_configs.dynamic_table import ( + RefreshMode, SnowflakeDynamicTableConfig, SnowflakeDynamicTableConfigChangeset, SnowflakeDynamicTableRefreshModeConfigChange, diff --git a/tests/functional/relation_tests/dynamic_table_tests/models.py b/tests/functional/relation_tests/dynamic_table_tests/models.py index 4dcd6cf48..57d83f968 100644 --- a/tests/functional/relation_tests/dynamic_table_tests/models.py +++ b/tests/functional/relation_tests/dynamic_table_tests/models.py @@ -17,6 +17,26 @@ """ +EXPLICIT_AUTO_DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', + refresh_mode='AUTO', +) }} +select * from {{ ref('my_seed') }} +""" + +IMPLICIT_AUTO_DYNAMIC_TABLE = """ +{{ config( + materialized='dynamic_table', + snowflake_warehouse='DBT_TESTING', + target_lag='2 minutes', +) }} +select * from {{ ref('my_seed') }} +""" + + DYNAMIC_TABLE_DOWNSTREAM = """ {{ config( materialized='dynamic_table', diff --git a/tests/functional/relation_tests/dynamic_table_tests/test_basic.py b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py index 79a2241ca..8cdf59ebc 100644 --- a/tests/functional/relation_tests/dynamic_table_tests/test_basic.py +++ b/tests/functional/relation_tests/dynamic_table_tests/test_basic.py @@ -1,6 +1,6 @@ import pytest -from dbt.tests.util import run_dbt +from dbt.tests.util import assert_message_in_logs, run_dbt, run_dbt_and_capture from tests.functional.relation_tests.dynamic_table_tests import models from tests.functional.utils import query_relation_type @@ -46,3 +46,44 @@ class TestBasicIcebergOn(TestBasic): @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"enable_iceberg_materializations": True}} + + +class TestAutoConfigDoesntFullRefresh: + """ + AUTO refresh_strategy will be compared accurately with both INCREMENTAL and FULL. + https://github.com/dbt-labs/dbt-snowflake/issues/1267 + """ + + DT_NAME = "my_dynamic_table" + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": models.SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + f"explicit_{self.DT_NAME}.sql": models.EXPLICIT_AUTO_DYNAMIC_TABLE, + f"implicit_{self.DT_NAME}.sql": models.IMPLICIT_AUTO_DYNAMIC_TABLE, + } + + @pytest.mark.parametrize("test_dt", [f"explicit_{DT_NAME}", f"implicit_{DT_NAME}"]) + def test_auto_config_doesnt_full_refresh(self, project, test_dt): + model_qualified_name = f"{project.database}.{project.test_schema}.{test_dt}" + + run_dbt(["seed"]) + _, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"]) + assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs) + assert_message_in_logs("refresh_mode = AUTO", logs) + + _, logs = run_dbt_and_capture(["--debug", "run", "--select", f"{test_dt}.sql"]) + + assert_message_in_logs(f"create dynamic table {model_qualified_name}", logs, False) + assert_message_in_logs( + f"create or replace dynamic table {model_qualified_name}", logs, False + ) + assert_message_in_logs("refresh_mode = AUTO", logs, False) + assert_message_in_logs( + f"No configuration changes were identified on: `{model_qualified_name}`. Continuing.", + logs, + ) From efc68e4c32b58536f174e18ec28550bd9f88d629 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:17:05 -0500 Subject: [PATCH 5/5] Update default python version for python models (#1273) * update the default python version for python models; add a configuration file to collect future settings --- .changes/unreleased/Under the Hood-20241211-170831.yaml | 6 ++++++ dbt/adapters/snowflake/constants.py | 1 + dbt/adapters/snowflake/impl.py | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20241211-170831.yaml create mode 100644 dbt/adapters/snowflake/constants.py diff --git a/.changes/unreleased/Under the Hood-20241211-170831.yaml b/.changes/unreleased/Under the Hood-20241211-170831.yaml new file mode 100644 index 000000000..17bf42c1f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241211-170831.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Update default Python version for python models +time: 2024-12-11T17:08:31.842063-05:00 +custom: + Author: mikealfare + Issue: "1203" diff --git a/dbt/adapters/snowflake/constants.py b/dbt/adapters/snowflake/constants.py new file mode 100644 index 000000000..9c475dcd9 --- /dev/null +++ b/dbt/adapters/snowflake/constants.py @@ -0,0 +1 @@ +DEFAULT_PYTHON_VERSION_FOR_PYTHON_MODELS = "3.9" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 10ad2a8a1..7ccff9f8a 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -22,6 +22,7 @@ from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError from dbt_common.utils import filter_null_values +from dbt.adapters.snowflake import constants from dbt.adapters.snowflake.relation_configs import ( SnowflakeRelationType, TableFormat, @@ -345,7 +346,9 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str): schema = parsed_model["schema"] database = parsed_model["database"] identifier = parsed_model["alias"] - python_version = parsed_model["config"].get("python_version", "3.8") + python_version = parsed_model["config"].get( + "python_version", constants.DEFAULT_PYTHON_VERSION_FOR_PYTHON_MODELS + ) packages = parsed_model["config"].get("packages", []) imports = parsed_model["config"].get("imports", [])