diff --git a/.changes/1.10.2.md b/.changes/1.10.2.md new file mode 100644 index 00000000..09244521 --- /dev/null +++ b/.changes/1.10.2.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.10.2 - October 01, 2024 + +### Under the Hood + +- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315)) diff --git a/.changes/unreleased/Features-20240927-134248.yaml b/.changes/unreleased/Features-20240927-134248.yaml new file mode 100644 index 00000000..0b456244 --- /dev/null +++ b/.changes/unreleased/Features-20240927-134248.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Enable setting current value of dbt_valid_to +time: 2024-09-27T13:42:48.654556-04:00 +custom: + Author: gshank + Issue: "320" diff --git a/.changes/unreleased/Under the Hood-20240923-184719.yaml b/.changes/unreleased/Under the Hood-20240923-184719.yaml deleted file mode 100644 index 35e66b90..00000000 --- a/.changes/unreleased/Under the Hood-20240923-184719.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Under the Hood -body: 'dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch - test' -time: 2024-09-23T18:47:19.171618+01:00 -custom: - Author: michelleark - Issue: "315" diff --git a/CHANGELOG.md b/CHANGELOG.md index 1287c432..dfb37ad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.10.2 - October 01, 2024 + +### Under the Hood + +- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315)) + + + ## dbt-adapters 1.10.1 - September 16, 2024 ## dbt-adapters 1.10.0 - September 12, 2024 @@ -15,8 +23,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie). - Allow configuring of snapshot column names ([#289](https://github.com/dbt-labs/dbt-adapters/issues/289)) - - ## dbt-adapters 1.6.1 - September 16, 2024 ## dbt-adapters 1.6.0 - September 12, 2024 diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 8c5bf7a0..8c657eec 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.10.1" +version = "1.10.2" diff --git a/dbt-tests-adapter/dbt/tests/adapter/hooks/test_run_hooks.py b/dbt-tests-adapter/dbt/tests/adapter/hooks/test_run_hooks.py index 89565c70..0136e3f6 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/hooks/test_run_hooks.py +++ b/dbt-tests-adapter/dbt/tests/adapter/hooks/test_run_hooks.py @@ -3,6 +3,7 @@ import pytest +from dbt_common.exceptions import DbtDatabaseError from dbt.tests.adapter.hooks import fixtures from dbt.tests.util import check_table_does_not_exist, run_dbt @@ -11,8 +12,8 @@ class BasePrePostRunHooks: @pytest.fixture(scope="function") def setUp(self, project): project.run_sql_file(project.test_data_dir / Path("seed_run.sql")) - project.run_sql(f"drop table if exists { project.test_schema }.schemas") - project.run_sql(f"drop table if exists { project.test_schema }.db_schemas") + project.run_sql(f"drop table if exists {project.test_schema}.schemas") + project.run_sql(f"drop table if exists {project.test_schema}.db_schemas") os.environ["TERM_TEST"] = "TESTING" @pytest.fixture(scope="class") @@ -158,7 +159,8 @@ def project_config_update(self): } def test_missing_column_pre_hook(self, project): - run_dbt(["run"], expect_pass=False) + with pytest.raises(DbtDatabaseError): + run_dbt(["run"], expect_pass=False) class TestAfterRunHooks(BaseAfterRunHooks): diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 8d982855..52fdb9bf 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -53,7 +53,13 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ columns.dbt_valid_to }} is null + where + {% if config.get('dbt_valid_to_current') %} + {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #} + ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} ), @@ -64,7 +70,7 @@ {{ strategy.unique_key }} as dbt_unique_key, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}, + {{ get_dbt_valid_to_current(strategy, columns) }}, {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} from snapshot_query @@ -166,7 +172,7 @@ {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }} + {{ get_dbt_valid_to_current(strategy, columns) }} from ( {{ sql }} ) sbq @@ -210,3 +216,9 @@ {% endif %} {% endif %} {% endmacro %} + +{% macro get_dbt_valid_to_current(strategy, columns) %} + {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} + coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) + as {{ columns.dbt_valid_to }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 74494ed2..cf787e4f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -14,7 +14,12 @@ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% if config.get("dbt_valid_to_current") %} + and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}