From 5c9086a957a478b63e6f3fc6cd3e5d74b6e2900d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Fri, 6 Sep 2024 10:05:21 -0400 Subject: [PATCH 1/6] for testing: dbt-postgres 'microbatch' strategy --- dbt/adapters/postgres/impl.py | 2 +- .../materializations/incremental_strategies.sql | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/postgres/impl.py b/dbt/adapters/postgres/impl.py index d49d334b..b8d4f43d 100644 --- a/dbt/adapters/postgres/impl.py +++ b/dbt/adapters/postgres/impl.py @@ -151,7 +151,7 @@ def valid_incremental_strategies(self): """The set of standard builtin strategies which this adapter supports out-of-the-box. Not used to validate custom strategies defined by end users. """ - return ["append", "delete+insert", "merge"] + return ["append", "delete+insert", "merge", "microbatch"] def debug_query(self): self.execute("select 1 as id") diff --git a/dbt/include/postgres/macros/materializations/incremental_strategies.sql b/dbt/include/postgres/macros/materializations/incremental_strategies.sql index f2fbf41e..f41ed86d 100644 --- a/dbt/include/postgres/macros/materializations/incremental_strategies.sql +++ b/dbt/include/postgres/macros/materializations/incremental_strategies.sql @@ -7,3 +7,14 @@ {% endif %} {% endmacro %} + + +{% macro get_incremental_microbatch_sql(arg_dict) %} + + {% if arg_dict["unique_key"] %} + {% do return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) %} + {% else %} + {{ exceptions.raise_compiler_error("dbt-postgres 'microbatch' requires a `unique_key` config") }} + {% endif %} + +{% endmacro %} From 7876219be6ea31ca78a90459bb5d6e3e70be4e4d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 11 Sep 2024 13:48:01 -0400 Subject: [PATCH 2/6] add microbatch tests --- .../macros/materializations/incremental_strategies.sql | 2 +- pyproject.toml | 4 ++-- tests/functional/adapter/test_incremental_microbatch.py | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tests/functional/adapter/test_incremental_microbatch.py diff --git a/dbt/include/postgres/macros/materializations/incremental_strategies.sql b/dbt/include/postgres/macros/materializations/incremental_strategies.sql index f41ed86d..1d37366f 100644 --- a/dbt/include/postgres/macros/materializations/incremental_strategies.sql +++ b/dbt/include/postgres/macros/materializations/incremental_strategies.sql @@ -9,7 +9,7 @@ {% endmacro %} -{% macro get_incremental_microbatch_sql(arg_dict) %} +{% macro postgres__get_incremental_microbatch_sql(arg_dict) %} {% if arg_dict["unique_key"] %} {% do return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) %} diff --git a/pyproject.toml b/pyproject.toml index e6848a4f..0bf1fa15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,8 +56,8 @@ path = "dbt/adapters/postgres/__version__.py" dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@base-microbatch-tests#subdirectory=dbt-tests-adapter", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@event-time-ref-filtering#subdirectory=core", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', "freezegun", diff --git a/tests/functional/adapter/test_incremental_microbatch.py b/tests/functional/adapter/test_incremental_microbatch.py new file mode 100644 index 00000000..9d7083e0 --- /dev/null +++ b/tests/functional/adapter/test_incremental_microbatch.py @@ -0,0 +1,6 @@ +from dbt.tests.adapter.incremental.test_incremental_microbatch import ( + BaseMicrobatch, +) + +class TestPostgresMicrobatch(BaseMicrobatch): + pass From c64c13a5c22bf20c3fa19528901c383b2e4761b7 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 11 Sep 2024 14:14:23 -0400 Subject: [PATCH 3/6] changelog entry, linting --- .changes/unreleased/Features-20240911-141416.yaml | 6 ++++++ tests/functional/adapter/test_incremental_microbatch.py | 1 + 2 files changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20240911-141416.yaml diff --git a/.changes/unreleased/Features-20240911-141416.yaml b/.changes/unreleased/Features-20240911-141416.yaml new file mode 100644 index 00000000..990a09d9 --- /dev/null +++ b/.changes/unreleased/Features-20240911-141416.yaml @@ -0,0 +1,6 @@ +kind: Features +body: 'Microbatch incremental strategy implementation: merge' +time: 2024-09-11T14:14:16.538536-04:00 +custom: + Author: michelleark + Issue: "149" diff --git a/tests/functional/adapter/test_incremental_microbatch.py b/tests/functional/adapter/test_incremental_microbatch.py index 9d7083e0..ce5855b6 100644 --- a/tests/functional/adapter/test_incremental_microbatch.py +++ b/tests/functional/adapter/test_incremental_microbatch.py @@ -2,5 +2,6 @@ BaseMicrobatch, ) + class TestPostgresMicrobatch(BaseMicrobatch): pass From 06ede4b5c21d4fcd777bad31a486750670694062 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 11 Sep 2024 14:30:47 -0400 Subject: [PATCH 4/6] test against dbt-adapters@@base-microbatch-test --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0bf1fa15..594a67cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@base-microbatch-tests", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@base-microbatch-tests#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@event-time-ref-filtering#subdirectory=core", From fba2fccfe70e5b7eb3da1a39512fbedcba3e1d9d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 11 Sep 2024 22:15:24 -0400 Subject: [PATCH 5/6] restore pyproject.toml dependencies for dbt-adapters --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 594a67cd..62e9d7f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,9 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@base-microbatch-tests", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@base-microbatch-tests#subdirectory=dbt-tests-adapter", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@event-time-ref-filtering#subdirectory=core", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', From 35fb48f93911f41d2bf375577f4881d72023efba Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 12 Sep 2024 11:43:38 -0400 Subject: [PATCH 6/6] restore pyproject.toml dependency on dbt-core --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 62e9d7f4..e6848a4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@event-time-ref-filtering#subdirectory=core", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', "freezegun",