From 3bb686e3dd8961048e4918f31b185ef3e92bad81 Mon Sep 17 00:00:00 2001 From: devmessias Date: Mon, 21 Oct 2024 09:19:42 -0300 Subject: [PATCH 1/7] fix: unit tests with versioned refs missing latest prop --- core/dbt/parser/unit_tests.py | 16 ++- .../unit_testing/test_unit_testing.py | 133 ++++++++++++++++-- 2 files changed, 134 insertions(+), 15 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index c227319f7e9..7dbef68aad5 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -110,7 +110,6 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): # unit_test_node now has a populated refs/sources self.unit_test_manifest.nodes[unit_test_node.unique_id] = unit_test_node - # Now create input_nodes for the test inputs """ given: @@ -132,7 +131,6 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): given.input, tested_node, test_case.name ) input_name = original_input_node.name - common_fields = { "resource_type": NodeType.Model, # root directory for input and output fixtures @@ -149,21 +147,25 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): "name": input_name, "path": f"{input_name}.sql", } + resource_type = original_input_node.resource_type - if original_input_node.resource_type in ( + if resource_type in ( NodeType.Model, NodeType.Seed, NodeType.Snapshot, ): + input_node = ModelNode( **common_fields, defer_relation=original_input_node.defer_relation, ) - if ( - original_input_node.resource_type == NodeType.Model - and original_input_node.version - ): + + if not resource_type == NodeType.Model: + continue + if original_input_node.version: input_node.version = original_input_node.version + if original_input_node.latest_version: + input_node.latest_version = original_input_node.latest_version elif original_input_node.resource_type == NodeType.Source: # We are reusing the database/schema/identifier from the original source, diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 160f528787d..514e712a725 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -291,22 +291,139 @@ def test_basic(self, project): assert len(results) == 2 -class TestUnitTestIncrementalModelWithVersion: +schema_ref_with_version = """ +models: + - name: source + latest_version: {latest_version} + versions: + - v: 1 + - v: 2 + - name: model_to_test +unit_tests: + - name: ref_versioned + model: 'model_to_test' + given: + - input: {input} + rows: + - {{result: 3}} + expect: + rows: + - {{result: 3}} + +""" + + +class TestUnitTestRefWithVersion: @pytest.fixture(scope="class") def models(self): return { - "my_incremental_model.sql": my_incremental_model_sql, - "events.sql": event_sql, - "schema.yml": my_incremental_model_versioned_yml + test_my_model_incremental_yml_basic, + "model_to_test.sql": "select result from {{ ref('source')}}", + "source.sql": "select 2 as result", + "source_v2.sql": "select 2 as result", + "schema.yml": schema_ref_with_version.format( + **{"latest_version": 1, "input": "ref('source')"} + ), } def test_basic(self, project): results = run_dbt(["run"]) - assert len(results) == 2 - # Select by model name - results = run_dbt(["test", "--select", "my_incremental_model"], expect_pass=True) - assert len(results) == 2 + results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) + assert len(results) == 1 + + +class TestUnitTestRefMissingVersionModel: + @pytest.fixture(scope="class") + def models(self): + return { + "model_to_test.sql": "select result from {{ ref('source')}}", + "source_v1.sql": "select 2 as result", + "source_v2.sql": "select 2 as result", + "schema.yml": schema_ref_with_version.format( + **{"latest_version": 1, "input": "ref('source', v=1)"} + ), + } + + def test_basic(self, project): + results = run_dbt(["run"]) + + results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) + assert len(results) == 1 + + +class TestUnitTestRefWithMissingVersionRef: + @pytest.fixture(scope="class") + def models(self): + return { + "model_to_test.sql": "select result from {{ ref('source', v=1)}}", + "source_v1.sql": "select 2 as result", + "source_v2.sql": "select 2 as result", + "schema.yml": schema_ref_with_version.format( + **{"latest_version": 1, "input": "ref('source')"} + ), + } + + def test_basic(self, project): + results = run_dbt(["run"]) + + results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) + assert len(results) == 1 + + +class TestUnitTestRefWithVersionLatestSecond: + @pytest.fixture(scope="class") + def models(self): + return { + "model_to_test.sql": "select result from {{ ref('source')}}", + "source_v1.sql": "select 2 as result", + "source_v2.sql": "select 2 as result", + "schema.yml": schema_ref_with_version.format( + **{"latest_version": 2, "input": "ref('source')"} + ), + } + + def test_basic(self, project): + results = run_dbt(["run"]) + + results = run_dbt(["test", "--select", "model_to_test"], expect_pass=True) + assert len(results) == 1 + + +class TestUnitTestRefWithVersionMissingRefTest: + @pytest.fixture(scope="class") + def models(self): + return { + "model_to_test.sql": "select result from {{ ref('source', v=2)}}", + "source_v1.sql": "select 2 as result", + "source_v2.sql": "select 2 as result", + "schema.yml": schema_ref_with_version.format( + **{"latest_version": 1, "input": "ref('source')"} + ), + } + + def test_basic(self, project): + results = run_dbt(["run"]) + assert len(results) == 3 + # TODO: How to capture an compilation Error? pytest.raises(CompilationError) not working + run_dbt(["test", "--select", "model_to_test"], expect_pass=False) + + +class TestUnitTestRefWithVersionDiffLatest: + @pytest.fixture(scope="class") + def models(self): + return { + "model_to_test.sql": "select result from {{ ref('source', v=2)}}", + "source_v1.sql": "select 2 as result", + "source_v2.sql": "select 2 as result", + "schema.yml": schema_ref_with_version.format( + **{"latest_version": 1, "input": "ref('source', v=2)"} + ), + } + + def test_basic(self, project): + results = run_dbt(["run"]) + assert len(results) == 3 + run_dbt(["test", "--select", "model_to_test"], expect_pass=True) class TestUnitTestExplicitSeed: From f37a2d281462f423bfd67cf514c102176c9e435e Mon Sep 17 00:00:00 2001 From: devmessias Date: Mon, 21 Oct 2024 09:31:13 -0300 Subject: [PATCH 2/7] chore: changie yaml --- .changes/unreleased/Fixes-20241021-093047.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20241021-093047.yaml diff --git a/.changes/unreleased/Fixes-20241021-093047.yaml b/.changes/unreleased/Fixes-20241021-093047.yaml new file mode 100644 index 00000000000..4d691aa8837 --- /dev/null +++ b/.changes/unreleased/Fixes-20241021-093047.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: unit tests with versioned refs +time: 2024-10-21T09:30:47.949023898-03:00 +custom: + Author: devmessias + Issue: 10880 10528 10623 From 32a794aa7349036b458f31d7a010ad6cd9430495 Mon Sep 17 00:00:00 2001 From: devmessias Date: Mon, 21 Oct 2024 09:46:07 -0300 Subject: [PATCH 3/7] test: incremental unit test deleted before due a mistake --- .../unit_testing/test_unit_testing.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 514e712a725..571affe16de 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -291,6 +291,24 @@ def test_basic(self, project): assert len(results) == 2 +class TestUnitTestIncrementalModelWithVersion: + @pytest.fixture(scope="class") + def models(self): + return { + "my_incremental_model.sql": my_incremental_model_sql, + "events.sql": event_sql, + "schema.yml": my_incremental_model_versioned_yml + test_my_model_incremental_yml_basic, + } + + def test_basic(self, project): + results = run_dbt(["run"]) + assert len(results) == 2 + + # Select by model name + results = run_dbt(["test", "--select", "my_incremental_model"], expect_pass=True) + assert len(results) == 2 + + schema_ref_with_version = """ models: - name: source From ba528e909250d0291a16b58b153bfd27dcf9b98a Mon Sep 17 00:00:00 2001 From: devmessias Date: Mon, 21 Oct 2024 09:49:38 -0300 Subject: [PATCH 4/7] tidy: var name --- core/dbt/parser/unit_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 7dbef68aad5..fc6a7ef406d 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -167,7 +167,7 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): if original_input_node.latest_version: input_node.latest_version = original_input_node.latest_version - elif original_input_node.resource_type == NodeType.Source: + elif resource_type == NodeType.Source: # We are reusing the database/schema/identifier from the original source, # but that shouldn't matter since this acts as an ephemeral model which just # wraps a CTE around the unit test node. From c98293b6bca5d968bbf1b82de9da12d68c87a90c Mon Sep 17 00:00:00 2001 From: devmessias Date: Mon, 11 Nov 2024 17:50:12 -0300 Subject: [PATCH 5/7] test(unit-test): capture compilation error --- .../unit_testing/test_unit_testing.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index 571affe16de..cd9083686da 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -309,7 +309,7 @@ def test_basic(self, project): assert len(results) == 2 -schema_ref_with_version = """ +schema_ref_with_versioned_model = """ models: - name: source latest_version: {latest_version} @@ -338,7 +338,7 @@ def models(self): "model_to_test.sql": "select result from {{ ref('source')}}", "source.sql": "select 2 as result", "source_v2.sql": "select 2 as result", - "schema.yml": schema_ref_with_version.format( + "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source')"} ), } @@ -357,7 +357,7 @@ def models(self): "model_to_test.sql": "select result from {{ ref('source')}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", - "schema.yml": schema_ref_with_version.format( + "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source', v=1)"} ), } @@ -376,7 +376,7 @@ def models(self): "model_to_test.sql": "select result from {{ ref('source', v=1)}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", - "schema.yml": schema_ref_with_version.format( + "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source')"} ), } @@ -395,7 +395,7 @@ def models(self): "model_to_test.sql": "select result from {{ ref('source')}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", - "schema.yml": schema_ref_with_version.format( + "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 2, "input": "ref('source')"} ), } @@ -414,16 +414,21 @@ def models(self): "model_to_test.sql": "select result from {{ ref('source', v=2)}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", - "schema.yml": schema_ref_with_version.format( + "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source')"} ), } def test_basic(self, project): results = run_dbt(["run"]) + assert len(results) == 3 - # TODO: How to capture an compilation Error? pytest.raises(CompilationError) not working - run_dbt(["test", "--select", "model_to_test"], expect_pass=False) + # run_dbt(["test", "--select", "model_to_test"], expect_pass=False) + exec_result, _ = run_dbt_and_capture( + ["test", "--select", "model_to_test"], expect_pass=False + ) + msg_error = exec_result[0].message + assert msg_error.lower().lstrip().startswith("compilation error") class TestUnitTestRefWithVersionDiffLatest: @@ -433,7 +438,7 @@ def models(self): "model_to_test.sql": "select result from {{ ref('source', v=2)}}", "source_v1.sql": "select 2 as result", "source_v2.sql": "select 2 as result", - "schema.yml": schema_ref_with_version.format( + "schema.yml": schema_ref_with_versioned_model.format( **{"latest_version": 1, "input": "ref('source', v=2)"} ), } From f155805de7efbc91e37ff504062ec7a9123e846d Mon Sep 17 00:00:00 2001 From: devmessias Date: Mon, 11 Nov 2024 17:52:21 -0300 Subject: [PATCH 6/7] style(unit_test): avoid if continue --- core/dbt/parser/unit_tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index fc6a7ef406d..47240dc8348 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -160,12 +160,15 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): defer_relation=original_input_node.defer_relation, ) - if not resource_type == NodeType.Model: - continue if original_input_node.version: input_node.version = original_input_node.version if original_input_node.latest_version: input_node.latest_version = original_input_node.latest_version + elif resource_type in (NodeType.Seed, NodeType.Snapshot): + input_node = ModelNode( + **common_fields, + defer_relation=original_input_node.defer_relation, + ) elif resource_type == NodeType.Source: # We are reusing the database/schema/identifier from the original source, From 82b205614f18a489b8b441e5db585d6601dd7834 Mon Sep 17 00:00:00 2001 From: devmessias Date: Wed, 13 Nov 2024 11:00:39 -0300 Subject: [PATCH 7/7] fix: wrong conditional --- core/dbt/parser/unit_tests.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 47240dc8348..38a9c81fb3d 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -159,16 +159,11 @@ def parse_unit_test_case(self, test_case: UnitTestDefinition): **common_fields, defer_relation=original_input_node.defer_relation, ) - - if original_input_node.version: - input_node.version = original_input_node.version - if original_input_node.latest_version: - input_node.latest_version = original_input_node.latest_version - elif resource_type in (NodeType.Seed, NodeType.Snapshot): - input_node = ModelNode( - **common_fields, - defer_relation=original_input_node.defer_relation, - ) + if resource_type == NodeType.Model: + if original_input_node.version: + input_node.version = original_input_node.version + if original_input_node.latest_version: + input_node.latest_version = original_input_node.latest_version elif resource_type == NodeType.Source: # We are reusing the database/schema/identifier from the original source,