From 320a719705e04c5c613a5d04f0ddbe0c4cf3c736 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 9 May 2024 09:49:18 -0400 Subject: [PATCH] empty unit test csv fixture values default to null (#10117) (cherry picked from commit 55aad328ea6878ca8bbc5dc53fb734b6142c0ec7) --- .../unreleased/Fixes-20240509-091411.yaml | 6 ++ core/dbt/events/types.py | 2 +- core/dbt/parser/unit_tests.py | 10 ++- tests/functional/unit_testing/fixtures.py | 50 +++++++++++++ .../unit_testing/test_csv_fixtures.py | 70 ++++++++++++++++--- 5 files changed, 124 insertions(+), 14 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240509-091411.yaml diff --git a/.changes/unreleased/Fixes-20240509-091411.yaml b/.changes/unreleased/Fixes-20240509-091411.yaml new file mode 100644 index 00000000000..a4c243779c5 --- /dev/null +++ b/.changes/unreleased/Fixes-20240509-091411.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Unit test fixture (csv) returns null for empty value +time: 2024-05-09T09:14:11.772709-04:00 +custom: + Author: michelleark + Issue: "9881" diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 8ca9c9279f2..2ce73981a33 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1815,7 +1815,7 @@ def code(self) -> str: return "Z026" def message(self) -> str: - return f" compiled Code at {self.path}" + return f" compiled code at {self.path}" class CheckNodeTestFailure(InfoLevel): diff --git a/core/dbt/parser/unit_tests.py b/core/dbt/parser/unit_tests.py index 8e2dd58a00e..8ae58428dcc 100644 --- a/core/dbt/parser/unit_tests.py +++ b/core/dbt/parser/unit_tests.py @@ -366,11 +366,17 @@ def _validate_and_normalize_rows(self, ut_fixture, unit_test_definition, fixture ) if ut_fixture.fixture: - ut_fixture.rows = self.get_fixture_file_rows( + csv_rows = self.get_fixture_file_rows( ut_fixture.fixture, self.project.project_name, unit_test_definition.unique_id ) else: - ut_fixture.rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows) + csv_rows = self._convert_csv_to_list_of_dicts(ut_fixture.rows) + + # Empty values (e.g. ,,) in a csv fixture should default to null, not "" + ut_fixture.rows = [ + {k: (None if v == "" else v) for k, v in row.items()} for row in csv_rows + ] + elif ut_fixture.format == UnitTestFormat.SQL: if not (isinstance(ut_fixture.rows, str) or isinstance(ut_fixture.fixture, str)): raise ParsingError( diff --git a/tests/functional/unit_testing/fixtures.py b/tests/functional/unit_testing/fixtures.py index 56f4e22210b..3028e0bc1e6 100644 --- a/tests/functional/unit_testing/fixtures.py +++ b/tests/functional/unit_testing/fixtures.py @@ -41,6 +41,15 @@ 'b' as string_b """ +my_model_check_null_sql = """ +SELECT +CASE + WHEN a IS null THEN True + ELSE False +END a_is_null +FROM {{ ref('my_model_a') }} +""" + test_my_model_yml = """ unit_tests: - name: test_my_model @@ -507,6 +516,11 @@ 1,a """ +test_my_model_a_with_null_fixture_csv = """id,a +1, +2,3 +""" + test_my_model_a_empty_fixture_csv = """ """ @@ -1060,3 +1074,39 @@ def external_package(): rows: - {id: 2} """ + + +test_my_model_csv_null_yml = """ +unit_tests: + - name: test_my_model_check_null + model: my_model_check_null + given: + - input: ref('my_model_a') + format: csv + rows: | + id,a + 1, + 2,3 + expect: + format: csv + rows: | + a_is_null + True + False +""" + +test_my_model_file_csv_null_yml = """ +unit_tests: + - name: test_my_model_check_null + model: my_model_check_null + given: + - input: ref('my_model_a') + format: csv + fixture: test_my_model_a_with_null_fixture + expect: + format: csv + rows: | + a_is_null + True + False +""" diff --git a/tests/functional/unit_testing/test_csv_fixtures.py b/tests/functional/unit_testing/test_csv_fixtures.py index 6aae95abed6..c19b2949dbd 100644 --- a/tests/functional/unit_testing/test_csv_fixtures.py +++ b/tests/functional/unit_testing/test_csv_fixtures.py @@ -2,24 +2,28 @@ from dbt.exceptions import ParsingError, YamlParseDictError, DuplicateResourceNameError from dbt.tests.util import run_dbt, write_file, rm_file from fixtures import ( - my_model_sql, - my_model_a_sql, - my_model_b_sql, - test_my_model_csv_yml, datetime_test, - datetime_test_invalid_format_key, datetime_test_invalid_csv_values, - test_my_model_file_csv_yml, - test_my_model_fixture_csv, + datetime_test_invalid_format_key, + my_model_a_sql, + my_model_b_sql, + my_model_check_null_sql, + my_model_sql, + test_my_model_a_empty_fixture_csv, test_my_model_a_fixture_csv, + test_my_model_a_numeric_fixture_csv, + test_my_model_a_with_null_fixture_csv, test_my_model_b_fixture_csv, test_my_model_basic_fixture_csv, - test_my_model_a_numeric_fixture_csv, - test_my_model_a_empty_fixture_csv, test_my_model_concat_fixture_csv, - test_my_model_mixed_csv_yml, - test_my_model_missing_csv_yml, + test_my_model_csv_null_yml, + test_my_model_csv_yml, test_my_model_duplicate_csv_yml, + test_my_model_file_csv_null_yml, + test_my_model_file_csv_yml, + test_my_model_fixture_csv, + test_my_model_missing_csv_yml, + test_my_model_mixed_csv_yml, ) @@ -207,6 +211,50 @@ def test_unit_test(self, project): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) +class TestUnitTestsInlineCSVEmptyValueIsNull: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_a.sql": my_model_a_sql, + "my_model_check_null.sql": my_model_check_null_sql, + "test_my_model_csv_null.yml": test_my_model_csv_null_yml, + } + + def test_unit_test(self, project): + results = run_dbt(["run"]) + assert len(results) == 2 + + # Select by model name + results = run_dbt(["test", "--select", "my_model_check_null"], expect_pass=True) + assert len(results) == 1 + + +class TestUnitTestsFileCSVEmptyValueIsNull: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_a.sql": my_model_a_sql, + "my_model_check_null.sql": my_model_check_null_sql, + "test_my_model_file_csv_null.yml": test_my_model_file_csv_null_yml, + } + + @pytest.fixture(scope="class") + def tests(self): + return { + "fixtures": { + "test_my_model_a_with_null_fixture.csv": test_my_model_a_with_null_fixture_csv, + } + } + + def test_unit_test(self, project): + results = run_dbt(["run"]) + assert len(results) == 2 + + # Select by model name + results = run_dbt(["test", "--select", "my_model_check_null"], expect_pass=True) + assert len(results) == 1 + + class TestUnitTestsMissingCSVFile: @pytest.fixture(scope="class") def models(self):