From 90e12212fa9bd8593a7d1d2f0eb692c6e09653cd Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 22 Jan 2024 16:25:06 -0800 Subject: [PATCH 1/2] update dbt-adapters to include unit test feature updates --- .../macros/materializations/tests/helpers.sql | 30 ++++++++ .../macros/materializations/tests/unit.sql | 29 +++++++ .../macros/unit_test_sql/get_fixture_sql.sql | 76 +++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 dbt/include/global_project/macros/materializations/tests/unit.sql create mode 100644 dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql diff --git a/dbt/include/global_project/macros/materializations/tests/helpers.sql b/dbt/include/global_project/macros/materializations/tests/helpers.sql index efc55288..13e640c2 100644 --- a/dbt/include/global_project/macros/materializations/tests/helpers.sql +++ b/dbt/include/global_project/macros/materializations/tests/helpers.sql @@ -12,3 +12,33 @@ {{ "limit " ~ limit if limit != none }} ) dbt_internal_test {%- endmacro %} + + + + +{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%} + {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }} +{%- endmacro %} + +{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%} +-- Build actual result given inputs +with dbt_internal_unit_test_actual AS ( + select + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal("actual") }} as actual_or_expected + from ( + {{ main_sql }} + ) _dbt_internal_unit_test_actual +), +-- Build expected result +dbt_internal_unit_test_expected AS ( + select + {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal("expected") }} as actual_or_expected + from ( + {{ expected_fixture_sql }} + ) _dbt_internal_unit_test_expected +) +-- Union actual and expected results +select * from dbt_internal_unit_test_actual +union all +select * from dbt_internal_unit_test_expected +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/global_project/macros/materializations/tests/unit.sql b/dbt/include/global_project/macros/materializations/tests/unit.sql new file mode 100644 index 00000000..79d5631b --- /dev/null +++ b/dbt/include/global_project/macros/materializations/tests/unit.sql @@ -0,0 +1,29 @@ +{%- materialization unit, default -%} + + {% set relations = [] %} + + {% set expected_rows = config.get('expected_rows') %} + {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %} + + {%- set target_relation = this.incorporate(type='table') -%} + {%- set temp_relation = make_temp_relation(target_relation)-%} + {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %} + {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%} + {%- set column_name_to_data_types = {} -%} + {%- for column in columns_in_relation -%} + {%- do column_name_to_data_types.update({column.name: column.dtype}) -%} + {%- endfor -%} + + {% set unit_test_sql = get_unit_test_sql(sql, get_expected_sql(expected_rows, column_name_to_data_types), tested_expected_column_names) %} + + {% call statement('main', fetch_result=True) -%} + + {{ unit_test_sql }} + + {%- endcall %} + + {% do adapter.drop_relation(temp_relation) %} + + {{ return({'relations': relations}) }} + +{%- endmaterialization -%} diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql new file mode 100644 index 00000000..2f90a561 --- /dev/null +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -0,0 +1,76 @@ +{% macro get_fixture_sql(rows, column_name_to_data_types) %} +-- Fixture for {{ model.name }} +{% set default_row = {} %} + +{%- if not column_name_to_data_types -%} +{%- set columns_in_relation = adapter.get_columns_in_relation(this) -%} +{%- set column_name_to_data_types = {} -%} +{%- for column in columns_in_relation -%} +{%- do column_name_to_data_types.update({column.name: column.dtype}) -%} +{%- endfor -%} +{%- endif -%} + +{%- if not column_name_to_data_types -%} + {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this) }} +{%- endif -%} + +{%- for column_name, column_type in column_name_to_data_types.items() -%} + {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%} +{%- endfor -%} + +{%- for row in rows -%} +{%- do format_row(row, column_name_to_data_types) -%} +{%- set default_row_copy = default_row.copy() -%} +{%- do default_row_copy.update(row) -%} +select +{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%}, {%- endif %} +{%- endfor %} +{%- if not loop.last %} +union all +{% endif %} +{%- endfor -%} + +{%- if (rows | length) == 0 -%} + select + {%- for column_name, column_value in default_row.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%},{%- endif %} + {%- endfor %} + limit 0 +{%- endif -%} +{% endmacro %} + + +{% macro get_expected_sql(rows, column_name_to_data_types) %} + +{%- if (rows | length) == 0 -%} + select * FROM dbt_internal_unit_test_actual + limit 0 +{%- else -%} +{%- for row in rows -%} +{%- do format_row(row, column_name_to_data_types) -%} +select +{%- for column_name, column_value in row.items() %} {{ column_value }} AS {{ column_name }}{% if not loop.last -%}, {%- endif %} +{%- endfor %} +{%- if not loop.last %} +union all +{% endif %} +{%- endfor -%} +{%- endif -%} + +{% endmacro %} + +{%- macro format_row(row, column_name_to_data_types) -%} + +{#-- wrap yaml strings in quotes, apply cast --#} +{%- for column_name, column_value in row.items() -%} +{% set row_update = {column_name: column_value} %} +{%- if column_value is string -%} +{%- set row_update = {column_name: safe_cast(dbt.string_literal(column_value), column_name_to_data_types[column_name]) } -%} +{%- elif column_value is none -%} +{%- set row_update = {column_name: safe_cast('null', column_name_to_data_types[column_name]) } -%} +{%- else -%} +{%- set row_update = {column_name: safe_cast(column_value, column_name_to_data_types[column_name]) } -%} +{%- endif -%} +{%- do row.update(row_update) -%} +{%- endfor -%} + +{%- endmacro -%} From ae0f652af7f4ef081ac09bc308ab05f8a521e458 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 23 Jan 2024 09:17:50 -0800 Subject: [PATCH 2/2] add none default limit to create_ephemeral --- dbt/adapters/base/relation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index 13f48977..ea03b067 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -228,7 +228,7 @@ def add_ephemeral_prefix(name: str): def create_ephemeral_from( cls: Type[Self], relation_config: RelationConfig, - limit: Optional[int], + limit: Optional[int] = None, ) -> Self: # Note that ephemeral models are based on the name. identifier = cls.add_ephemeral_prefix(relation_config.name)