Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Respect the limit config when storing test failures #375

Draft
wants to merge 9 commits into
base: main
Choose a base branch
from
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,45 @@



{% macro store_failures(main_sql) -%}
{{ adapter.dispatch('store_failures', 'dbt')(main_sql) }}
{%- endmacro %}

{% macro default__store_failures(main_sql) -%}

{% set identifier = model['alias'] %}
{% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}

-- if `--store-failures` is invoked via command line and `store_failures_as` is not set,
-- config.get('store_failures_as', 'table') returns None, not 'table'
{% set store_failures_as = config.get('store_failures_as') or 'table' %}
{% if store_failures_as not in ['table', 'view'] %}
{{ exceptions.raise_compiler_error(
"'" ~ store_failures_as ~ "' is not a valid value for `store_failures_as`. "
"Accepted values are: ['ephemeral', 'table', 'view']"
) }}
{% endif %}

{% set target_relation = api.Relation.create(
identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}

{% if old_relation %}
{% do adapter.drop_relation(old_relation) %}
{% endif %}

{% call statement(auto_begin=True) %}
{{ get_create_sql(target_relation, main_sql) }}
{% endcall %}

{{ adapter.commit() }}

{{ return(target_relation) }}

{%- endmacro %}




{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}
{{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}
{%- endmacro %}
Expand Down
50 changes: 13 additions & 37 deletions dbt/include/global_project/macros/materializations/tests/test.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,32 @@

{% set relations = [] %}

{% if should_store_failures() %}

{% set identifier = model['alias'] %}
{% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}

{% set store_failures_as = config.get('store_failures_as') %}
-- if `--store-failures` is invoked via command line and `store_failures_as` is not set,
-- config.get('store_failures_as', 'table') returns None, not 'table'
{% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}
{% if store_failures_as not in ['table', 'view'] %}
{{ exceptions.raise_compiler_error(
"'" ~ store_failures_as ~ "' is not a valid value for `store_failures_as`. "
"Accepted values are: ['ephemeral', 'table', 'view']"
) }}
{% endif %}

{% set target_relation = api.Relation.create(
identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}
{% set fail_calc = config.get('fail_calc') %}
{% set warn_if = config.get('warn_if') %}
{% set error_if = config.get('error_if') %}
{% set limit = config.get('limit') %}

{% if old_relation %}
{% do adapter.drop_relation(old_relation) %}
{% endif %}
{% set main_sql %}
{{ sql }}
{{ "limit " ~ limit if limit != none }}
{% endset %}

{% call statement(auto_begin=True) %}
{{ get_create_sql(target_relation, sql) }}
{% endcall %}
{% if should_store_failures() %}

{% do relations.append(target_relation) %}
{% set target_relation = store_failures(main_sql) %}

{# Since the test failures have already been saved to the database, reuse that result rather than querying again #}
{% set main_sql %}
select *
from {{ target_relation }}
{% endset %}

{{ adapter.commit() }}

{% else %}

{% set main_sql = sql %}

{% endif %}

{% set limit = config.get('limit') %}
{% set fail_calc = config.get('fail_calc') %}
{% set warn_if = config.get('warn_if') %}
{% set error_if = config.get('error_if') %}

{% call statement('main', fetch_result=True) -%}

{{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}
{# Since the limit has already been applied above, no need to apply it again! #}
{{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit=none)}}

{%- endcall %}

Expand Down
Loading