From b38cb0ca72439023c21b7d81974d48a5c5323a95 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Fri, 22 Jul 2022 11:00:10 -0600 Subject: [PATCH 01/10] initial test materialization with query tagging --- dbt/include/snowflake/macros/materializations/test.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 dbt/include/snowflake/macros/materializations/test.sql diff --git a/dbt/include/snowflake/macros/materializations/test.sql b/dbt/include/snowflake/macros/materializations/test.sql new file mode 100644 index 000000000..f2f74755e --- /dev/null +++ b/dbt/include/snowflake/macros/materializations/test.sql @@ -0,0 +1,10 @@ +{%- materialization test, default -%} + + {% set original_query_tag = set_query_tag() %} + {% set relations = materialization_test_default() %} + + {% do unset_query_tag(original_query_tag) %} + + {{ return({'relations': relations}) }} + +{%- endmaterialization -%} \ No newline at end of file From 29cce4637fc29f2fea34e39bd605c4a2bf39a9d0 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Fri, 22 Jul 2022 11:02:04 -0600 Subject: [PATCH 02/10] set adapter to snowflake --- dbt/include/snowflake/macros/materializations/test.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/materializations/test.sql b/dbt/include/snowflake/macros/materializations/test.sql index f2f74755e..18e1456a5 100644 --- a/dbt/include/snowflake/macros/materializations/test.sql +++ b/dbt/include/snowflake/macros/materializations/test.sql @@ -1,4 +1,4 @@ -{%- materialization test, default -%} +{%- materialization test, adapter='snowflake' -%} {% set original_query_tag = set_query_tag() %} {% set relations = materialization_test_default() %} From ccc77d3baf39bddb497b939afc891a3674127b8b Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Fri, 22 Jul 2022 11:03:46 -0600 Subject: [PATCH 03/10] move over core materialization code for testing --- .../macros/materializations/test.sql | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/dbt/include/snowflake/macros/materializations/test.sql b/dbt/include/snowflake/macros/materializations/test.sql index 18e1456a5..49a9f4d2f 100644 --- a/dbt/include/snowflake/macros/materializations/test.sql +++ b/dbt/include/snowflake/macros/materializations/test.sql @@ -1,10 +1,48 @@ {%- materialization test, adapter='snowflake' -%} - {% set original_query_tag = set_query_tag() %} - {% set relations = materialization_test_default() %} + {% set relations = [] %} + + {% if should_store_failures() %} + + {% set identifier = model['alias'] %} + {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} + {% set target_relation = api.Relation.create( + identifier=identifier, schema=schema, database=database, type='table') -%} %} + + {% if old_relation %} + {% do adapter.drop_relation(old_relation) %} + {% endif %} + + {% call statement(auto_begin=True) %} + {{ create_table_as(False, target_relation, sql) }} + {% endcall %} + + {% do relations.append(target_relation) %} + + {% set main_sql %} + select * + from {{ target_relation }} + {% endset %} + + {{ adapter.commit() }} + + {% else %} + + {% set main_sql = sql %} + + {% endif %} + + {% set limit = config.get('limit') %} + {% set fail_calc = config.get('fail_calc') %} + {% set warn_if = config.get('warn_if') %} + {% set error_if = config.get('error_if') %} + + {% call statement('main', fetch_result=True) -%} + + {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}} + + {%- endcall %} - {% do unset_query_tag(original_query_tag) %} - {{ return({'relations': relations}) }} {%- endmaterialization -%} \ No newline at end of file From 76593af9f628108459dd922742806c6587211f0a Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Fri, 22 Jul 2022 11:47:15 -0600 Subject: [PATCH 04/10] add query tagging macros --- dbt/include/snowflake/macros/materializations/test.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbt/include/snowflake/macros/materializations/test.sql b/dbt/include/snowflake/macros/materializations/test.sql index 49a9f4d2f..ec0387045 100644 --- a/dbt/include/snowflake/macros/materializations/test.sql +++ b/dbt/include/snowflake/macros/materializations/test.sql @@ -1,5 +1,5 @@ {%- materialization test, adapter='snowflake' -%} - + {% set original_query_tag = set_query_tag() %} {% set relations = [] %} {% if should_store_failures() %} @@ -42,7 +42,8 @@ {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}} {%- endcall %} - + + {% do unset_query_tag(original_query_tag) %} {{ return({'relations': relations}) }} {%- endmaterialization -%} \ No newline at end of file From c7f9d1cc2e57fe0de7af70cc8b4b72f8fe9cc7e7 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 25 Jul 2022 08:54:36 -0600 Subject: [PATCH 05/10] revise with jtcohen's version --- .../macros/materializations/test.sql | 51 ++----------------- 1 file changed, 5 insertions(+), 46 deletions(-) diff --git a/dbt/include/snowflake/macros/materializations/test.sql b/dbt/include/snowflake/macros/materializations/test.sql index ec0387045..685bb853a 100644 --- a/dbt/include/snowflake/macros/materializations/test.sql +++ b/dbt/include/snowflake/macros/materializations/test.sql @@ -1,49 +1,8 @@ {%- materialization test, adapter='snowflake' -%} - {% set original_query_tag = set_query_tag() %} - {% set relations = [] %} - - {% if should_store_failures() %} - - {% set identifier = model['alias'] %} - {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} - {% set target_relation = api.Relation.create( - identifier=identifier, schema=schema, database=database, type='table') -%} %} - - {% if old_relation %} - {% do adapter.drop_relation(old_relation) %} - {% endif %} - - {% call statement(auto_begin=True) %} - {{ create_table_as(False, target_relation, sql) }} - {% endcall %} - - {% do relations.append(target_relation) %} - - {% set main_sql %} - select * - from {{ target_relation }} - {% endset %} - - {{ adapter.commit() }} - - {% else %} - - {% set main_sql = sql %} - - {% endif %} - - {% set limit = config.get('limit') %} - {% set fail_calc = config.get('fail_calc') %} - {% set warn_if = config.get('warn_if') %} - {% set error_if = config.get('error_if') %} - - {% call statement('main', fetch_result=True) -%} - - {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}} - - {%- endcall %} - - {% do unset_query_tag(original_query_tag) %} - {{ return({'relations': relations}) }} + + {% set original_query_tag = set_query_tag() %} + {% set relations = materialization_test_default() %} + {% do unset_query_tag(original_query_tag) %} + {{ return(relations) }} {%- endmaterialization -%} \ No newline at end of file From baceaea0ed70989adce9f177e943ef110c9a52b8 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 25 Jul 2022 13:34:51 -0600 Subject: [PATCH 06/10] add validation on query tags for tests --- tests/integration/query_tag_tests/models/models.yml | 8 ++++++++ tests/integration/query_tag_tests/test_query_tags.py | 6 ++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/integration/query_tag_tests/models/models.yml diff --git a/tests/integration/query_tag_tests/models/models.yml b/tests/integration/query_tag_tests/models/models.yml new file mode 100644 index 000000000..49ab57765 --- /dev/null +++ b/tests/integration/query_tag_tests/models/models.yml @@ -0,0 +1,8 @@ +version: 2 + +models: + - name: view_model_query_tag + columns: + - name: id + tests: + - unique \ No newline at end of file diff --git a/tests/integration/query_tag_tests/test_query_tags.py b/tests/integration/query_tag_tests/test_query_tags.py index 24cc7c52a..f81d64a89 100644 --- a/tests/integration/query_tag_tests/test_query_tags.py +++ b/tests/integration/query_tag_tests/test_query_tags.py @@ -33,6 +33,12 @@ def project_config(self): 'post-hook': '{{ check_query_tag() }}' }, }, + 'tests': { + 'test': { + 'query_tag': self.prefix, + 'post-hook': '{{ check_query_tag() }}' + } + }, } def build_all_with_query_tags(self): From faa780c73fff81e66c7c92bc60f6b08fa823e385 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 15 Aug 2022 23:01:52 -0600 Subject: [PATCH 07/10] add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d75b0bf5d..84ec58fd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ## dbt-snowflake 1.3.0-b1 - August 03, 2022 ### Features - Support python model through create stored procedure with python in it, currently supported materializations are table and incremental. ([#216](https://github.com/dbt-labs/dbt-snowflake/issues/216), [#182](https://github.com/dbt-labs/dbt-snowflake/pull/182)) +- Support applying query tags in dbt tests using the `+query_tag` config. ([#211](https://github.com/dbt-labs/dbt-snowflake/issues/211)) Note this does not include the ability to specify tests in `models.yml` files, per dbt-core issue [#5532](https://github.com/dbt-labs/dbt-core/issues/5532). ### Under the Hood - Reformat overridden macro location of grants work to a apply_grants.sql file in snowflake ([#193](https://github.com/dbt-labs/dbt-snowflake/issues/193), [#192](https://github.com/dbt-labs/dbt-snowflake/pull/192)) - Support dbt Core incrmental materialization refactor ([#195](https://github.com/dbt-labs/dbt-snowflake/issues/195), [#196](https://github.com/dbt-labs/dbt-snowflake/pull/196)) From 7df015e632173384770aedfccf1c57e9919d0347 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 15 Aug 2022 23:06:01 -0600 Subject: [PATCH 08/10] update changelog again --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84ec58fd2..cf8781cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ ## dbt-snowflake 1.3.0-b1 - August 03, 2022 ### Features - Support python model through create stored procedure with python in it, currently supported materializations are table and incremental. ([#216](https://github.com/dbt-labs/dbt-snowflake/issues/216), [#182](https://github.com/dbt-labs/dbt-snowflake/pull/182)) -- Support applying query tags in dbt tests using the `+query_tag` config. ([#211](https://github.com/dbt-labs/dbt-snowflake/issues/211)) Note this does not include the ability to specify tests in `models.yml` files, per dbt-core issue [#5532](https://github.com/dbt-labs/dbt-core/issues/5532). +- Support applying query tags in dbt tests using the `+query_tag` config in `dbt_project.yml`. ([#211](https://github.com/dbt-labs/dbt-snowflake/issues/211)) + + ### Under the Hood - Reformat overridden macro location of grants work to a apply_grants.sql file in snowflake ([#193](https://github.com/dbt-labs/dbt-snowflake/issues/193), [#192](https://github.com/dbt-labs/dbt-snowflake/pull/192)) - Support dbt Core incrmental materialization refactor ([#195](https://github.com/dbt-labs/dbt-snowflake/issues/195), [#196](https://github.com/dbt-labs/dbt-snowflake/pull/196)) From 1f0773ee90880c27eb9e76386787231e46b43d88 Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 15 Aug 2022 23:09:04 -0600 Subject: [PATCH 09/10] fix linting issue & update changelog with PR --- CHANGELOG.md | 2 +- dbt/include/snowflake/macros/materializations/test.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf8781cb1..3fb83090a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ## dbt-snowflake 1.3.0-b1 - August 03, 2022 ### Features - Support python model through create stored procedure with python in it, currently supported materializations are table and incremental. ([#216](https://github.com/dbt-labs/dbt-snowflake/issues/216), [#182](https://github.com/dbt-labs/dbt-snowflake/pull/182)) -- Support applying query tags in dbt tests using the `+query_tag` config in `dbt_project.yml`. ([#211](https://github.com/dbt-labs/dbt-snowflake/issues/211)) +- Support applying query tags in dbt tests using the `+query_tag` config in `dbt_project.yml`. ([#211](https://github.com/dbt-labs/dbt-snowflake/issues/211), [#210](https://github.com/dbt-labs/dbt-snowflake/pull/210)). ### Under the Hood diff --git a/dbt/include/snowflake/macros/materializations/test.sql b/dbt/include/snowflake/macros/materializations/test.sql index 685bb853a..816a74893 100644 --- a/dbt/include/snowflake/macros/materializations/test.sql +++ b/dbt/include/snowflake/macros/materializations/test.sql @@ -1,8 +1,8 @@ {%- materialization test, adapter='snowflake' -%} - + {% set original_query_tag = set_query_tag() %} {% set relations = materialization_test_default() %} {% do unset_query_tag(original_query_tag) %} {{ return(relations) }} -{%- endmaterialization -%} \ No newline at end of file +{%- endmaterialization -%} From 5efc47dbf13bed0012ecb3c492e405b3e3edaafb Mon Sep 17 00:00:00 2001 From: Matt Winkler Date: Mon, 15 Aug 2022 23:35:56 -0600 Subject: [PATCH 10/10] update changelog using changie --- .changes/unreleased/Features-20220815-233505.yaml | 7 +++++++ CHANGELOG.md | 3 --- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Features-20220815-233505.yaml diff --git a/.changes/unreleased/Features-20220815-233505.yaml b/.changes/unreleased/Features-20220815-233505.yaml new file mode 100644 index 000000000..cc29f7444 --- /dev/null +++ b/.changes/unreleased/Features-20220815-233505.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Support query tags in dbt tests using the `+query_tag` config in `dbt_project.yml` +time: 2022-08-15T23:35:05.967484-06:00 +custom: + Author: matt-winkler + Issue: "211" + PR: "210" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fb83090a..d75b0bf5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,6 @@ ## dbt-snowflake 1.3.0-b1 - August 03, 2022 ### Features - Support python model through create stored procedure with python in it, currently supported materializations are table and incremental. ([#216](https://github.com/dbt-labs/dbt-snowflake/issues/216), [#182](https://github.com/dbt-labs/dbt-snowflake/pull/182)) -- Support applying query tags in dbt tests using the `+query_tag` config in `dbt_project.yml`. ([#211](https://github.com/dbt-labs/dbt-snowflake/issues/211), [#210](https://github.com/dbt-labs/dbt-snowflake/pull/210)). - - ### Under the Hood - Reformat overridden macro location of grants work to a apply_grants.sql file in snowflake ([#193](https://github.com/dbt-labs/dbt-snowflake/issues/193), [#192](https://github.com/dbt-labs/dbt-snowflake/pull/192)) - Support dbt Core incrmental materialization refactor ([#195](https://github.com/dbt-labs/dbt-snowflake/issues/195), [#196](https://github.com/dbt-labs/dbt-snowflake/pull/196))