Use comments and query tags (#8)

* Use comments and query tags
get-select · Feb 24, 2023 · 1c39941 · 1c39941
1 parent b75b31d
commit 1c39941
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 77 deletions.
diff --git a/.changes/2.0.0.md b/.changes/2.0.0.md
@@ -0,0 +1,7 @@
+## dbt-snowflake-query-tags 2.0.0 - February 24, 2023
+
+### Fixes
+
+- Use query comments and query tags to avoid query tag character limit ([#8](https://github.com/get-select/dbt-snowflake-query-tags/pull/8))
+
+
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
+## dbt-snowflake-query-tags 2.0.0 - February 24, 2023
+
+### Fixes
+
+- Use query comments and query tags to avoid query tag character limit ([#8](https://github.com/get-select/dbt-snowflake-query-tags/pull/8))
+
+
+
 ## dbt-snowflake-query-tags 1.1.3 - February 13, 2023
 
 ### Fixes

diff --git a/README.md b/README.md
@@ -1,41 +1,50 @@
 # dbt-snowflake-query-tags
 
-From the [SELECT](https://select.dev) team, a dbt package to automatically tag dbt-issued queries with informative metadata. Example metadata is:
+From the [SELECT](https://select.dev) team, a dbt package to automatically tag dbt-issued queries with informative metadata. This package uses both query comments and query tagging.
+
+An example query comment contains:
 
 ```json
 {
-    "dbt_snowflake_query_tags_version": "1.1.3",
+    "dbt_snowflake_query_tags_version": "2.0.0",
     "app": "dbt",
     "dbt_version": "1.4.0",
     "project_name": "my_project",
     "target_name": "dev",
     "target_database": "dev",
-    "target_schema": "larry_goldings",
-    "invocation_id": "c784c7d0-5c3f-4765-805c-0a377fefcaa0",
-    "node_name": "orders",
-    "node_alias": "orders",
+    "target_schema": "dev",
+    "invocation_id": "4ffa20a1-5d90-4a27-a58a-553bb6890f25",
+    "node_refs": [
+        "model_b",
+        "model_c"
+    ],
+    "node_name": "model_a",
+    "node_alias": "model_a",
     "node_package_name": "my_project",
-    "node_original_file_path": "models/staging/orders.sql",
+    "node_original_file_path": "models/model_a.sql",
     "node_database": "dev",
-    "node_schema": "mart",
-    "node_id": "model.my_project.orders",
+    "node_schema": "dev",
+    "node_id": "model.my_project.model_a",
     "node_resource_type": "model",
     "materialized": "incremental",
-    "is_incremental": true,
-    "node_refs": [
-        "raw_orders",
-        "product_mapping"
-    ]
+
+    -- dbt Cloud only
+    "dbt_cloud_project_id": "146126",
+    "dbt_cloud_job_id": "184124",
+    "dbt_cloud_run_id": "107122910",
+    "dbt_cloud_run_reason_category": "other",
+    "dbt_cloud_run_reason": "Kicked off from UI by [email protected]",
 }
 ```
 
-When running in dbt Cloud, this package also adds the following metadata:
-```
-dbt_cloud_project_id
-dbt_cloud_job_id
-dbt_cloud_run_id
-dbt_cloud_run_reason_category
-dbt_cloud_run_reason
+Query tags are used solely for attaching the `is_incremental` flag, as this isn't available to the query comment:
+
+```json
+{
+    "dbt_snowflake_query_tags_version": "2.0.0",
+    "app": "dbt",
+    "is_incremental": true
+}
 ```
 
 ## Quickstart
@@ -45,10 +54,10 @@ dbt_cloud_run_reason
 ```yaml
 packages:
   - package: get-select/dbt_snowflake_query_tags
-    version: 1.1.3
+    version: 1.2.0
 ```
 
-2. Adding the macros
+2. Adding the query tags
 
 Option 1: If running dbt < 1.2, create a folder named `macros` in your dbt project's top level directory (if it doesn't exist). Inside, make a new file called `query_tags.sql` with the following content:
 
@@ -62,7 +71,7 @@ Option 1: If running dbt < 1.2, create a folder named `macros` in your dbt proje
 {% endmacro %}
 ```
 
-Option 2: If running dbt >= 1.2, you can simply configure the dispatch search order in your `dbt_project.yml`.
+Option 2: If running dbt >= 1.2, simply configure the dispatch search order in `dbt_project.yml`.
 
 ```yaml
 dispatch:
@@ -73,6 +82,14 @@ dispatch:
       - dbt
 ```
 
+3. To configure the query comments, add the following config to `dbt_project.yml`.
+
+```yaml
+query-comment:
+  comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}'
+  append: true # Snowflake removes prefixed comments.
+```
+
 That's it! All dbt-issued queries will now be tagged.
 
 ## Contributing

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,3 +1,3 @@
 name: 'dbt_snowflake_query_tags'
-version: '1.1.3'
+version: '2.0.0'
 config-version: 2
diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml
@@ -11,3 +11,7 @@ dispatch:
       - dbt_snowflake_query_tags_tests
       - dbt_snowflake_query_tags
       - dbt
+
+query-comment:
+  comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}'
+  append: true # Snowflake removes prefixed comments.
diff --git a/macros/query_comment.sql b/macros/query_comment.sql
@@ -0,0 +1,75 @@
+{% macro get_query_comment(node) %}
+    {%- set comment_dict = {} -%}
+    {%- do comment_dict.update(
+        app='dbt',
+        dbt_snowflake_query_tags_version='2.0.0',
+        dbt_version=dbt_version,
+        project_name=project_name,
+        target_name=target.name,
+        target_database=target.database,
+        target_schema=target.schema,
+        invocation_id=invocation_id
+    ) -%}
+
+    {%- if node is not none -%}
+        {%- do comment_dict.update(
+            node_name=node.name,
+            node_alias=node.alias,
+            node_package_name=node.package_name,
+            node_original_file_path=node.original_file_path,
+            node_database=node.database,
+            node_schema=node.schema,
+            node_id=node.unique_id,
+            node_resource_type=node.resource_type,
+        ) -%}
+
+        {%- if node.resource_type != ('seed') -%} {# Otherwise this throws an error saying 'Seeds cannot depend on other nodes.' #}
+            {%- if node.refs is defined -%}
+                {% set refs = [] %}
+                {% for ref in node.refs %}
+                    {%- do refs.append(ref[0]) -%}
+                {% endfor %}
+                {%- do comment_dict.update(
+                    node_refs=refs | unique | list
+                ) -%}
+            {%- endif -%}
+        {%- endif -%}
+        {%- if node.resource_type == 'model' -%}
+            {%- do comment_dict.update(
+                materialized=node.config.materialized,
+            ) -%}
+        {%- endif -%}
+    {%- endif -%}
+
+    {%- if env_var('DBT_CLOUD_PROJECT_ID', False) -%}
+    {%- do comment_dict.update(
+        dbt_cloud_project_id=env_var('DBT_CLOUD_PROJECT_ID')
+    ) -%}
+    {%- endif -%}
+
+    {%- if env_var('DBT_CLOUD_JOB_ID', False) -%}
+    {%- do comment_dict.update(
+        dbt_cloud_job_id=env_var('DBT_CLOUD_JOB_ID')
+    ) -%}
+    {%- endif -%}
+
+    {%- if env_var('DBT_CLOUD_RUN_ID', False) -%}
+    {%- do comment_dict.update(
+        dbt_cloud_run_id=env_var('DBT_CLOUD_RUN_ID')
+    ) -%}
+    {%- endif -%}
+
+    {%- if env_var('DBT_CLOUD_RUN_REASON_CATEGORY', False) -%}
+    {%- do comment_dict.update(
+        dbt_cloud_run_reason_category=env_var('DBT_CLOUD_RUN_REASON_CATEGORY')
+    ) -%}
+    {%- endif -%}
+
+    {%- if env_var('DBT_CLOUD_RUN_REASON', False) -%}
+    {%- do comment_dict.update(
+        dbt_cloud_run_reason=env_var('DBT_CLOUD_RUN_REASON')
+    ) -%}
+    {%- endif -%}
+
+    {{ return(tojson(comment_dict)) }}
+{% endmacro %}
diff --git a/macros/query_tags.sql b/macros/query_tags.sql
@@ -6,64 +6,17 @@
     {# Start with any model-configured dict #}
     {% set tag_dict = config.get('query_tag', default={}) %}
 
-    {# Regardless of resource type, we can always access the config via the 'model' variable #}
     {%- do tag_dict.update(
-        dbt_snowflake_query_tags_version='1.1.3',
         app='dbt',
-        dbt_version=dbt_version,
-        project_name=project_name,
-        target_name=target.name,
-        target_database=target.database,
-        target_schema=target.schema,
-        invocation_id=invocation_id,
-        node_name=model.name,
-        node_alias=model.alias,
-        node_package_name=model.package_name,
-        node_original_file_path=model.original_file_path,
-        node_database=model.database,
-        node_schema=model.schema,
-        node_id=model.unique_id,
-        node_resource_type=model.resource_type,
-        materialized=model.config.materialized,
-        is_incremental=is_incremental(),
+        dbt_snowflake_query_tags_version='2.0.0',
     ) -%}
 
-    {%- if model.refs is defined -%}
-        {% set refs = [] %}
-        {% for ref in model.refs %}
-            {%- do refs.append(ref[0]) -%}
-        {% endfor %}
+    {# We have to bring is_incremental through here because its not available in the comment context #}
+    {% if model.resource_type == 'model' %}
         {%- do tag_dict.update(
-            node_refs=refs | unique | list
+            is_incremental=is_incremental()
         ) -%}
-    {%- endif -%}
-
-    {# dbt Cloud stuff #}
-    {%- if env_var('DBT_CLOUD_PROJECT_ID', False) -%}
-        {%- do tag_dict.update(
-            dbt_cloud_project_id=env_var('DBT_CLOUD_PROJECT_ID')
-        ) -%}
-    {%- endif -%}
-    {%- if env_var('DBT_CLOUD_JOB_ID', False) -%}
-        {%- do tag_dict.update(
-            dbt_cloud_job_id=env_var('DBT_CLOUD_JOB_ID')
-        ) -%}
-    {%- endif -%}
-    {%- if env_var('DBT_CLOUD_RUN_ID', False) -%}
-        {%- do tag_dict.update(
-            dbt_cloud_run_id=env_var('DBT_CLOUD_RUN_ID')
-        ) -%}
-    {%- endif -%}
-    {%- if env_var('DBT_CLOUD_RUN_REASON_CATEGORY', False) -%}
-        {%- do tag_dict.update(
-            dbt_cloud_run_reason_category=env_var('DBT_CLOUD_RUN_REASON_CATEGORY')
-        ) -%}
-    {%- endif -%}
-    {%- if env_var('DBT_CLOUD_RUN_REASON', False) -%}
-        {%- do tag_dict.update(
-            dbt_cloud_run_reason=env_var('DBT_CLOUD_RUN_REASON')
-        ) -%}
-    {%- endif -%}
+    {% endif %}
 
     {% set new_query_tag = tojson(tag_dict) %}
     {% set original_query_tag = get_current_query_tag() %}