From 97ffc3740542bda6a2cf5cef75b8b4131248705b Mon Sep 17 00:00:00 2001 From: Patrick Yost <159199360+theyostalservice@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:18:50 -0800 Subject: [PATCH] Add tags to SavedQueries (#10987) --- .../unreleased/Features-20241216-095435.yaml | 6 ++++ .../dbt/artifacts/resources/v1/saved_query.py | 8 ++++- core/dbt/contracts/graph/nodes.py | 4 +++ core/dbt/contracts/graph/unparsed.py | 9 +++++ core/dbt/parser/schema_yaml_readers.py | 13 +++++++ schemas/dbt/catalog/v1.json | 2 +- schemas/dbt/manifest/v12.json | 26 ++++++++++++++ schemas/dbt/run-results/v6.json | 5 +-- schemas/dbt/sources/v3.json | 2 +- tests/functional/saved_queries/fixtures.py | 24 +++++++++++++ .../functional/saved_queries/test_configs.py | 34 +++++++++++++++++++ 11 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 .changes/unreleased/Features-20241216-095435.yaml diff --git a/.changes/unreleased/Features-20241216-095435.yaml b/.changes/unreleased/Features-20241216-095435.yaml new file mode 100644 index 00000000000..706a84f8f39 --- /dev/null +++ b/.changes/unreleased/Features-20241216-095435.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support "tags" in Saved Queries +time: 2024-12-16T09:54:35.327675-08:00 +custom: + Author: theyostalservice + Issue: "11155" diff --git a/core/dbt/artifacts/resources/v1/saved_query.py b/core/dbt/artifacts/resources/v1/saved_query.py index e1d056d0422..8d51845755b 100644 --- a/core/dbt/artifacts/resources/v1/saved_query.py +++ b/core/dbt/artifacts/resources/v1/saved_query.py @@ -2,16 +2,18 @@ import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional, Union from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import DependsOn, RefArgs +from dbt.artifacts.resources.v1.config import list_str, metas from dbt.artifacts.resources.v1.semantic_layer_components import ( SourceFileMetadata, WhereFilterIntersection, ) from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior +from dbt_common.contracts.config.metadata import ShowBehavior from dbt_common.dataclass_schema import dbtClassMixin from dbt_semantic_interfaces.type_enums.export_destination_type import ( ExportDestinationType, @@ -95,6 +97,10 @@ class SavedQuery(SavedQueryMandatory): depends_on: DependsOn = field(default_factory=DependsOn) created_at: float = field(default_factory=lambda: time.time()) refs: List[RefArgs] = field(default_factory=list) + tags: Union[List[str], str] = field( + default_factory=list_str, + metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), + ) @property def metrics(self) -> List[str]: diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 4bb70db5d9c..f753a6afff1 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1647,6 +1647,9 @@ def same_exports(self, old: "SavedQuery") -> bool: return True + def same_tags(self, old: "SavedQuery") -> bool: + return self.tags == old.tags + def same_contents(self, old: Optional["SavedQuery"]) -> bool: # existing when it didn't before is a change! # metadata/tags changes are not "changes" @@ -1662,6 +1665,7 @@ def same_contents(self, old: Optional["SavedQuery"]) -> bool: and self.same_config(old) and self.same_group(old) and self.same_exports(old) + and self.same_tags(old) and True ) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index f78ba15a50f..f8d5c581aca 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -27,8 +27,11 @@ UnitTestOutputFixture, UnitTestOverrides, ) +from dbt.artifacts.resources.v1.config import list_str, metas from dbt.exceptions import ParsingError from dbt.node_types import NodeType +from dbt_common.contracts.config.base import CompareBehavior, MergeBehavior +from dbt_common.contracts.config.metadata import ShowBehavior from dbt_common.contracts.config.properties import AdditionalPropertiesMixin from dbt_common.contracts.util import Mergeable from dbt_common.dataclass_schema import ( @@ -740,6 +743,12 @@ class UnparsedSavedQuery(dbtClassMixin): label: Optional[str] = None exports: List[UnparsedExport] = field(default_factory=list) config: Dict[str, Any] = field(default_factory=dict) + # Note: the order of the types is critical; it's the order that they will be checked against inputs. + # if reversed, a single-string tag like `tag: "good"` becomes ['g','o','o','d'] + tags: Union[str, List[str]] = field( + default_factory=list_str, + metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), + ) def normalize_date(d: Optional[datetime.date]) -> Optional[datetime.datetime]: diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 6e312780141..aca239db153 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -799,6 +799,18 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: rendered=False, ) + # The parser handles plain strings just fine, but we need to be able + # to join two lists, remove duplicates, and sort, so we have to wrap things here. + def wrap_tags(s: Union[List[str], str]) -> List[str]: + if s is None: + return [] + return [s] if isinstance(s, str) else s + + config_tags = wrap_tags(config.get("tags")) + unparsed_tags = wrap_tags(unparsed.tags) + tags = list(set([*unparsed_tags, *config_tags])) + tags.sort() + parsed = SavedQuery( description=unparsed.description, label=unparsed.label, @@ -814,6 +826,7 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: config=config, unrendered_config=unrendered_config, group=config.group, + tags=tags, ) for export in parsed.exports: diff --git a/schemas/dbt/catalog/v1.json b/schemas/dbt/catalog/v1.json index f104c5b977f..ef54c19d714 100644 --- a/schemas/dbt/catalog/v1.json +++ b/schemas/dbt/catalog/v1.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b2" + "default": "1.10.0a1" }, "generated_at": { "type": "string" diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index d0466e4852c..21f2a9a77f5 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -19781,6 +19781,19 @@ "name" ] } + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] } }, "additionalProperties": false, @@ -21399,6 +21412,19 @@ "name" ] } + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] } }, "additionalProperties": false, diff --git a/schemas/dbt/run-results/v6.json b/schemas/dbt/run-results/v6.json index 1bf1cf75e83..32f0ec46748 100644 --- a/schemas/dbt/run-results/v6.json +++ b/schemas/dbt/run-results/v6.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b2" + "default": "1.10.0a1" }, "generated_at": { "type": "string" @@ -55,7 +55,8 @@ "success", "error", "skipped", - "partial success" + "partial success", + "no-op" ] }, { diff --git a/schemas/dbt/sources/v3.json b/schemas/dbt/sources/v3.json index df2784f1a81..c07cb574df6 100644 --- a/schemas/dbt/sources/v3.json +++ b/schemas/dbt/sources/v3.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b2" + "default": "1.10.0a1" }, "generated_at": { "type": "string" diff --git a/tests/functional/saved_queries/fixtures.py b/tests/functional/saved_queries/fixtures.py index 58ed73c81b0..ac45f8aeb76 100644 --- a/tests/functional/saved_queries/fixtures.py +++ b/tests/functional/saved_queries/fixtures.py @@ -164,3 +164,27 @@ export_as: table schema: my_export_schema_name """ + +saved_query_with_tags_defined_yml = """ +saved_queries: + - name: test_saved_query + description: "{{ doc('saved_query_description') }}" + label: Test Saved Query + tags: + - tag_a + - tag_c + query_params: + metrics: + - simple_metric + group_by: + - "Dimension('id__ds')" + where: + - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" + - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" + exports: + - name: my_export + config: + alias: my_export_alias + export_as: table + schema: my_export_schema_name +""" diff --git a/tests/functional/saved_queries/test_configs.py b/tests/functional/saved_queries/test_configs.py index df4be7aa5b6..943f4819fcd 100644 --- a/tests/functional/saved_queries/test_configs.py +++ b/tests/functional/saved_queries/test_configs.py @@ -14,6 +14,7 @@ saved_query_with_cache_configs_defined_yml, saved_query_with_export_configs_defined_at_saved_query_level_yml, saved_query_with_extra_config_attributes_yml, + saved_query_with_tags_defined_yml, saved_query_without_export_configs_defined_yml, ) from tests.functional.semantic_models.fixtures import ( @@ -322,3 +323,36 @@ def test_override_saved_query_config( result = runner.invoke(["parse"]) assert result.success assert saved_query.config.cache.enabled is True + + +# the tags defined in project yaml for the SavedQuery is additive to the query's +class TestSavedQueryTagsAdditiveWithConfig(BaseConfigProject): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "saved-queries": {"+tags": ["tag_b", "tag_c"]}, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "saved_queries.yml": saved_query_with_tags_defined_yml, + "schema.yml": schema_yml, + "fct_revenue.sql": fct_revenue_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "docs.md": saved_query_description, + } + + def test_saved_query_tags_are_additive_unique_and_sorted( + self, + project, + ): + runner = dbtTestRunner() + + # parse with default fixture project config + result = runner.invoke(["parse"]) + assert result.success + assert isinstance(result.result, Manifest) + assert len(result.result.saved_queries) == 1 + saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] + assert saved_query.tags == ["tag_a", "tag_b", "tag_c"]