Add SavedQuery nodes

.changes/unreleased/Features-20231010-154217.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add support semantic layer SavedQuery node type
+time: 2023-10-10T15:42:17.796231-07:00
+custom:
+  Author: QMalcolm
+  Issue: "8594"

core/dbt/config/project.py

@@ -427,6 +427,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         tests: Dict[str, Any]
         metrics: Dict[str, Any]
         semantic_models: Dict[str, Any]
+        saved_queries: Dict[str, Any]
         exposures: Dict[str, Any]
         vars_value: VarProvider
         dbt_cloud: Dict[str, Any]
@@ -439,6 +440,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
         tests = cfg.tests
         metrics = cfg.metrics
         semantic_models = cfg.semantic_models
+        saved_queries = cfg.saved_queries
         exposures = cfg.exposures
         if cfg.vars is None:
             vars_dict: Dict[str, Any] = {}
@@ -498,6 +500,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
             tests=tests,
             metrics=metrics,
             semantic_models=semantic_models,
+            saved_queries=saved_queries,
             exposures=exposures,
             vars=vars_value,
             config_version=cfg.config_version,
@@ -606,6 +609,7 @@ class Project:
     tests: Dict[str, Any]
     metrics: Dict[str, Any]
     semantic_models: Dict[str, Any]
+    saved_queries: Dict[str, Any]
     exposures: Dict[str, Any]
     vars: VarProvider
     dbt_version: List[VersionSpecifier]
@@ -683,6 +687,7 @@ def to_project_config(self, with_packages=False):
                 "tests": self.tests,
                 "metrics": self.metrics,
                 "semantic-models": self.semantic_models,
+                "saved-queries": self.saved_queries,
                 "exposures": self.exposures,
                 "vars": self.vars.to_dict(),
                 "require-dbt-version": [v.to_version_string() for v in self.dbt_version],

core/dbt/config/runtime.py

@@ -168,6 +168,7 @@ def from_parts(
             tests=project.tests,
             metrics=project.metrics,
             semantic_models=project.semantic_models,
+            saved_queries=project.saved_queries,
             exposures=project.exposures,
             vars=project.vars,
             config_version=project.config_version,
@@ -325,6 +326,7 @@ def get_resource_config_paths(self) -> Dict[str, PathSet]:
             "tests": self._get_config_paths(self.tests),
             "metrics": self._get_config_paths(self.metrics),
             "semantic_models": self._get_config_paths(self.semantic_models),
+            "saved_queries": self._get_config_paths(self.saved_queries),
             "exposures": self._get_config_paths(self.exposures),
         }
 

core/dbt/context/context_config.py

@@ -47,6 +47,8 @@ def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]:
             model_configs = unrendered.get("metrics")
         elif resource_type == NodeType.SemanticModel:
             model_configs = unrendered.get("semantic_models")
+        elif resource_type == NodeType.SavedQuery:
+            model_configs = unrendered.get("saved_queries")
         elif resource_type == NodeType.Exposure:
             model_configs = unrendered.get("exposures")
         else:
@@ -74,6 +76,8 @@ def get_config_dict(self, resource_type: NodeType) -> Dict[str, Any]:
             model_configs = self.project.metrics
         elif resource_type == NodeType.SemanticModel:
             model_configs = self.project.semantic_models
+        elif resource_type == NodeType.SavedQuery:
+            model_configs = self.project.saved_queries
         elif resource_type == NodeType.Exposure:
             model_configs = self.project.exposures
         else:

core/dbt/contracts/files.py

@@ -231,6 +231,7 @@ class SchemaSourceFile(BaseSourceFile):
     # node patches contain models, seeds, snapshots, analyses
     ndp: List[str] = field(default_factory=list)
     semantic_models: List[str] = field(default_factory=list)
+    saved_queries: List[str] = field(default_factory=list)
     # any macro patches in this file by macro unique_id.
     mcp: Dict[str, str] = field(default_factory=dict)
     # any source patches in this file. The entries are package, name pairs

core/dbt/contracts/graph/manifest.py

@@ -37,6 +37,7 @@
     ModelNode,
     DeferRelation,
     ResultNode,
+    SavedQuery,
     SemanticModel,
     SourceDefinition,
     UnpatchedSourceDefinition,
@@ -299,6 +300,41 @@
         return manifest.metrics[unique_id]
 
 
+class SavedQueryLookup(dbtClassMixin):
+    """Lookup utility for finding SavedQuery nodes"""
+
+    def __init__(self, manifest: "Manifest") -> None:
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, search_name, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, search_name, package)
+
+    def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"):
+        unique_id = self.get_unique_id(search_name, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
+    def add_saved_query(self, saved_query: SavedQuery):
+        if saved_query.search_name not in self.storage:
+            self.storage[saved_query.search_name] = {}
+
+        self.storage[saved_query.search_name][saved_query.package_name] = saved_query.unique_id
+
+    def populate(self, manifest):
+        for saved_query in manifest.saved_queries.values():
+            if hasattr(saved_query, "name"):
+                self.add_saved_query(saved_query)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SavedQuery:
+        if unique_id not in manifest.saved_queries:
+            raise dbt.exceptions.DbtInternalError(
+                f"SavedQUery {unique_id} found in cache but not found in manifest"
+            )
+        return manifest.saved_queries[unique_id]
+
+
 class SemanticModelByMeasureLookup(dbtClassMixin):
     """Lookup utility for finding SemanticModel by measure
 
@@ -609,6 +645,9 @@
 MaybeMetricNode = Optional[Union[Metric, Disabled[Metric]]]
 
 
+MaybeSavedQueryNode = Optional[Union[SavedQuery, Disabled[SavedQuery]]]
+
+
 MaybeDocumentation = Optional[Documentation]
 
 
@@ -753,6 +792,7 @@
     disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict)
     env_vars: MutableMapping[str, str] = field(default_factory=dict)
     semantic_models: MutableMapping[str, SemanticModel] = field(default_factory=dict)
+    saved_queries: MutableMapping[str, SavedQuery] = field(default_factory=dict)
 
     _doc_lookup: Optional[DocLookup] = field(
         default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
@@ -766,6 +806,9 @@
     _metric_lookup: Optional[MetricLookup] = field(
         default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
     )
+    _saved_query_lookup: Optional[SavedQueryLookup] = field(
+        default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
+    )
     _semantic_model_by_measure_lookup: Optional[SemanticModelByMeasureLookup] = field(
         default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
     )
@@ -810,6 +853,9 @@
             "semantic_models": {
                 k: v.to_dict(omit_none=False) for k, v in self.semantic_models.items()
             },
+            "saved_queries": {
+                k: v.to_dict(omit_none=False) for k, v in self.saved_queries.items()
+            },
         }
 
     def build_disabled_by_file_id(self):
@@ -871,6 +917,7 @@
             self.sources.values(),
             self.metrics.values(),
             self.semantic_models.values(),
+            self.saved_queries.values(),
         )
         for resource in all_resources:
             resource_type_plural = resource.resource_type.pluralize()
@@ -906,6 +953,7 @@
             files={k: _deepcopy(v) for k, v in self.files.items()},
             state_check=_deepcopy(self.state_check),
             semantic_models={k: _deepcopy(v) for k, v in self.semantic_models.items()},
+            saved_queries={k: _deepcopy(v) for k, v in self.saved_queries.items()},
         )
         copy.build_flat_graph()
         return copy
@@ -918,6 +966,7 @@
                 self.exposures.values(),
                 self.metrics.values(),
                 self.semantic_models.values(),
+                self.saved_queries.values(),
             )
         )
         forward_edges, backward_edges = build_node_edges(edge_members)
@@ -938,6 +987,7 @@
         groupable_nodes = list(
             chain(
                 self.nodes.values(),
+                self.saved_queries.values(),
                 self.semantic_models.values(),
                 self.metrics.values(),
             )
@@ -966,6 +1016,7 @@
             parent_map=self.parent_map,
             group_map=self.group_map,
             semantic_models=self.semantic_models,
+            saved_queries=self.saved_queries,
         )
 
     def write(self, path):
@@ -984,6 +1035,8 @@
             return self.metrics[unique_id]
         elif unique_id in self.semantic_models:
             return self.semantic_models[unique_id]
+        elif unique_id in self.saved_queries:
+            return self.saved_queries[unique_id]
         else:
             # something terrible has happened
             raise dbt.exceptions.DbtInternalError(
@@ -1020,6 +1073,13 @@
             self._metric_lookup = MetricLookup(self)
         return self._metric_lookup
 
+    @property
+    def saved_query_lookup(self) -> SavedQueryLookup:
+        """Retuns a SavedQueryLookup, instantiating it first if necessary."""
+        if self._saved_query_lookup is None:
+            self._saved_query_lookup = SavedQueryLookup(self)
+        return self._saved_query_lookup
+
     @property
     def semantic_model_by_measure_lookup(self) -> SemanticModelByMeasureLookup:
         """Gets (and creates if necessary) the lookup utility for getting SemanticModels by measures"""
@@ -1153,6 +1213,35 @@
             return Disabled(disabled[0])
         return None
 
+    def resolve_saved_query(
+        self,
+        target_saved_query_name: str,
+        target_saved_query_package: Optional[str],
+        current_project: str,
+        node_package: str,
+    ) -> MaybeSavedQueryNode:
+        """Tries to find the SavedQuery by name within the available project and packages.
+
+        Will return the first enabled SavedQuery matching the name found while iterating over
+        the scoped packages. If no enabled SavedQuery node match is found, returns the last
+        disabled SavedQuery node. Otherwise it returns None.
+        """
+        disabled: Optional[List[SavedQuery]] = None
+        candidates = _packages_to_search(current_project, node_package, target_saved_query_package)
+        for pkg in candidates:
+            saved_query = self.saved_query_lookup.find(target_saved_query_name, pkg, self)
+
+            if saved_query is not None and saved_query.config.enabled:
+                return saved_query
+
+            # it's possible that the node is disabled
+            if disabled is None:
+                disabled = self.disabled_lookup.find(f"{target_saved_query_name}", pkg)
+        if disabled:
+            return Disabled(disabled[0])
+
+        return None
+
     def resolve_semantic_model_for_measure(
         self,
         target_measure_name: str,
@@ -1371,6 +1460,8 @@
                 source_file.add_test(node.unique_id, test_from)
             if isinstance(node, Metric):
                 source_file.metrics.append(node.unique_id)
+            if isinstance(node, SavedQuery):
+                source_file.saved_queries.append(node.unique_id)
             if isinstance(node, SemanticModel):
                 source_file.semantic_models.append(node.unique_id)
             if isinstance(node, Exposure):
@@ -1388,6 +1479,11 @@
         self.semantic_models[semantic_model.unique_id] = semantic_model
         source_file.semantic_models.append(semantic_model.unique_id)
 
+    def add_saved_query(self, source_file: SchemaSourceFile, saved_query: SavedQuery) -> None:
+        _check_duplicates(saved_query, self.saved_queries)
+        self.saved_queries[saved_query.unique_id] = saved_query
+        source_file.saved_queries.append(saved_query.unique_id)
+
     # end of methods formerly in ParseResult
 
     # Provide support for copy.deepcopy() - we just need to avoid the lock!
@@ -1415,6 +1511,7 @@
             self.disabled,
             self.env_vars,
             self.semantic_models,
+            self.saved_queries,
             self._doc_lookup,
             self._source_lookup,
             self._ref_lookup,
@@ -1485,6 +1582,9 @@
             description="A mapping from group names to their nodes",
         )
     )
+    saved_queries: Mapping[UniqueID, SavedQuery] = field(
+        metadata=dict(description=("The saved queries defined in the dbt project"))
+    )
     semantic_models: Mapping[UniqueID, SemanticModel] = field(
         metadata=dict(description=("The semantic models defined in the dbt project"))
     )

core/dbt/contracts/graph/manifest_upgrade.py

@@ -104,4 +104,6 @@ def upgrade_manifest_json(manifest: dict, manifest_schema_version: int) -> dict:
         doc_content["resource_type"] = "doc"
     if "semantic_models" not in manifest:
         manifest["semantic_models"] = {}
+    if "saved_queries" not in manifest:
+        manifest["saved_queries"] = {}
     return manifest

core/dbt/contracts/graph/model_config.py

@@ -388,6 +388,27 @@ class SemanticModelConfig(BaseConfig):
     )
 
 
+@dataclass
+class SavedQueryConfig(BaseConfig):
+    """Where config options for SavedQueries are stored.
+
+    This class is much like many other node config classes. It's likely that
+    this class will expand in the direction of what's in the `NodeAndTestConfig`
+    class. It might make sense to clean the various *Config classes into one at
+    some point.
+    """
+
+    enabled: bool = True
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+
+
 @dataclass
 class MetricConfig(BaseConfig):
     enabled: bool = True
@@ -655,6 +676,7 @@ def finalize_and_validate(self):
 RESOURCE_TYPES: Dict[NodeType, Type[BaseConfig]] = {
     NodeType.Metric: MetricConfig,
     NodeType.SemanticModel: SemanticModelConfig,
+    NodeType.SavedQuery: SavedQueryConfig,
     NodeType.Exposure: ExposureConfig,
     NodeType.Source: SourceConfig,
     NodeType.Seed: SeedConfig,