From 0836095a572532fc22b1c0981346eebe56f7d1fb Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Wed, 31 Jan 2024 16:50:50 -0800 Subject: [PATCH] Move `SemanticModel` data artifacts to `dbt/artifacts` (#9485) * Move `SemanticModel` sub dataclasses to dbt/artifacts * Move `NodeRelation` to dbt/artifacts * Move `SemanticModelConfig` to dbt/artifacts * Move data portion of `SemanticModel` to dbt/artifacts * Add contextual comments to `semantic_model.py` about DSI protocols * Fixup mypy complaint * Migrate v12 manifest to use artifact definitions of `SavedQuery`, `Metric`, and `SemanticModel` * Convert `SemanticModel` and `Metric` resources to full nodes in selector search In the `search` method in `selector_methods.py`, we were getting object representations from the incoming writable manifest by unique id. What we get from the writable manifest though is increasingly the `resource` (data artifact) part of the node, not the full node. This was problematic because a number of the selector processes _compare_ the old node to the new node, but the `resource` representation doesn't have the comparator methods. In this commit we dict-ify the resource and then get the full node by undictifying that. We should probably have a better built in process to the full node objects to do this, but this will do for now. * Add `from_resource` implementation on `BaseNode` to ease resource to node conversion We want to easily be able to create nodes from their resource counter parts. It's actually imperative that we can do so. The previous commit had a manual way to do so where needed. However, we don't want to have to put `from_dict(.to_dict())` everywhere. So here we hadded a `from_resource` class method to `BaseNode`. Everything that inherits from `BaseNode` thus automatically gets this functionality. HOWEVER, the implementation currently has a problem. Specifically, the type for `resource_instance` is `BaseResource`. Which means if one is calling say `Metric.from_resource()`, one could hand it a `SemanticModelResource` and mypy won't complain. In this case, a semi-cryptic error might get raised at runtime. Whether or not an error gets raised depends entirely on whether or not the dictified resource instance manages to satisfy all the required attributes of the desired node class. THIS IS VERY BAD. We should be able to solve this issue in an upcoming (hopefully next) commit, wherein we genericize `BaseNode` such that when inheriting it you declare it with a resource type. Technically a runtime error will still be possible, however any mixups should be caught by mypy on pre-commit hooks as well as PRs. * Make `BaseNode` a generic that is defined with a `ResourceType` Turning `BaseNode` into an ABC generic allows us to say that the inheriting class can define what resource type from artifacts it should be used with. This gives us added type safety to what resource type can be passed into `from_resource` when called via `SemanticModel.from_resource(...)`, `Metric.from_resource(...)`, and etc. NOTE: This only gives us type safety from mypy. If we begin ignoring mypy errors during development, we can still get into a situation for runtime errors (it's just harder to do so now). --- .../Under the Hood-20240129-163800.yaml | 6 + core/dbt/artifacts/resources/__init__.py | 13 + .../artifacts/resources/v1/semantic_model.py | 273 ++++++++++++++++++ .../schemas/manifest/v12/manifest.py | 12 +- core/dbt/contracts/graph/model_config.py | 19 +- core/dbt/contracts/graph/nodes.py | 165 ++--------- core/dbt/contracts/graph/semantic_models.py | 127 -------- core/dbt/contracts/graph/unparsed.py | 2 +- core/dbt/graph/selector_methods.py | 4 +- core/dbt/parser/manifest.py | 3 +- core/dbt/parser/schema_yaml_readers.py | 12 +- .../test_semantic_model_configs.py | 3 +- tests/unit/test_contracts_graph_parsed.py | 4 +- tests/unit/test_graph_selector_methods.py | 2 +- ..._semantic_layer_nodes_satisfy_protocols.py | 20 +- tests/unit/test_semantic_models.py | 2 +- 16 files changed, 361 insertions(+), 306 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240129-163800.yaml create mode 100644 core/dbt/artifacts/resources/v1/semantic_model.py delete mode 100644 core/dbt/contracts/graph/semantic_models.py diff --git a/.changes/unreleased/Under the Hood-20240129-163800.yaml b/.changes/unreleased/Under the Hood-20240129-163800.yaml new file mode 100644 index 00000000000..0e724751aae --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240129-163800.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move data portion of `SemanticModel` to dbt/artifacts +time: 2024-01-29T16:38:00.245253-08:00 +custom: + Author: QMalcolm + Issue: "9387" diff --git a/core/dbt/artifacts/resources/__init__.py b/core/dbt/artifacts/resources/__init__.py index 6e22c65966a..80727abb4aa 100644 --- a/core/dbt/artifacts/resources/__init__.py +++ b/core/dbt/artifacts/resources/__init__.py @@ -31,3 +31,16 @@ WhereFilter, WhereFilterIntersection, ) +from dbt.artifacts.resources.v1.semantic_model import ( + Defaults, + Dimension, + DimensionTypeParams, + DimensionValidityParams, + Entity, + Measure, + MeasureAggregationParameters, + NodeRelation, + NonAdditiveDimension, + SemanticModel, + SemanticModelConfig, +) diff --git a/core/dbt/artifacts/resources/v1/semantic_model.py b/core/dbt/artifacts/resources/v1/semantic_model.py new file mode 100644 index 00000000000..b219b2bdcc8 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/semantic_model.py @@ -0,0 +1,273 @@ +import time + +from dataclasses import dataclass, field +from dbt.artifacts.resources.base import GraphResource +from dbt.artifacts.resources.v1.components import DependsOn, RefArgs +from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior +from dbt_common.dataclass_schema import dbtClassMixin +from dbt_semantic_interfaces.references import ( + DimensionReference, + EntityReference, + LinkableElementReference, + MeasureReference, + SemanticModelReference, + TimeDimensionReference, +) +from dbt_semantic_interfaces.type_enums import ( + AggregationType, + DimensionType, + EntityType, + TimeGranularity, +) +from dbt.artifacts.resources import SourceFileMetadata +from typing import Any, Dict, List, Optional, Sequence + + +""" +The classes in this file are dataclasses which are used to construct the Semantic +Model node in dbt-core. Additionally, these classes need to at a minimum support +what is specified in their protocol definitions in dbt-semantic-interfaces. +Their protocol definitions can be found here: +https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/semantic_model.py +""" + + +@dataclass +class Defaults(dbtClassMixin): + agg_time_dimension: Optional[str] = None + + +@dataclass +class NodeRelation(dbtClassMixin): + alias: str + schema_name: str # TODO: Could this be called simply "schema" so we could reuse StateRelation? + database: Optional[str] = None + relation_name: Optional[str] = None + + +# ==================================== +# Dimension objects +# Dimension protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/dimension.py +# ==================================== + + +@dataclass +class DimensionValidityParams(dbtClassMixin): + is_start: bool = False + is_end: bool = False + + +@dataclass +class DimensionTypeParams(dbtClassMixin): + time_granularity: TimeGranularity + validity_params: Optional[DimensionValidityParams] = None + + +@dataclass +class Dimension(dbtClassMixin): + name: str + type: DimensionType + description: Optional[str] = None + label: Optional[str] = None + is_partition: bool = False + type_params: Optional[DimensionTypeParams] = None + expr: Optional[str] = None + metadata: Optional[SourceFileMetadata] = None + + @property + def reference(self) -> DimensionReference: + return DimensionReference(element_name=self.name) + + @property + def time_dimension_reference(self) -> Optional[TimeDimensionReference]: + if self.type == DimensionType.TIME: + return TimeDimensionReference(element_name=self.name) + else: + return None + + @property + def validity_params(self) -> Optional[DimensionValidityParams]: + if self.type_params: + return self.type_params.validity_params + else: + return None + + +# ==================================== +# Entity objects +# Entity protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/entity.py +# ==================================== + + +@dataclass +class Entity(dbtClassMixin): + name: str + type: EntityType + description: Optional[str] = None + label: Optional[str] = None + role: Optional[str] = None + expr: Optional[str] = None + + @property + def reference(self) -> EntityReference: + return EntityReference(element_name=self.name) + + @property + def is_linkable_entity_type(self) -> bool: + return self.type in (EntityType.PRIMARY, EntityType.UNIQUE, EntityType.NATURAL) + + +# ==================================== +# Measure objects +# Measure protocols: https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/dbt_semantic_interfaces/protocols/measure.py +# ==================================== + + +@dataclass +class MeasureAggregationParameters(dbtClassMixin): + percentile: Optional[float] = None + use_discrete_percentile: bool = False + use_approximate_percentile: bool = False + + +@dataclass +class NonAdditiveDimension(dbtClassMixin): + name: str + window_choice: AggregationType + window_groupings: List[str] + + +@dataclass +class Measure(dbtClassMixin): + name: str + agg: AggregationType + description: Optional[str] = None + label: Optional[str] = None + create_metric: bool = False + expr: Optional[str] = None + agg_params: Optional[MeasureAggregationParameters] = None + non_additive_dimension: Optional[NonAdditiveDimension] = None + agg_time_dimension: Optional[str] = None + + @property + def reference(self) -> MeasureReference: + return MeasureReference(element_name=self.name) + + +# ==================================== +# SemanticModel final parts +# ==================================== + + +@dataclass +class SemanticModelConfig(BaseConfig): + enabled: bool = True + group: Optional[str] = field( + default=None, + metadata=CompareBehavior.Exclude.meta(), + ) + meta: Dict[str, Any] = field( + default_factory=dict, + metadata=MergeBehavior.Update.meta(), + ) + + +@dataclass +class SemanticModel(GraphResource): + model: str + node_relation: Optional[NodeRelation] + description: Optional[str] = None + label: Optional[str] = None + defaults: Optional[Defaults] = None + entities: Sequence[Entity] = field(default_factory=list) + measures: Sequence[Measure] = field(default_factory=list) + dimensions: Sequence[Dimension] = field(default_factory=list) + metadata: Optional[SourceFileMetadata] = None + depends_on: DependsOn = field(default_factory=DependsOn) + refs: List[RefArgs] = field(default_factory=list) + created_at: float = field(default_factory=lambda: time.time()) + config: SemanticModelConfig = field(default_factory=SemanticModelConfig) + unrendered_config: Dict[str, Any] = field(default_factory=dict) + primary_entity: Optional[str] = None + group: Optional[str] = None + + @property + def entity_references(self) -> List[LinkableElementReference]: + return [entity.reference for entity in self.entities] + + @property + def dimension_references(self) -> List[LinkableElementReference]: + return [dimension.reference for dimension in self.dimensions] + + @property + def measure_references(self) -> List[MeasureReference]: + return [measure.reference for measure in self.measures] + + @property + def has_validity_dimensions(self) -> bool: + return any([dim.validity_params is not None for dim in self.dimensions]) + + @property + def validity_start_dimension(self) -> Optional[Dimension]: + validity_start_dims = [ + dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_start + ] + if not validity_start_dims: + return None + return validity_start_dims[0] + + @property + def validity_end_dimension(self) -> Optional[Dimension]: + validity_end_dims = [ + dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_end + ] + if not validity_end_dims: + return None + return validity_end_dims[0] + + @property + def partitions(self) -> List[Dimension]: # noqa: D + return [dim for dim in self.dimensions or [] if dim.is_partition] + + @property + def partition(self) -> Optional[Dimension]: + partitions = self.partitions + if not partitions: + return None + return partitions[0] + + @property + def reference(self) -> SemanticModelReference: + return SemanticModelReference(semantic_model_name=self.name) + + def checked_agg_time_dimension_for_measure( + self, measure_reference: MeasureReference + ) -> TimeDimensionReference: + measure: Optional[Measure] = None + for measure in self.measures: + if measure.reference == measure_reference: + measure = measure + + assert ( + measure is not None + ), f"No measure with name ({measure_reference.element_name}) in semantic_model with name ({self.name})" + + default_agg_time_dimension = ( + self.defaults.agg_time_dimension if self.defaults is not None else None + ) + + agg_time_dimension_name = measure.agg_time_dimension or default_agg_time_dimension + assert agg_time_dimension_name is not None, ( + f"Aggregation time dimension for measure {measure.name} on semantic model {self.name} is not set! " + "To fix this either specify a default `agg_time_dimension` for the semantic model or define an " + "`agg_time_dimension` on the measure directly." + ) + return TimeDimensionReference(element_name=agg_time_dimension_name) + + @property + def primary_entity_reference(self) -> Optional[EntityReference]: + return ( + EntityReference(element_name=self.primary_entity) + if self.primary_entity is not None + else None + ) diff --git a/core/dbt/artifacts/schemas/manifest/v12/manifest.py b/core/dbt/artifacts/schemas/manifest/v12/manifest.py index 3da46c47fe1..d4fbccbacbd 100644 --- a/core/dbt/artifacts/schemas/manifest/v12/manifest.py +++ b/core/dbt/artifacts/schemas/manifest/v12/manifest.py @@ -9,7 +9,14 @@ get_artifact_schema_version, ) from dbt.artifacts.schemas.upgrades import upgrade_manifest_json -from dbt.artifacts.resources import Documentation, Group, Macro +from dbt.artifacts.resources import ( + Documentation, + Group, + Macro, + Metric, + SavedQuery, + SemanticModel, +) # TODO: remove usage of dbt modules other than dbt.artifacts from dbt import tracking @@ -18,9 +25,6 @@ Exposure, GraphMemberNode, ManifestNode, - Metric, - SavedQuery, - SemanticModel, SourceDefinition, UnitTestDefinition, ) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index 15b721cfe9b..a2fc7801d8d 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -2,7 +2,11 @@ from typing import Any, List, Optional, Dict, Union, Type from typing_extensions import Annotated -from dbt.artifacts.resources import MetricConfig, SavedQueryConfig +from dbt.artifacts.resources import ( + MetricConfig, + SavedQueryConfig, + SemanticModelConfig, +) from dbt_common.contracts.config.base import BaseConfig, MergeBehavior, CompareBehavior from dbt_common.contracts.config.materialization import OnConfigurationChangeOption from dbt_common.contracts.config.metadata import Metadata, ShowBehavior @@ -49,19 +53,6 @@ class Hook(dbtClassMixin, Replaceable): index: Optional[int] = None -@dataclass -class SemanticModelConfig(BaseConfig): - enabled: bool = True - group: Optional[str] = field( - default=None, - metadata=CompareBehavior.Exclude.meta(), - ) - meta: Dict[str, Any] = field( - default_factory=dict, - metadata=MergeBehavior.Update.meta(), - ) - - @dataclass class ExposureConfig(BaseConfig): enabled: bool = True diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 96b95f885a4..28d8bd34142 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -4,8 +4,21 @@ from dataclasses import dataclass, field import hashlib +from abc import ABC from mashumaro.types import SerializableType -from typing import Optional, Union, List, Dict, Any, Sequence, Tuple, Iterator, Literal +from typing import ( + Optional, + Union, + List, + Dict, + Any, + Sequence, + Tuple, + Iterator, + Literal, + Generic, + TypeVar, +) from dbt import deprecations from dbt_common.contracts.constraints import ( @@ -17,12 +30,6 @@ from dbt_common.clients.system import write_file from dbt.contracts.files import FileHash -from dbt.contracts.graph.semantic_models import ( - Defaults, - Dimension, - Entity, - Measure, -) from dbt.contracts.graph.unparsed import ( ExposureType, ExternalTable, @@ -60,13 +67,6 @@ REFABLE_NODE_TYPES, VERSIONED_NODE_TYPES, ) -from dbt_semantic_interfaces.references import ( - EntityReference, - MeasureReference, - LinkableElementReference, - SemanticModelReference, - TimeDimensionReference, -) from .model_config import ( NodeConfig, @@ -77,7 +77,6 @@ ExposureConfig, EmptySnapshotConfig, SnapshotConfig, - SemanticModelConfig, UnitTestConfig, UnitTestNodeConfig, ) @@ -96,10 +95,9 @@ GraphResource, RefArgs as RefArgsResource, SavedQuery as SavedQueryResource, - SourceFileMetadata as SourceFileMetadataResource, + SemanticModel as SemanticModelResource, ) - # ===================================================================== # This contains the classes for all of the nodes and node-like objects # in the manifest. In the "nodes" dictionary of the manifest we find @@ -123,8 +121,11 @@ # ================================================== +ResourceTypeT = TypeVar("ResourceTypeT", bound="BaseResource") + + @dataclass -class BaseNode(BaseResource): +class BaseNode(ABC, Generic[ResourceTypeT], BaseResource): """All nodes or node-like objects in this file should have this as a base class""" @property @@ -163,9 +164,13 @@ def is_ephemeral_model(self): def get_materialization(self): return self.config.materialized + @classmethod + def from_resource(cls, resource_instance: ResourceTypeT): + return cls.from_dict(resource_instance.to_dict()) + @dataclass -class GraphNode(GraphResource, BaseNode): +class GraphNode(GraphResource, BaseNode[ResourceTypeT], Generic[ResourceTypeT]): """Nodes in the DAG. Macro and Documentation don't have fqn.""" def same_fqn(self, other) -> bool: @@ -228,7 +233,7 @@ def identifier(self): @dataclass -class ParsedNodeMandatory(GraphNode, HasRelationMetadata, Replaceable): +class ParsedNodeMandatory(GraphNode[GraphResource], HasRelationMetadata, Replaceable): alias: str checksum: FileHash config: NodeConfig = field(default_factory=NodeConfig) @@ -1023,7 +1028,7 @@ class UnitTestDefinitionMandatory: @dataclass -class UnitTestDefinition(NodeInfoMixin, GraphNode, UnitTestDefinitionMandatory): +class UnitTestDefinition(NodeInfoMixin, GraphNode[GraphResource], UnitTestDefinitionMandatory): description: str = "" overrides: Optional[UnitTestOverrides] = None depends_on: DependsOn = field(default_factory=DependsOn) @@ -1240,7 +1245,7 @@ def tests(self) -> List[TestDef]: @dataclass -class ParsedSourceMandatory(GraphNode, HasRelationMetadata): +class ParsedSourceMandatory(GraphNode[GraphResource], HasRelationMetadata): source_name: str source_description: str loader: str @@ -1373,7 +1378,7 @@ def group(self): @dataclass -class Exposure(GraphNode): +class Exposure(GraphNode[GraphResource]): type: ExposureType owner: Owner resource_type: Literal[NodeType.Exposure] @@ -1456,7 +1461,7 @@ def group(self): @dataclass -class Metric(GraphNode, MetricResource): +class Metric(GraphNode[MetricResource], MetricResource): @property def depends_on_nodes(self): return self.depends_on.nodes @@ -1518,86 +1523,12 @@ class Group(GroupResource, BaseNode): # ==================================== -# SemanticModel and related classes +# SemanticModel node # ==================================== @dataclass -class NodeRelation(dbtClassMixin): - alias: str - schema_name: str # TODO: Could this be called simply "schema" so we could reuse StateRelation? - database: Optional[str] = None - relation_name: Optional[str] = None - - -@dataclass -class SemanticModel(GraphNode): - model: str - node_relation: Optional[NodeRelation] - description: Optional[str] = None - label: Optional[str] = None - defaults: Optional[Defaults] = None - entities: Sequence[Entity] = field(default_factory=list) - measures: Sequence[Measure] = field(default_factory=list) - dimensions: Sequence[Dimension] = field(default_factory=list) - metadata: Optional[SourceFileMetadataResource] = None - depends_on: DependsOn = field(default_factory=DependsOn) - refs: List[RefArgsResource] = field(default_factory=list) - created_at: float = field(default_factory=lambda: time.time()) - config: SemanticModelConfig = field(default_factory=SemanticModelConfig) - unrendered_config: Dict[str, Any] = field(default_factory=dict) - primary_entity: Optional[str] = None - group: Optional[str] = None - - @property - def entity_references(self) -> List[LinkableElementReference]: - return [entity.reference for entity in self.entities] - - @property - def dimension_references(self) -> List[LinkableElementReference]: - return [dimension.reference for dimension in self.dimensions] - - @property - def measure_references(self) -> List[MeasureReference]: - return [measure.reference for measure in self.measures] - - @property - def has_validity_dimensions(self) -> bool: - return any([dim.validity_params is not None for dim in self.dimensions]) - - @property - def validity_start_dimension(self) -> Optional[Dimension]: - validity_start_dims = [ - dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_start - ] - if not validity_start_dims: - return None - return validity_start_dims[0] - - @property - def validity_end_dimension(self) -> Optional[Dimension]: - validity_end_dims = [ - dim for dim in self.dimensions if dim.validity_params and dim.validity_params.is_end - ] - if not validity_end_dims: - return None - return validity_end_dims[0] - - @property - def partitions(self) -> List[Dimension]: # noqa: D - return [dim for dim in self.dimensions or [] if dim.is_partition] - - @property - def partition(self) -> Optional[Dimension]: - partitions = self.partitions - if not partitions: - return None - return partitions[0] - - @property - def reference(self) -> SemanticModelReference: - return SemanticModelReference(semantic_model_name=self.name) - +class SemanticModel(GraphNode[SemanticModelResource], SemanticModelResource): @property def depends_on_nodes(self): return self.depends_on.nodes @@ -1606,38 +1537,6 @@ def depends_on_nodes(self): def depends_on_macros(self): return self.depends_on.macros - def checked_agg_time_dimension_for_measure( - self, measure_reference: MeasureReference - ) -> TimeDimensionReference: - measure: Optional[Measure] = None - for measure in self.measures: - if measure.reference == measure_reference: - measure = measure - - assert ( - measure is not None - ), f"No measure with name ({measure_reference.element_name}) in semantic_model with name ({self.name})" - - default_agg_time_dimension = ( - self.defaults.agg_time_dimension if self.defaults is not None else None - ) - - agg_time_dimension_name = measure.agg_time_dimension or default_agg_time_dimension - assert agg_time_dimension_name is not None, ( - f"Aggregation time dimension for measure {measure.name} on semantic model {self.name} is not set! " - "To fix this either specify a default `agg_time_dimension` for the semantic model or define an " - "`agg_time_dimension` on the measure directly." - ) - return TimeDimensionReference(element_name=agg_time_dimension_name) - - @property - def primary_entity_reference(self) -> Optional[EntityReference]: - return ( - EntityReference(element_name=self.primary_entity) - if self.primary_entity is not None - else None - ) - def same_model(self, old: "SemanticModel") -> bool: return self.model == old.same_model @@ -1695,7 +1594,7 @@ def same_contents(self, old: Optional["SemanticModel"]) -> bool: @dataclass -class SavedQuery(NodeInfoMixin, GraphNode, SavedQueryResource): +class SavedQuery(NodeInfoMixin, GraphNode[SavedQueryResource], SavedQueryResource): def same_metrics(self, old: "SavedQuery") -> bool: return self.query_params.metrics == old.query_params.metrics diff --git a/core/dbt/contracts/graph/semantic_models.py b/core/dbt/contracts/graph/semantic_models.py deleted file mode 100644 index 53394d02f80..00000000000 --- a/core/dbt/contracts/graph/semantic_models.py +++ /dev/null @@ -1,127 +0,0 @@ -from dataclasses import dataclass -from dbt_common.dataclass_schema import dbtClassMixin -from dbt_semantic_interfaces.references import ( - DimensionReference, - EntityReference, - MeasureReference, - TimeDimensionReference, -) -from dbt_semantic_interfaces.type_enums import ( - AggregationType, - DimensionType, - EntityType, - TimeGranularity, -) -from dbt.artifacts.resources import SourceFileMetadata -from typing import List, Optional - - -@dataclass -class Defaults(dbtClassMixin): - agg_time_dimension: Optional[str] = None - - -# ==================================== -# Dimension objects -# ==================================== - - -@dataclass -class DimensionValidityParams(dbtClassMixin): - is_start: bool = False - is_end: bool = False - - -@dataclass -class DimensionTypeParams(dbtClassMixin): - time_granularity: TimeGranularity - validity_params: Optional[DimensionValidityParams] = None - - -@dataclass -class Dimension(dbtClassMixin): - name: str - type: DimensionType - description: Optional[str] = None - label: Optional[str] = None - is_partition: bool = False - type_params: Optional[DimensionTypeParams] = None - expr: Optional[str] = None - metadata: Optional[SourceFileMetadata] = None - - @property - def reference(self) -> DimensionReference: - return DimensionReference(element_name=self.name) - - @property - def time_dimension_reference(self) -> Optional[TimeDimensionReference]: - if self.type == DimensionType.TIME: - return TimeDimensionReference(element_name=self.name) - else: - return None - - @property - def validity_params(self) -> Optional[DimensionValidityParams]: - if self.type_params: - return self.type_params.validity_params - else: - return None - - -# ==================================== -# Entity objects -# ==================================== - - -@dataclass -class Entity(dbtClassMixin): - name: str - type: EntityType - description: Optional[str] = None - label: Optional[str] = None - role: Optional[str] = None - expr: Optional[str] = None - - @property - def reference(self) -> EntityReference: - return EntityReference(element_name=self.name) - - @property - def is_linkable_entity_type(self) -> bool: - return self.type in (EntityType.PRIMARY, EntityType.UNIQUE, EntityType.NATURAL) - - -# ==================================== -# Measure objects -# ==================================== - - -@dataclass -class MeasureAggregationParameters(dbtClassMixin): - percentile: Optional[float] = None - use_discrete_percentile: bool = False - use_approximate_percentile: bool = False - - -@dataclass -class NonAdditiveDimension(dbtClassMixin): - name: str - window_choice: AggregationType - window_groupings: List[str] - - -@dataclass -class Measure(dbtClassMixin): - name: str - agg: AggregationType - description: Optional[str] = None - label: Optional[str] = None - create_metric: bool = False - expr: Optional[str] = None - agg_params: Optional[MeasureAggregationParameters] = None - non_additive_dimension: Optional[NonAdditiveDimension] = None - agg_time_dimension: Optional[str] = None - - @property - def reference(self) -> MeasureReference: - return MeasureReference(element_name=self.name) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index a1475a50178..852c747dedc 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -15,7 +15,7 @@ ValidationError, ) from dbt.node_types import NodeType -from dbt.contracts.graph.semantic_models import ( +from dbt.artifacts.resources import ( Defaults, DimensionValidityParams, MeasureAggregationParameters, diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index e9a627b55b2..cd2c3af934f 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -737,9 +737,9 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu elif unique_id in manifest.exposures: previous_node = manifest.exposures[unique_id] elif unique_id in manifest.metrics: - previous_node = manifest.metrics[unique_id] + previous_node = Metric.from_resource(manifest.metrics[unique_id]) elif unique_id in manifest.semantic_models: - previous_node = manifest.semantic_models[unique_id] + previous_node = SemanticModel.from_resource(manifest.semantic_models[unique_id]) elif unique_id in manifest.unit_tests: previous_node = manifest.unit_tests[unique_id] diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 644009a06cb..66cccfbcac0 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -106,9 +106,8 @@ ManifestNode, ResultNode, ModelNode, - NodeRelation, ) -from dbt.artifacts.resources import NodeVersion +from dbt.artifacts.resources import NodeRelation, NodeVersion from dbt.artifacts.schemas.base import Writable from dbt.exceptions import ( TargetNotFoundError, diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 654ae5269d7..2b64312bf87 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -28,25 +28,23 @@ ) from dbt.artifacts.resources import ( ConversionTypeParams, + Dimension, + DimensionTypeParams, + Entity, Export, ExportConfig, + Measure, MetricConfig, MetricInput, MetricInputMeasure, MetricTimeWindow, MetricTypeParams, + NonAdditiveDimension, QueryParams, SavedQueryConfig, WhereFilter, WhereFilterIntersection, ) -from dbt.contracts.graph.semantic_models import ( - Dimension, - DimensionTypeParams, - Entity, - Measure, - NonAdditiveDimension, -) from dbt_common.exceptions import DbtInternalError from dbt.exceptions import YamlParseDictError, JSONValidationError from dbt.context.providers import generate_parse_exposure, generate_parse_semantic_models diff --git a/tests/functional/semantic_models/test_semantic_model_configs.py b/tests/functional/semantic_models/test_semantic_model_configs.py index bd74ad95edd..407fb2c3d4d 100644 --- a/tests/functional/semantic_models/test_semantic_model_configs.py +++ b/tests/functional/semantic_models/test_semantic_model_configs.py @@ -1,7 +1,6 @@ import pytest +from dbt.artifacts.resources import SemanticModelConfig from dbt.exceptions import ParsingError -from dbt.contracts.graph.model_config import SemanticModelConfig - from dbt.tests.util import run_dbt, update_config_file, get_manifest from tests.functional.semantic_models.fixtures import ( diff --git a/tests/unit/test_contracts_graph_parsed.py b/tests/unit/test_contracts_graph_parsed.py index b7d01185da6..f3cf839731f 100644 --- a/tests/unit/test_contracts_graph_parsed.py +++ b/tests/unit/test_contracts_graph_parsed.py @@ -5,6 +5,9 @@ from hypothesis.strategies import builds, lists from dbt.artifacts.resources import ( + Dimension, + Entity, + Measure, MetricInputMeasure, MetricTypeParams, RefArgs, @@ -42,7 +45,6 @@ TestMetadata, SemanticModel, ) -from dbt.contracts.graph.semantic_models import Dimension, Entity, Measure from dbt.contracts.graph.unparsed import ( ExposureType, FreshnessThreshold, diff --git a/tests/unit/test_graph_selector_methods.py b/tests/unit/test_graph_selector_methods.py index ba7ff1c0c45..2b06a1dbdb8 100644 --- a/tests/unit/test_graph_selector_methods.py +++ b/tests/unit/test_graph_selector_methods.py @@ -15,7 +15,6 @@ Exposure, Metric, Group, - NodeRelation, SavedQuery, SeedNode, SemanticModel, @@ -32,6 +31,7 @@ from dbt.artifacts.resources import ( MetricInputMeasure, MetricTypeParams, + NodeRelation, QueryParams, ) from dbt.contracts.graph.unparsed import ( diff --git a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py index d793b9285fb..e2765499355 100644 --- a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py +++ b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py @@ -3,30 +3,28 @@ from dbt.contracts.graph.nodes import ( Metric, - NodeRelation, SavedQuery, SemanticModel, ) from dbt.artifacts.resources import ( ConstantPropertyInput, ConversionTypeParams, - FileSlice, - MetricInput, - MetricInputMeasure, - MetricTimeWindow, - MetricTypeParams, - SourceFileMetadata, - WhereFilter, -) -from dbt.contracts.graph.semantic_models import ( + Defaults, Dimension, DimensionTypeParams, DimensionValidityParams, - Defaults, Entity, + FileSlice, Measure, MeasureAggregationParameters, + MetricInput, + MetricInputMeasure, + MetricTimeWindow, + MetricTypeParams, + NodeRelation, NonAdditiveDimension, + SourceFileMetadata, + WhereFilter, ) from dbt.node_types import NodeType from dbt_semantic_interfaces.protocols import ( diff --git a/tests/unit/test_semantic_models.py b/tests/unit/test_semantic_models.py index b1052eb2150..154c57d8585 100644 --- a/tests/unit/test_semantic_models.py +++ b/tests/unit/test_semantic_models.py @@ -2,8 +2,8 @@ from typing import List +from dbt.artifacts.resources import Dimension, Entity, Measure, Defaults from dbt.contracts.graph.nodes import SemanticModel -from dbt.contracts.graph.semantic_models import Dimension, Entity, Measure, Defaults from dbt.node_types import NodeType from dbt_semantic_interfaces.references import MeasureReference from dbt_semantic_interfaces.type_enums import AggregationType, DimensionType, EntityType