diff --git a/.changes/unreleased/Features-20240426-144119.yaml b/.changes/unreleased/Features-20240426-144119.yaml new file mode 100644 index 0000000000..c3b6afa637 --- /dev/null +++ b/.changes/unreleased/Features-20240426-144119.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add a Dependency Resolver for Saved Queries +time: 2024-04-26T14:41:19.27946-07:00 +custom: + Author: plypaul + Issue: "1155" diff --git a/metricflow-semantics/metricflow_semantics/api/__init__.py b/metricflow-semantics/metricflow_semantics/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/metricflow_semantics/api/v0_1/__init__.py b/metricflow-semantics/metricflow_semantics/api/v0_1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/metricflow_semantics/api/v0_1/saved_query_dependency_resolver.py b/metricflow-semantics/metricflow_semantics/api/v0_1/saved_query_dependency_resolver.py new file mode 100644 index 0000000000..50893504cf --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/api/v0_1/saved_query_dependency_resolver.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Tuple + +from dbt_semantic_interfaces.protocols import SemanticManifest +from dbt_semantic_interfaces.references import ( + SemanticModelReference, +) + +from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup +from metricflow_semantics.query.query_parser import MetricFlowQueryParser +from metricflow_semantics.specs.query_param_implementations import SavedQueryParameter + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class SavedQueryDependencySet: + """The dependencies of a saved query. + + The primary use case is to handle creation of the cache item associated with the saved query. The dependencies + listed in this class must be up-to-date before the cache associated with the saved query can be created. Otherwise, + running the export / creating the cache may create a cache item that is out-of-date / unusable. + """ + + # The semantic models that the saved query depends on. + semantic_model_references: Tuple[SemanticModelReference, ...] + + +class SavedQueryDependencyResolver: + """Resolves the dependencies of a saved query. Also see `SavedQueryDependencySet`.""" + + def __init__(self, semantic_manifest: SemanticManifest) -> None: # noqa: D107 + self._semantic_manifest = semantic_manifest + self._query_parser = MetricFlowQueryParser(SemanticManifestLookup(semantic_manifest)) + + def _resolve_dependencies(self, saved_query_name: str) -> SavedQueryDependencySet: + parse_result = self._query_parser.parse_and_validate_saved_query( + saved_query_parameter=SavedQueryParameter(saved_query_name), + where_filter=None, + limit=None, + time_constraint_start=None, + time_constraint_end=None, + order_by_names=None, + order_by_parameters=None, + ) + + return SavedQueryDependencySet( + semantic_model_references=tuple( + sorted( + parse_result.queried_semantic_models, + key=lambda reference: reference.semantic_model_name, + ) + ), + ) + + def resolve_dependencies(self, saved_query_name: str) -> SavedQueryDependencySet: + """Return the dependencies of the given saved query in the manifest.""" + try: + return self._resolve_dependencies(saved_query_name) + except Exception: + logger.exception( + f"Got an exception while getting the dependencies of saved-query {repr(saved_query_name)}. " + f"All semantic models will be returned instead for safety." + ) + return SavedQueryDependencySet( + semantic_model_references=tuple( + sorted( + (semantic_model.reference for semantic_model in self._semantic_manifest.semantic_models), + key=lambda reference: reference.semantic_model_name, + ) + ), + ) diff --git a/metricflow-semantics/metricflow_semantics/collection_helpers/dedupe.py b/metricflow-semantics/metricflow_semantics/collection_helpers/dedupe.py new file mode 100644 index 0000000000..1cc5b46091 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/collection_helpers/dedupe.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from typing import Dict, Iterable, Tuple, TypeVar + +IterableT = TypeVar("IterableT") + + +def ordered_dedupe(*iterables: Iterable[IterableT]) -> Tuple[IterableT, ...]: + """De-duplicates the items in the iterables while preserving the order.""" + ordered_results: Dict[IterableT, None] = {} + for iterable in iterables: + for item in iterable: + ordered_results[item] = None + + return tuple(ordered_results.keys()) diff --git a/metricflow-semantics/metricflow_semantics/instances.py b/metricflow-semantics/metricflow_semantics/instances.py index 2e6c4499c3..1ee3cd90f3 100644 --- a/metricflow-semantics/metricflow_semantics/instances.py +++ b/metricflow-semantics/metricflow_semantics/instances.py @@ -16,12 +16,12 @@ EntitySpec, GroupByMetricSpec, InstanceSpec, - InstanceSpecSet, MeasureSpec, MetadataSpec, MetricSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet # Type for the specification used in the instance. SpecT = TypeVar("SpecT", bound=InstanceSpec) diff --git a/metricflow-semantics/metricflow_semantics/model/linkable_element_property.py b/metricflow-semantics/metricflow_semantics/model/linkable_element_property.py new file mode 100644 index 0000000000..668be83974 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/model/linkable_element_property.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from enum import Enum +from typing import FrozenSet + + +class LinkableElementProperty(Enum): + """The properties associated with a valid linkable element. + + Local means an element that is defined within the same semantic model as the measure. This definition is used + throughout the related classes. + """ + + # A local element as per above definition. + LOCAL = "local" + # A local dimension that is prefixed with a local primary entity. + LOCAL_LINKED = "local_linked" + # An element that was joined to the measure semantic model by an entity. + JOINED = "joined" + # An element that was joined to the measure semantic model by joining multiple semantic models. + MULTI_HOP = "multi_hop" + # A time dimension that is a version of a time dimension in a semantic model, but at a different granularity. + DERIVED_TIME_GRANULARITY = "derived_time_granularity" + # Refers to an entity, not a dimension. + ENTITY = "entity" + # See metric_time in DataSet + METRIC_TIME = "metric_time" + # Refers to a metric, not a dimension. + METRIC = "metric" + + @staticmethod + def all_properties() -> FrozenSet[LinkableElementProperty]: # noqa: D102 + return frozenset( + { + LinkableElementProperty.LOCAL, + LinkableElementProperty.LOCAL_LINKED, + LinkableElementProperty.JOINED, + LinkableElementProperty.MULTI_HOP, + LinkableElementProperty.DERIVED_TIME_GRANULARITY, + LinkableElementProperty.METRIC_TIME, + LinkableElementProperty.METRIC, + } + ) diff --git a/metricflow-semantics/metricflow_semantics/model/semantic_model_derivation.py b/metricflow-semantics/metricflow_semantics/model/semantic_model_derivation.py new file mode 100644 index 0000000000..ba541b68ba --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/model/semantic_model_derivation.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Sequence + +from dbt_semantic_interfaces.references import SemanticModelReference + + +class SemanticModelDerivation(ABC): + """Interface for an object that can be described as derived from a semantic model.""" + + @property + @abstractmethod + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + """The semantic models that this was derived from. + + The returned sequence should be ordered and not contain duplicates. + """ + raise NotImplementedError diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py index 7167a784b1..58a65f35a0 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py @@ -1,16 +1,34 @@ from __future__ import annotations +import logging +from abc import ABC from dataclasses import dataclass from enum import Enum -from typing import FrozenSet, Optional, Tuple +from typing import FrozenSet, Optional, Sequence, Tuple from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from dbt_semantic_interfaces.protocols.dimension import DimensionType -from dbt_semantic_interfaces.references import DimensionReference, MetricReference, SemanticModelReference +from dbt_semantic_interfaces.references import ( + DimensionReference, + EntityReference, + MetricReference, + SemanticModelReference, +) from dbt_semantic_interfaces.type_enums.date_part import DatePart from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from typing_extensions import override -from metricflow_semantics.specs.spec_classes import EntityReference +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + GroupByMetricSpec, + LinkableInstanceSpec, + TimeDimensionSpec, +) + +logger = logging.getLogger(__name__) class LinkableElementType(Enum): @@ -40,45 +58,6 @@ def is_dimension_type(self) -> bool: return assert_values_exhausted(element_type) -class LinkableElementProperty(Enum): - """The properties associated with a valid linkable element. - - Local means an element that is defined within the same semantic model as the measure. This definition is used - throughout the related classes. - """ - - # A local element as per above definition. - LOCAL = "local" - # A local dimension that is prefixed with a local primary entity. - LOCAL_LINKED = "local_linked" - # An element that was joined to the measure semantic model by an entity. - JOINED = "joined" - # An element that was joined to the measure semantic model by joining multiple semantic models. - MULTI_HOP = "multi_hop" - # A time dimension that is a version of a time dimension in a semantic model, but at a different granularity. - DERIVED_TIME_GRANULARITY = "derived_time_granularity" - # Refers to an entity, not a dimension. - ENTITY = "entity" - # See metric_time in DataSet - METRIC_TIME = "metric_time" - # Refers to a metric, not a dimension. - METRIC = "metric" - - @staticmethod - def all_properties() -> FrozenSet[LinkableElementProperty]: # noqa: D102 - return frozenset( - { - LinkableElementProperty.LOCAL, - LinkableElementProperty.LOCAL_LINKED, - LinkableElementProperty.JOINED, - LinkableElementProperty.MULTI_HOP, - LinkableElementProperty.DERIVED_TIME_GRANULARITY, - LinkableElementProperty.METRIC_TIME, - LinkableElementProperty.METRIC, - } - ) - - @dataclass(frozen=True) class ElementPathKey: """A key that can uniquely identify an element and the joins used to realize the element.""" @@ -101,6 +80,37 @@ def __post_init__(self) -> None: else: assert_values_exhausted(element_type) + @property + def spec(self) -> LinkableInstanceSpec: + """The corresponding spec object for this path key.""" + if self.element_type is LinkableElementType.DIMENSION: + return DimensionSpec( + element_name=self.element_name, + entity_links=self.entity_links, + ) + elif self.element_type is LinkableElementType.TIME_DIMENSION: + assert ( + self.time_granularity is not None + ), f"{self.time_granularity=} should not be None as per check in dataclass validation" + return TimeDimensionSpec( + element_name=self.element_name, + entity_links=self.entity_links, + time_granularity=self.time_granularity, + date_part=self.date_part, + ) + elif self.element_type is LinkableElementType.ENTITY: + return EntitySpec( + element_name=self.element_name, + entity_links=self.entity_links, + ) + elif self.element_type is LinkableElementType.METRIC: + return GroupByMetricSpec( + element_name=self.element_name, + entity_links=self.entity_links, + ) + else: + assert_values_exhausted(self.element_type) + @dataclass(frozen=True) class SemanticModelJoinPathElement: @@ -110,8 +120,14 @@ class SemanticModelJoinPathElement: join_on_entity: EntityReference +class LinkableElement(SemanticModelDerivation, ABC): + """An entity / dimension that may have been joined by entities.""" + + pass + + @dataclass(frozen=True) -class LinkableDimension: +class LinkableDimension(LinkableElement): """Describes how a dimension can be realized by joining based on entity links.""" # The semantic model where this dimension was defined. @@ -143,9 +159,20 @@ def path_key(self) -> ElementPathKey: # noqa: D102 def reference(self) -> DimensionReference: # noqa: D102 return DimensionReference(element_name=self.element_name) + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references = set() + if self.semantic_model_origin: + semantic_model_references.add(self.semantic_model_origin) + for join_path_item in self.join_path: + semantic_model_references.add(join_path_item.semantic_model_reference) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + @dataclass(frozen=True) -class LinkableEntity: +class LinkableEntity(LinkableElement): """Describes how an entity can be realized by joining based on entity links.""" # The semantic model where this entity was defined. @@ -165,9 +192,18 @@ def path_key(self) -> ElementPathKey: # noqa: D102 def reference(self) -> EntityReference: # noqa: D102 return EntityReference(element_name=self.element_name) + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references = {self.semantic_model_origin} + for join_path_item in self.join_path: + semantic_model_references.add(join_path_item.semantic_model_reference) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + @dataclass(frozen=True) -class LinkableMetric: +class LinkableMetric(LinkableElement): """Describes how a metric can be realized by joining based on entity links.""" element_name: str @@ -187,6 +223,15 @@ def path_key(self) -> ElementPathKey: # noqa: D102 def reference(self) -> MetricReference: # noqa: D102 return MetricReference(element_name=self.element_name) + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references = {self.join_by_semantic_model} + for join_path_item in self.join_path: + semantic_model_references.add(join_path_item.semantic_model_reference) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + @dataclass(frozen=True) class SemanticModelJoinPath: diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py index 3f3d6d09e5..54b76fd147 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py @@ -4,25 +4,27 @@ from dataclasses import dataclass, field from typing import Dict, FrozenSet, List, Sequence, Set, Tuple +from dbt_semantic_interfaces.references import SemanticModelReference +from typing_extensions import override + +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation from metricflow_semantics.model.semantics.linkable_element import ( ElementPathKey, LinkableDimension, - LinkableElementProperty, LinkableElementType, LinkableEntity, LinkableMetric, ) +from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( - DimensionSpec, - EntitySpec, - GroupByMetricSpec, - LinkableSpecSet, - TimeDimensionSpec, + InstanceSpec, + LinkableInstanceSpec, ) @dataclass(frozen=True) -class LinkableElementSet: +class LinkableElementSet(SemanticModelDerivation): """Container class for storing all linkable elements for a metric. TODO: There are similarities with LinkableSpecSet - consider consolidation. @@ -61,6 +63,33 @@ def __post_init__(self) -> None: f"type! Mismatched elements: {mismatched_elements}" ) + # There shouldn't be a path key without any concrete items. Can be an issue as specs contained in this set are + # generated from the path keys. + for key, value in ( + tuple(self.path_key_to_linkable_dimensions.items()) + + tuple(self.path_key_to_linkable_entities.items()) + + tuple(self.path_key_to_linkable_metrics.items()) + ): + assert len(value) > 0, f"{key} is empty" + + # There shouldn't be any duplicate specs. + specs = self.specs + deduped_specs = set(specs) + assert len(deduped_specs) == len(specs) + assert len(deduped_specs) == ( + len(self.path_key_to_linkable_dimensions) + + len(self.path_key_to_linkable_entities) + + len(self.path_key_to_linkable_metrics) + ) + + # Check time dimensions have the grain set. + for path_key, linkable_dimensions in self.path_key_to_linkable_dimensions.items(): + if path_key.element_type is LinkableElementType.TIME_DIMENSION: + for linkable_dimension in linkable_dimensions: + assert ( + linkable_dimension.time_granularity is not None + ), f"{path_key} has a dimension without the time granularity set: {linkable_dimension}" + @staticmethod def merge_by_path_key(linkable_element_sets: Sequence[LinkableElementSet]) -> LinkableElementSet: """Combine multiple sets together by the path key. @@ -108,6 +137,8 @@ def intersection_by_path_key(linkable_element_sets: Sequence[LinkableElementSet] """ if len(linkable_element_sets) == 0: return LinkableElementSet() + elif len(linkable_element_sets) == 1: + return linkable_element_sets[0] # Find path keys that are common to all LinkableElementSets. dimension_path_keys: List[Set[ElementPathKey]] = [] @@ -120,7 +151,6 @@ def intersection_by_path_key(linkable_element_sets: Sequence[LinkableElementSet] common_linkable_dimension_path_keys = set.intersection(*dimension_path_keys) if dimension_path_keys else set() common_linkable_entity_path_keys = set.intersection(*entity_path_keys) if entity_path_keys else set() common_linkable_metric_path_keys = set.intersection(*metric_path_keys) if metric_path_keys else set() - # Create a new LinkableElementSet that only includes items where the path key is common to all sets. join_path_to_linkable_dimensions: Dict[ElementPathKey, Set[LinkableDimension]] = defaultdict(set) join_path_to_linkable_entities: Dict[ElementPathKey, Set[LinkableEntity]] = defaultdict(set) @@ -233,43 +263,6 @@ def filter( path_key_to_linkable_metrics=key_to_linkable_metrics, ) - @property - def as_spec_set(self) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=tuple( - DimensionSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - for path_key in self.path_key_to_linkable_dimensions.keys() - if path_key.element_type is LinkableElementType.DIMENSION - ), - time_dimension_specs=tuple( - TimeDimensionSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - time_granularity=path_key.time_granularity, - date_part=path_key.date_part, - ) - for path_key in self.path_key_to_linkable_dimensions.keys() - if path_key.element_type is LinkableElementType.TIME_DIMENSION and path_key.time_granularity - ), - entity_specs=tuple( - EntitySpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - for path_key in self.path_key_to_linkable_entities - ), - group_by_metric_specs=tuple( - GroupByMetricSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - for path_key in self.path_key_to_linkable_metrics - ), - ) - @property def only_unique_path_keys(self) -> LinkableElementSet: """Returns a set that only includes path keys that map to a single distinct element.""" @@ -290,3 +283,75 @@ def only_unique_path_keys(self) -> LinkableElementSet: if len(set(linkable_metrics)) <= 1 }, ) + + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references: Set[SemanticModelReference] = set() + for linkable_dimensions in self.path_key_to_linkable_dimensions.values(): + for linkable_dimension in linkable_dimensions: + semantic_model_references.update(linkable_dimension.derived_from_semantic_models) + for linkable_entities in self.path_key_to_linkable_entities.values(): + for linkable_entity in linkable_entities: + semantic_model_references.update(linkable_entity.derived_from_semantic_models) + for linkable_metrics in self.path_key_to_linkable_metrics.values(): + for linkable_metric in linkable_metrics: + semantic_model_references.update(linkable_metric.derived_from_semantic_models) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + + @property + def spec_count(self) -> int: + """If this is mapped to spec objects, the number of specs that would be produced.""" + return ( + len(self.path_key_to_linkable_dimensions.keys()) + + len(self.path_key_to_linkable_entities.keys()) + + len(self.path_key_to_linkable_metrics.keys()) + ) + + @property + def specs(self) -> Sequence[LinkableInstanceSpec]: + """Converts the items in a `LinkableElementSet` to their corresponding spec objects.""" + return tuple( + path_key.spec + for path_key in ( + tuple(self.path_key_to_linkable_dimensions.keys()) + + tuple(self.path_key_to_linkable_entities.keys()) + + tuple(self.path_key_to_linkable_metrics.keys()) + ) + ) + + def filter_by_spec_patterns(self, spec_patterns: Sequence[SpecPattern]) -> LinkableElementSet: + """Filter the elements in the set by the given spec patters. + + Returns a new set consisting of the elements in the `LinkableElementSet` that have a corresponding spec that + match all the given spec patterns. + """ + # Spec patterns need all specs to match properly e.g. `BaseTimeGrainPattern`. + matching_specs: Sequence[InstanceSpec] = self.specs + + for spec_pattern in spec_patterns: + matching_specs = spec_pattern.match(matching_specs) + specs_to_include = set(matching_specs) + + path_key_to_linkable_dimensions: Dict[ElementPathKey, Tuple[LinkableDimension, ...]] = {} + path_key_to_linkable_entities: Dict[ElementPathKey, Tuple[LinkableEntity, ...]] = {} + path_key_to_linkable_metrics: Dict[ElementPathKey, Tuple[LinkableMetric, ...]] = {} + + for path_key, linkable_dimensions in self.path_key_to_linkable_dimensions.items(): + if path_key.spec in specs_to_include: + path_key_to_linkable_dimensions[path_key] = linkable_dimensions + + for path_key, linkable_entities in self.path_key_to_linkable_entities.items(): + if path_key.spec in specs_to_include: + path_key_to_linkable_entities[path_key] = linkable_entities + + for path_key, linkable_metrics in self.path_key_to_linkable_metrics.items(): + if path_key.spec in specs_to_include: + path_key_to_linkable_metrics[path_key] = linkable_metrics + + return LinkableElementSet( + path_key_to_linkable_dimensions=path_key_to_linkable_dimensions, + path_key_to_linkable_entities=path_key_to_linkable_entities, + path_key_to_linkable_metrics=path_key_to_linkable_metrics, + ) diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py index ce81cf8699..171c1897e0 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py @@ -22,10 +22,10 @@ from metricflow_semantics.errors.error_classes import UnknownMetricLinkingError from metricflow_semantics.mf_logging.pretty_print import mf_pformat +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element import ( ElementPathKey, LinkableDimension, - LinkableElementProperty, LinkableElementType, LinkableEntity, LinkableMetric, diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py b/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py index 0809a811ba..51c95ab88c 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py @@ -9,7 +9,7 @@ from dbt_semantic_interfaces.references import MeasureReference, MetricReference from metricflow_semantics.errors.error_classes import DuplicateMetricError, MetricNotFoundError, NonExistentMeasureError -from metricflow_semantics.model.semantics.linkable_element import LinkableElementProperty +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.model.semantics.linkable_spec_resolver import ( ValidLinkableSpecResolver, diff --git a/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py b/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py index 40c43f7a43..53a56acd14 100644 --- a/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py +++ b/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py @@ -17,9 +17,8 @@ ) from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, - InstanceSpecSetTransform, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform, group_spec_by_type class DunderNamingScheme(QueryItemNamingScheme): @@ -37,7 +36,7 @@ def date_part_suffix(date_part: DatePart) -> str: @override def input_str(self, instance_spec: InstanceSpec) -> Optional[str]: - spec_set = InstanceSpecSet.from_specs((instance_spec,)) + spec_set = group_spec_by_type(instance_spec) for time_dimension_spec in spec_set.time_dimension_specs: # From existing comment in StructuredLinkableSpecName: diff --git a/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py b/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py index 8fc0cd7c48..92015b4fa7 100644 --- a/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py +++ b/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py @@ -9,8 +9,8 @@ from metricflow_semantics.specs.patterns.metric_pattern import MetricSpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, ) +from metricflow_semantics.specs.spec_set import group_spec_by_type class MetricNamingScheme(QueryItemNamingScheme): @@ -18,7 +18,7 @@ class MetricNamingScheme(QueryItemNamingScheme): @override def input_str(self, instance_spec: InstanceSpec) -> Optional[str]: - spec_set = InstanceSpecSet.from_specs((instance_spec,)) + spec_set = group_spec_by_type(instance_spec) names = tuple(spec.element_name for spec in spec_set.metric_specs) if len(names) != 1: diff --git a/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py b/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py index ce20280268..d1bd3453de 100644 --- a/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py +++ b/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py @@ -14,7 +14,8 @@ from dbt_semantic_interfaces.type_enums.date_part import DatePart from typing_extensions import override -from metricflow_semantics.specs.spec_classes import InstanceSpec, InstanceSpecSet, InstanceSpecSetTransform +from metricflow_semantics.specs.spec_classes import InstanceSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform, group_spec_by_type class ObjectBuilderNameConverter: @@ -139,9 +140,7 @@ def transform(self, spec_set: InstanceSpecSet) -> Sequence[str]: @staticmethod def input_str_from_spec(instance_spec: InstanceSpec) -> str: # noqa: D102 - names = ObjectBuilderNameConverter._ObjectBuilderNameTransform().transform( - InstanceSpecSet.from_specs((instance_spec,)) - ) + names = ObjectBuilderNameConverter._ObjectBuilderNameTransform().transform(group_spec_by_type(instance_spec)) if len(names) != 1: raise RuntimeError(f"Did not get exactly 1 name from {instance_spec}. Got {names}") diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py index bedfe43650..edecad3525 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py @@ -1,19 +1,25 @@ from __future__ import annotations import itertools +import logging from dataclasses import dataclass -from typing import Iterable, Tuple +from typing import Sequence, Tuple +from dbt_semantic_interfaces.references import SemanticModelReference from typing_extensions import override +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.query.group_by_item.path_prefixable import PathPrefixable from metricflow_semantics.query.group_by_item.resolution_path import MetricFlowQueryResolutionPath from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec, LinkableSpecSet +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec + +logger = logging.getLogger(__name__) @dataclass(frozen=True) -class GroupByItemCandidateSet(PathPrefixable): +class GroupByItemCandidateSet(PathPrefixable, SemanticModelDerivation): """The set of candidate specs that could match a given spec pattern. This candidate set is refined as it is passed from the root node (representing a measure) to the leaf node @@ -29,7 +35,7 @@ class GroupByItemCandidateSet(PathPrefixable): error messages, you start analyzing from the leaf node. """ - specs: Tuple[LinkableInstanceSpec, ...] + linkable_element_set: LinkableElementSet measure_paths: Tuple[MetricFlowQueryResolutionPath, ...] path_from_leaf_node: MetricFlowQueryResolutionPath @@ -39,18 +45,30 @@ def __post_init__(self) -> None: # noqa: D105 len(self.specs) == 0 and len(self.measure_paths) == 0 ) + @property + def specs(self) -> Sequence[LinkableInstanceSpec]: # noqa: D102 + return self.linkable_element_set.specs + @staticmethod def intersection( - path_from_leaf_node: MetricFlowQueryResolutionPath, candidate_sets: Iterable[GroupByItemCandidateSet] + path_from_leaf_node: MetricFlowQueryResolutionPath, candidate_sets: Sequence[GroupByItemCandidateSet] ) -> GroupByItemCandidateSet: """Create a new candidate set that is the intersection of the given candidate sets. The intersection is defined as the specs common to all candidate sets. path_from_leaf_node is used to indicate where the new candidate set was created. """ - specs_as_sets = tuple(set(candidate_set.specs) for candidate_set in candidate_sets) - common_specs = set.intersection(*specs_as_sets) if specs_as_sets else set() - if len(common_specs) == 0: + if len(candidate_sets) == 0: + return GroupByItemCandidateSet.empty_instance() + elif len(candidate_sets) == 1: + return GroupByItemCandidateSet( + linkable_element_set=candidate_sets[0].linkable_element_set, + measure_paths=candidate_sets[0].measure_paths, + path_from_leaf_node=path_from_leaf_node, + ) + linkable_element_set_candidates = tuple(candidate_set.linkable_element_set for candidate_set in candidate_sets) + intersection_result = LinkableElementSet.intersection_by_path_key(linkable_element_set_candidates) + if intersection_result.spec_count == 0: return GroupByItemCandidateSet.empty_instance() measure_paths = tuple( @@ -58,7 +76,9 @@ def intersection( ) return GroupByItemCandidateSet( - specs=tuple(common_specs), measure_paths=measure_paths, path_from_leaf_node=path_from_leaf_node + linkable_element_set=intersection_result, + measure_paths=measure_paths, + path_from_leaf_node=path_from_leaf_node, ) @property @@ -72,31 +92,34 @@ def num_candidates(self) -> int: # noqa: D102 @staticmethod def empty_instance() -> GroupByItemCandidateSet: # noqa: D102 return GroupByItemCandidateSet( - specs=(), measure_paths=(), path_from_leaf_node=MetricFlowQueryResolutionPath.empty_instance() + linkable_element_set=LinkableElementSet(), + measure_paths=(), + path_from_leaf_node=MetricFlowQueryResolutionPath.empty_instance(), ) - @property - def spec_set(self) -> LinkableSpecSet: - """Return the candidates as a spec set.""" - return LinkableSpecSet.from_specs(self.specs) - def filter_candidates_by_pattern( self, spec_pattern: SpecPattern, ) -> GroupByItemCandidateSet: """Return a new candidate set that only contains specs that match the given pattern.""" - matching_specs = tuple(InstanceSpecSet.from_specs(spec_pattern.match(self.specs)).linkable_specs) - if len(matching_specs) == 0: + filtered_element_set = self.linkable_element_set.filter_by_spec_patterns((spec_pattern,)) + if filtered_element_set.spec_count == 0: return GroupByItemCandidateSet.empty_instance() - return GroupByItemCandidateSet( - specs=matching_specs, measure_paths=self.measure_paths, path_from_leaf_node=self.path_from_leaf_node + linkable_element_set=filtered_element_set, + measure_paths=self.measure_paths, + path_from_leaf_node=self.path_from_leaf_node, ) @override def with_path_prefix(self, path_prefix: MetricFlowQueryResolutionPath) -> GroupByItemCandidateSet: return GroupByItemCandidateSet( - specs=self.specs, + linkable_element_set=self.linkable_element_set, measure_paths=tuple(path.with_path_prefix(path_prefix) for path in self.measure_paths), path_from_leaf_node=self.path_from_leaf_node.with_path_prefix(path_prefix), ) + + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + return self.linkable_element_set.derived_from_semantic_models diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py index 7abca90066..d44180783e 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py @@ -11,8 +11,8 @@ from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat, mf_pformat_many +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup -from metricflow_semantics.model.semantics.linkable_element import LinkableElementProperty from metricflow_semantics.query.group_by_item.candidate_push_down.group_by_item_candidate import GroupByItemCandidateSet from metricflow_semantics.query.group_by_item.resolution_dag.resolution_nodes.base_node import ( GroupByItemResolutionNode, @@ -49,7 +49,6 @@ from metricflow_semantics.specs.patterns.base_time_grain import BaseTimeGrainPattern from metricflow_semantics.specs.patterns.none_date_part import NoneDatePartPattern from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec logger = logging.getLogger(__name__) @@ -156,13 +155,11 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu """Push the group-by-item specs that are available to the measure and match the source patterns to the child.""" with self._path_from_start_node_tracker.track_node_visit(node) as current_traversal_path: logger.info(f"Handling {node.ui_description}") - specs_available_for_measure: Sequence[ - LinkableInstanceSpec - ] = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_measure( + items_available_for_measure = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_measure( measure_reference=node.measure_reference, with_any_of=self._with_any_property, without_any_of=self._without_any_property, - ).as_spec_set.as_tuple + ) # The following is needed to handle limitation of cumulative metrics. Filtering could be done at the measure # node, but doing it here makes it a little easier to generate the error message. @@ -192,32 +189,28 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu else: assert_values_exhausted(metric.type) - specs_available_for_measure_given_child_metric = specs_available_for_measure - - for pattern_to_apply in patterns_to_apply: - specs_available_for_measure_given_child_metric = InstanceSpecSet.from_specs( - pattern_to_apply.match(specs_available_for_measure_given_child_metric) - ).linkable_specs - - matching_specs = specs_available_for_measure_given_child_metric - - for source_spec_pattern in self._source_spec_patterns: - matching_specs = InstanceSpecSet.from_specs(source_spec_pattern.match(matching_specs)).linkable_specs + matching_items = items_available_for_measure.filter_by_spec_patterns( + patterns_to_apply + self._source_spec_patterns + ) - logger.debug( - f"For {node.ui_description}:\n" - + indent( - "After applying patterns:\n" - + indent(mf_pformat(patterns_to_apply)) - + "\n" - + "to inputs, matches are:\n" - + indent(mf_pformat(matching_specs)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + f"For {node.ui_description}:\n" + + indent( + "After applying patterns:\n" + + indent(mf_pformat(patterns_to_apply)) + + "\n" + + "to inputs, matches are:\n" + + indent(mf_pformat(matching_items.specs)) + ) ) - ) # The specified patterns don't match to any of the available group-by-items that can be queried for the # measure. - if len(matching_specs) == 0: + if matching_items.spec_count == 0: + items_available_for_measure_given_child_metric = items_available_for_measure.filter_by_spec_patterns( + patterns_to_apply + ) return PushDownResult( candidate_set=GroupByItemCandidateSet.empty_instance(), issue_set=MetricFlowQueryResolutionIssueSet.from_issue( @@ -227,7 +220,7 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu input_suggestions=( tuple( self._suggestion_generator.input_suggestions( - specs_available_for_measure_given_child_metric + items_available_for_measure_given_child_metric.specs ) ) if self._suggestion_generator is not None @@ -240,7 +233,7 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu return PushDownResult( candidate_set=GroupByItemCandidateSet( measure_paths=(current_traversal_path,), - specs=tuple(matching_specs), + linkable_element_set=matching_items, path_from_leaf_node=current_traversal_path, ), issue_set=MetricFlowQueryResolutionIssueSet(), @@ -314,9 +307,10 @@ def visit_metric_node(self, node: MetricGroupByItemResolutionNode) -> PushDownRe current_traversal_path=current_traversal_path, ) logger.info(f"Handling {node.ui_description}") - logger.debug( - "candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) - ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) + ) if merged_result_from_parents.candidate_set.is_empty: return merged_result_from_parents @@ -340,43 +334,47 @@ def visit_metric_node(self, node: MetricGroupByItemResolutionNode) -> PushDownRe else: assert_values_exhausted(metric.type) - candidate_specs: Sequence[LinkableInstanceSpec] = merged_result_from_parents.candidate_set.specs + candidate_items = merged_result_from_parents.candidate_set.linkable_element_set issue_sets_to_merge = [merged_result_from_parents.issue_set] - matched_specs = candidate_specs - for pattern_to_apply in patterns_to_apply: - matched_specs = InstanceSpecSet.from_specs(pattern_to_apply.match(matched_specs)).linkable_specs - - logger.debug( - f"For {node.ui_description}:\n" - + indent( - "After applying patterns:\n" - + indent(mf_pformat(patterns_to_apply)) - + "\n" - + "to inputs, outputs are:\n" - + indent(mf_pformat(matched_specs)) + matched_items = candidate_items.filter_by_spec_patterns(patterns_to_apply) + + if logger.isEnabledFor(logging.DEBUG): + matched_specs = matched_items.specs + logger.debug( + f"For {node.ui_description}:\n" + + indent( + "After applying patterns:\n" + + indent(mf_pformat(patterns_to_apply)) + + "\n" + + "to inputs, outputs are:\n" + + indent(mf_pformat(matched_specs)) + ) ) - ) # There were candidates that were common from the ones passed from parents, but after applying the filters, # none of the candidates were valid. - if len(matched_specs) == 0: + if matched_items.spec_count == 0: issue_sets_to_merge.append( MetricFlowQueryResolutionIssueSet.from_issue( MetricExcludesDatePartIssue.from_parameters( query_resolution_path=current_traversal_path, - candidate_specs=candidate_specs, + candidate_specs=candidate_items.specs, parent_issues=(), ) ) ) + if matched_items.spec_count == 0: + return PushDownResult( + candidate_set=GroupByItemCandidateSet.empty_instance(), + issue_set=MetricFlowQueryResolutionIssueSet.merge_iterable(issue_sets_to_merge), + ) + return PushDownResult( candidate_set=GroupByItemCandidateSet( - specs=tuple(matched_specs), - measure_paths=( - merged_result_from_parents.candidate_set.measure_paths if len(matched_specs) > 0 else () - ), + linkable_element_set=matched_items, + measure_paths=merged_result_from_parents.candidate_set.measure_paths, path_from_leaf_node=current_traversal_path, ), issue_set=MetricFlowQueryResolutionIssueSet.merge_iterable(issue_sets_to_merge), @@ -395,11 +393,10 @@ def visit_query_node(self, node: QueryGroupByItemResolutionNode) -> PushDownResu }, current_traversal_path=current_traversal_path, ) - - logger.info(f"Handling {node.ui_description}") - logger.debug( - "candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) - ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) + ) return merged_result_from_parents @@ -410,14 +407,11 @@ def visit_no_metrics_query_node(self, node: NoMetricsGroupByItemSourceNode) -> P logger.info(f"Handling {node.ui_description}") # This is a case for distinct dimension values from semantic models. candidate_elements = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_no_metrics_query() + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Candidate elements are:\n{mf_pformat(candidate_elements)}") + candidates_after_filtering = candidate_elements.filter_by_spec_patterns(self._source_spec_patterns) - matching_specs: Sequence[LinkableInstanceSpec] = tuple( - sorted(candidate_elements.as_spec_set.as_tuple, key=lambda x: x.qualified_name) - ) - for pattern_to_apply in self._source_spec_patterns: - matching_specs = InstanceSpecSet.from_specs(pattern_to_apply.match(matching_specs)).linkable_specs - - if len(matching_specs) == 0: + if candidates_after_filtering.spec_count == 0: return PushDownResult( candidate_set=GroupByItemCandidateSet.empty_instance(), issue_set=MetricFlowQueryResolutionIssueSet.from_issue( @@ -430,7 +424,7 @@ def visit_no_metrics_query_node(self, node: NoMetricsGroupByItemSourceNode) -> P return PushDownResult( candidate_set=GroupByItemCandidateSet( - specs=tuple(matching_specs), + linkable_element_set=candidates_after_filtering, measure_paths=(current_traversal_path,), path_from_leaf_node=current_traversal_path, ), diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py index 2c6468a5f9..6624928c9a 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py @@ -24,6 +24,7 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern if TYPE_CHECKING: + from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec logger = logging.getLogger(__name__) @@ -170,13 +171,30 @@ class FilterSpecResolution: lookup_key: ResolvedSpecLookUpKey where_filter_intersection: WhereFilterIntersection - resolved_spec: Optional[LinkableInstanceSpec] + resolved_linkable_element_set: Optional[LinkableElementSet] spec_pattern: SpecPattern issue_set: MetricFlowQueryResolutionIssueSet # Used for error messages. filter_location_path: MetricFlowQueryResolutionPath object_builder_str: str + def __post_init__(self) -> None: # noqa: D105 + if self.resolved_linkable_element_set is not None: + assert len(self.resolved_linkable_element_set.specs) <= 1 + + @property + def resolved_spec(self) -> Optional[LinkableInstanceSpec]: # noqa: D102 + if self.resolved_linkable_element_set is None: + return None + + specs = self.resolved_linkable_element_set.specs + if len(specs) == 0: + return None + elif len(specs) == 1: + return specs[0] + else: + raise RuntimeError(f"Found {len(specs)} in {self.resolved_linkable_element_set}") + CallParameterSet = Union[ DimensionCallParameterSet, TimeDimensionCallParameterSet, EntityCallParameterSet, MetricCallParameterSet diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py index c698ad5c99..0c3efcb5cc 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py @@ -362,7 +362,7 @@ def _resolve_specs_for_where_filters( call_parameter_set=group_by_item_in_where_filter.call_parameter_set, ), filter_location_path=resolution_path, - resolved_spec=group_by_item_resolution.spec, + resolved_linkable_element_set=group_by_item_resolution.linkable_element_set, where_filter_intersection=where_filter_intersection, spec_pattern=group_by_item_in_where_filter.spec_pattern, issue_set=group_by_item_resolution.issue_set.with_path_prefix(path_prefix), diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py index e501b2a499..5bda159f03 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py @@ -2,16 +2,19 @@ import logging from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional, Sequence, Tuple from dbt_semantic_interfaces.call_parameter_sets import TimeDimensionCallParameterSet from dbt_semantic_interfaces.naming.keywords import METRIC_TIME_ELEMENT_NAME -from dbt_semantic_interfaces.references import TimeDimensionReference +from dbt_semantic_interfaces.references import SemanticModelReference, TimeDimensionReference from dbt_semantic_interfaces.type_enums import TimeGranularity +from typing_extensions import override from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme from metricflow_semantics.query.group_by_item.candidate_push_down.push_down_visitor import ( PushDownResult, @@ -27,13 +30,14 @@ from metricflow_semantics.specs.patterns.base_time_grain import BaseTimeGrainPattern from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.patterns.typed_patterns import TimeDimensionPattern -from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec, LinkableSpecSet +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet, group_specs_by_type logger = logging.getLogger(__name__) @dataclass(frozen=True) -class GroupByItemResolution: +class GroupByItemResolution(SemanticModelDerivation): """Result object that contains matching spec for a potentially ambiguous input to a query. e.g. "TimeDimension('metric_time')" -> TimeDimensionSpec('metric_time', DAY) @@ -43,8 +47,14 @@ class GroupByItemResolution: # If the spec is None, then the pattern couldn't be resolved spec: Optional[LinkableInstanceSpec] + linkable_element_set: LinkableElementSet issue_set: MetricFlowQueryResolutionIssueSet + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + return self.linkable_element_set.derived_from_semantic_models + @dataclass(frozen=True) class AvailableGroupByItemsResolution: @@ -86,6 +96,7 @@ def resolve_matching_item_for_querying( if push_down_result.candidate_set.num_candidates == 0: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set, ) @@ -101,6 +112,7 @@ def resolve_matching_item_for_querying( if push_down_result.candidate_set.num_candidates > 1: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set.add_issue( AmbiguousGroupByItemIssue.from_parameters( candidate_set=push_down_result.candidate_set, @@ -111,7 +123,11 @@ def resolve_matching_item_for_querying( ), ) - return GroupByItemResolution(spec=push_down_result.candidate_set.specs[0], issue_set=push_down_result.issue_set) + return GroupByItemResolution( + spec=push_down_result.candidate_set.specs[0], + linkable_element_set=push_down_result.candidate_set.linkable_element_set, + issue_set=push_down_result.issue_set, + ) def resolve_matching_item_for_filters( self, @@ -147,12 +163,14 @@ def resolve_matching_item_for_filters( if push_down_result.candidate_set.num_candidates == 0: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set, ) if push_down_result.candidate_set.num_candidates > 1: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set.add_issue( AmbiguousGroupByItemIssue.from_parameters( candidate_set=push_down_result.candidate_set, @@ -163,7 +181,11 @@ def resolve_matching_item_for_filters( ), ) - return GroupByItemResolution(spec=push_down_result.candidate_set.specs[0], issue_set=push_down_result.issue_set) + return GroupByItemResolution( + spec=push_down_result.candidate_set.specs[0], + linkable_element_set=push_down_result.candidate_set.linkable_element_set, + issue_set=push_down_result.issue_set, + ) def resolve_available_items( self, @@ -186,7 +208,7 @@ def resolve_available_items( push_down_result: PushDownResult = resolution_node.accept(push_down_visitor) return AvailableGroupByItemsResolution( - specs=push_down_result.candidate_set.specs, + specs=tuple(push_down_result.candidate_set.specs), issue_set=push_down_result.issue_set, ) @@ -202,9 +224,9 @@ def resolve_min_metric_time_grain(self) -> TimeGranularity: suggestion_generator=None, ) metric_time_spec_set = ( - LinkableSpecSet.from_specs((metric_time_grain_resolution.spec,)) + group_specs_by_type((metric_time_grain_resolution.spec,)) if metric_time_grain_resolution.spec is not None - else LinkableSpecSet.empty_instance() + else InstanceSpecSet.empty_instance() ) if len(metric_time_spec_set.time_dimension_specs) != 1: raise RuntimeError( diff --git a/metricflow-semantics/metricflow_semantics/query/query_parser.py b/metricflow-semantics/metricflow_semantics/query/query_parser.py index 63486319ce..873120f59e 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_parser.py +++ b/metricflow-semantics/metricflow_semantics/query/query_parser.py @@ -2,6 +2,7 @@ import datetime import logging +from dataclasses import dataclass from typing import List, Optional, Sequence, Tuple, Union import pandas as pd @@ -11,6 +12,7 @@ ) from dbt_semantic_interfaces.protocols import SavedQuery from dbt_semantic_interfaces.protocols.where_filter import WhereFilter +from dbt_semantic_interfaces.references import SemanticModelReference from dbt_semantic_interfaces.type_enums import TimeGranularity from metricflow_semantics.assert_one_arg import assert_at_most_one_arg_set @@ -54,12 +56,12 @@ from metricflow_semantics.specs.patterns.base_time_grain import BaseTimeGrainPattern from metricflow_semantics.specs.patterns.metric_time_pattern import MetricTimePattern from metricflow_semantics.specs.patterns.none_date_part import NoneDatePartPattern +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, - MetricFlowQuerySpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.time.time_granularity import ( adjust_to_end_of_period, adjust_to_start_of_period, @@ -101,7 +103,7 @@ def parse_and_validate_saved_query( time_constraint_end: Optional[datetime.datetime], order_by_names: Optional[Sequence[str]], order_by_parameters: Optional[Sequence[OrderByQueryParameter]], - ) -> MetricFlowQuerySpec: + ) -> ParseQueryResult: """Parse and validate a query using parameters from a pre-defined / saved query. Additional parameters act in conjunction with the parameters in the saved query. @@ -115,15 +117,19 @@ def parse_and_validate_saved_query( if where_filter is not None: where_filters.append(where_filter) - return self.parse_and_validate_query( + return self._parse_and_validate_query( metric_names=saved_query.query_params.metrics, + metrics=None, group_by_names=saved_query.query_params.group_by, + group_by=None, where_constraint=merge_to_single_where_filter(PydanticWhereFilterIntersection(where_filters=where_filters)), + where_constraint_str=None, time_constraint_start=time_constraint_start, time_constraint_end=time_constraint_end, limit=limit, order_by_names=order_by_names, order_by=order_by_parameters, + min_max_only=False, ) def _get_saved_query(self, saved_query_parameter: SavedQueryParameter) -> SavedQuery: @@ -155,7 +161,7 @@ def _metric_time_granularity(time_dimension_specs: Sequence[TimeDimensionSpec]) ): matching_specs = pattern_to_apply.match(matching_specs) # The conversion below is awkward and needs some more thought. - time_dimension_specs = InstanceSpecSet.from_specs(matching_specs).time_dimension_specs + time_dimension_specs = group_specs_by_type(matching_specs).time_dimension_specs if len(time_dimension_specs) == 0: return None @@ -333,7 +339,7 @@ def parse_and_validate_query( order_by_names=order_by_names, order_by=order_by, min_max_only=min_max_only, - ) + ).query_spec @log_runtime() def _parse_and_validate_query( @@ -350,7 +356,7 @@ def _parse_and_validate_query( order_by_names: Optional[Sequence[str]], order_by: Optional[Sequence[OrderByQueryParameter]], min_max_only: bool, - ) -> MetricFlowQuerySpec: + ) -> ParseQueryResult: # TODO: validate min_max_only - can only be called for non-metric queries assert_at_most_one_arg_set(metric_names=metric_names, metrics=metrics) assert_at_most_one_arg_set(group_by_names=group_by_names, group_by=group_by) @@ -508,6 +514,20 @@ def _parse_and_validate_query( ) logger.info(f"Time constraint after adjustment is: {time_constraint}") - return query_spec.with_time_range_constraint(time_constraint) + return ParseQueryResult( + query_spec=query_spec.with_time_range_constraint(time_constraint), + queried_semantic_models=query_resolution.queried_semantic_models, + ) + + return ParseQueryResult( + query_spec=query_spec, + queried_semantic_models=query_resolution.queried_semantic_models, + ) + + +@dataclass(frozen=True) +class ParseQueryResult: + """Result of parsing a MetricFlow query.""" - return query_spec + query_spec: MetricFlowQuerySpec + queried_semantic_models: Tuple[SemanticModelReference, ...] diff --git a/metricflow-semantics/metricflow_semantics/query/query_resolution.py b/metricflow-semantics/metricflow_semantics/query/query_resolution.py index f715c048fd..e76e8e8e39 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_resolution.py +++ b/metricflow-semantics/metricflow_semantics/query/query_resolution.py @@ -1,19 +1,21 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional, Sized, Tuple +from typing import Optional, Sequence, Sized, Tuple +from dbt_semantic_interfaces.references import SemanticModelReference from typing_extensions import override from metricflow_semantics.collection_helpers.merger import Mergeable from metricflow_semantics.mf_logging.pretty_print import mf_pformat +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( FilterSpecResolutionLookUp, ) from metricflow_semantics.query.group_by_item.resolution_dag.dag import GroupByItemResolutionDag from metricflow_semantics.query.issues.issues_base import MetricFlowQueryResolutionIssueSet from metricflow_semantics.query.resolver_inputs.base_resolver_inputs import MetricFlowQueryResolverInput -from metricflow_semantics.specs.spec_classes import MetricFlowQuerySpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec @dataclass(frozen=True) @@ -67,12 +69,11 @@ def __len__(self) -> int: @dataclass(frozen=True) -class MetricFlowQueryResolution: +class MetricFlowQueryResolution(SemanticModelDerivation): """The result of resolving query inputs to specs.""" # Can be None if there were errors. query_spec: Optional[MetricFlowQuerySpec] - # The resolution DAG generated for the query. resolution_dag: Optional[GroupByItemResolutionDag] # The lookup that is used later in the DataflowPlanBuilder to figure out which specs are required by the filters @@ -80,6 +81,13 @@ class MetricFlowQueryResolution: filter_spec_lookup: FilterSpecResolutionLookUp # Mapping of issues with the inputs. input_to_issue_set: InputToIssueSetMapping + # The semantic models that would be queried to resolve the query. + queried_semantic_models: Tuple[SemanticModelReference, ...] + + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + return self.queried_semantic_models @property def checked_query_spec(self) -> MetricFlowQuerySpec: diff --git a/metricflow-semantics/metricflow_semantics/query/query_resolver.py b/metricflow-semantics/metricflow_semantics/query/query_resolver.py index 45f4e0ff39..1a41d3418c 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_resolver.py +++ b/metricflow-semantics/metricflow_semantics/query/query_resolver.py @@ -9,6 +9,7 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.mf_logging.runtime import log_runtime from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.metric_scheme import MetricNamingScheme from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_pattern_factory import ( WhereFilterPatternFactory, @@ -52,14 +53,14 @@ from metricflow_semantics.query.suggestion_generator import QueryItemSuggestionGenerator from metricflow_semantics.query.validation_rules.query_validator import PostResolutionQueryValidator from metricflow_semantics.specs.patterns.match_list_pattern import MatchListSpecPattern +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( InstanceSpec, LinkableInstanceSpec, - LinkableSpecSet, - MetricFlowQuerySpec, MetricSpec, OrderBySpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type logger = logging.getLogger(__name__) @@ -103,6 +104,7 @@ class ResolveGroupByItemsResult: resolution_dag: GroupByItemResolutionDag group_by_item_specs: Tuple[LinkableInstanceSpec, ...] input_to_issue_set_mapping: InputToIssueSetMapping + linkable_element_set: LinkableElementSet @dataclass(frozen=True) @@ -231,6 +233,7 @@ def _resolve_group_by_items_result( input_to_issue_set_mapping_items: List[InputToIssueSetMappingItem] = [] group_by_item_specs: List[LinkableInstanceSpec] = [] + linkable_element_sets: List[LinkableElementSet] = [] for group_by_item_input in group_by_item_inputs: resolution = MetricFlowQueryResolver._resolve_group_by_item_input( group_by_item_resolver=group_by_item_resolver, @@ -243,11 +246,13 @@ def _resolve_group_by_items_result( ) if resolution.spec is not None: group_by_item_specs.append(resolution.spec) + linkable_element_sets.append(resolution.linkable_element_set) return ResolveGroupByItemsResult( resolution_dag=resolution_dag, group_by_item_specs=tuple(group_by_item_specs), input_to_issue_set_mapping=InputToIssueSetMapping(tuple(input_to_issue_set_mapping_items)), + linkable_element_set=LinkableElementSet.merge_by_path_key(linkable_element_sets), ) @staticmethod @@ -419,6 +424,7 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=None, filter_spec_lookup=FilterSpecResolutionLookUp.empty_instance(), input_to_issue_set=issue_set_mapping_so_far, + queried_semantic_models=(), ) # Resolve group by items. @@ -478,11 +484,12 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=resolution_dag, filter_spec_lookup=filter_spec_lookup, input_to_issue_set=issue_set_mapping_so_far, + queried_semantic_models=(), ) # No errors. - linkable_spec_set = LinkableSpecSet.from_specs(group_by_item_specs) - logger.info(f"Group-by-items were resolved to:\n{mf_pformat(linkable_spec_set.as_tuple)}") + linkable_spec_set = group_specs_by_type(group_by_item_specs) + logger.info(f"Group-by-items were resolved to:\n{mf_pformat(linkable_spec_set.linkable_specs)}") # Run post-resolution validation rules to generate issues that are generated at the query-level. query_level_issue_set = self._post_resolution_query_validator.validate_query( @@ -510,6 +517,7 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=resolution_dag, filter_spec_lookup=filter_spec_lookup, input_to_issue_set=issue_set_mapping, + queried_semantic_models=(), ) return MetricFlowQueryResolution( @@ -527,4 +535,7 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=resolution_dag, filter_spec_lookup=filter_spec_lookup, input_to_issue_set=issue_set_mapping, + queried_semantic_models=tuple( + resolve_group_by_item_result.linkable_element_set.derived_from_semantic_models + ), ) diff --git a/metricflow-semantics/metricflow_semantics/specs/linkable_spec_set.py b/metricflow-semantics/metricflow_semantics/specs/linkable_spec_set.py new file mode 100644 index 0000000000..236172d8ab --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/linkable_spec_set.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import itertools +import typing +from dataclasses import dataclass +from typing import Dict, List, Sequence, Tuple + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass +from dbt_semantic_interfaces.references import MeasureReference, MetricReference +from typing_extensions import override + +from metricflow_semantics.collection_helpers.merger import Mergeable +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + GroupByMetricSpec, + InstanceSpec, + LinkableInstanceSpec, + TimeDimensionSpec, +) +from metricflow_semantics.specs.spec_set import InstanceSpecSet + +if typing.TYPE_CHECKING: + from metricflow_semantics.model.semantics.metric_lookup import MetricLookup + from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup + + +@dataclass(frozen=True) +class LinkableSpecSet(Mergeable, SerializableDataclass): + """Groups linkable specs.""" + + dimension_specs: Tuple[DimensionSpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () + entity_specs: Tuple[EntitySpec, ...] = () + group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () + + @property + def contains_metric_time(self) -> bool: + """Returns true if this set contains a spec referring to metric time at any grain.""" + return len(self.metric_time_specs) > 0 + + def included_agg_time_dimension_specs_for_metric( + self, metric_reference: MetricReference, metric_lookup: MetricLookup + ) -> List[TimeDimensionSpec]: + """Get the time dims included that are valid agg time dimensions for the specified metric.""" + queried_metric_time_specs = list(self.metric_time_specs) + + valid_agg_time_dimensions = metric_lookup.get_valid_agg_time_dimensions_for_metric(metric_reference) + queried_agg_time_dimension_specs = ( + list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) + + queried_metric_time_specs + ) + + return queried_agg_time_dimension_specs + + def included_agg_time_dimension_specs_for_measure( + self, measure_reference: MeasureReference, semantic_model_lookup: SemanticModelLookup + ) -> List[TimeDimensionSpec]: + """Get the time dims included that are valid agg time dimensions for the specified measure.""" + queried_metric_time_specs = list(self.metric_time_specs) + + valid_agg_time_dimensions = semantic_model_lookup.get_agg_time_dimension_specs_for_measure(measure_reference) + queried_agg_time_dimension_specs = ( + list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) + + queried_metric_time_specs + ) + + return queried_agg_time_dimension_specs + + @property + def metric_time_specs(self) -> Sequence[TimeDimensionSpec]: + """Returns any specs referring to metric time at any grain.""" + return tuple( + time_dimension_spec + for time_dimension_spec in self.time_dimension_specs + if time_dimension_spec.is_metric_time + ) + + @property + def as_tuple(self) -> Tuple[LinkableInstanceSpec, ...]: # noqa: D102 + return tuple( + itertools.chain( + self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs + ) + ) + + @override + def merge(self, other: LinkableSpecSet) -> LinkableSpecSet: + return LinkableSpecSet( + dimension_specs=self.dimension_specs + other.dimension_specs, + time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, + entity_specs=self.entity_specs + other.entity_specs, + group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, + ) + + @override + @classmethod + def empty_instance(cls) -> LinkableSpecSet: + return LinkableSpecSet() + + def dedupe(self) -> LinkableSpecSet: # noqa: D102 + # Use dictionaries to dedupe as it preserves insertion order. + + dimension_spec_dict: Dict[DimensionSpec, None] = {} + for dimension_spec in self.dimension_specs: + dimension_spec_dict[dimension_spec] = None + + time_dimension_spec_dict: Dict[TimeDimensionSpec, None] = {} + for time_dimension_spec in self.time_dimension_specs: + time_dimension_spec_dict[time_dimension_spec] = None + + entity_spec_dict: Dict[EntitySpec, None] = {} + for entity_spec in self.entity_specs: + entity_spec_dict[entity_spec] = None + + group_by_metric_spec_dict: Dict[GroupByMetricSpec, None] = {} + for group_by_metric in self.group_by_metric_specs: + group_by_metric_spec_dict[group_by_metric] = None + + return LinkableSpecSet( + dimension_specs=tuple(dimension_spec_dict.keys()), + time_dimension_specs=tuple(time_dimension_spec_dict.keys()), + entity_specs=tuple(entity_spec_dict.keys()), + group_by_metric_specs=tuple(group_by_metric_spec_dict.keys()), + ) + + def is_subset_of(self, other_set: LinkableSpecSet) -> bool: # noqa: D102 + return set(self.as_tuple).issubset(set(other_set.as_tuple)) + + @property + def as_spec_set(self) -> InstanceSpecSet: # noqa: D102 + return InstanceSpecSet( + dimension_specs=self.dimension_specs, + time_dimension_specs=self.time_dimension_specs, + entity_specs=self.entity_specs, + group_by_metric_specs=self.group_by_metric_specs, + ) + + def difference(self, other: LinkableSpecSet) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet( + dimension_specs=tuple(set(self.dimension_specs) - set(other.dimension_specs)), + time_dimension_specs=tuple(set(self.time_dimension_specs) - set(other.time_dimension_specs)), + entity_specs=tuple(set(self.entity_specs) - set(other.entity_specs)), + group_by_metric_specs=tuple(set(self.group_by_metric_specs) - set(other.group_by_metric_specs)), + ) + + def __len__(self) -> int: # noqa: D105 + return len(self.dimension_specs) + len(self.time_dimension_specs) + len(self.entity_specs) + + @staticmethod + def create_from_spec_set(spec_set: InstanceSpecSet) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet( + dimension_specs=spec_set.dimension_specs, + time_dimension_specs=spec_set.time_dimension_specs, + entity_specs=spec_set.entity_specs, + group_by_metric_specs=spec_set.group_by_metric_specs, + ) + + @staticmethod + def create_from_specs(specs: Sequence[InstanceSpec]) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet.create_from_spec_set(InstanceSpecSet.create_from_specs(specs)) diff --git a/metricflow-semantics/metricflow_semantics/specs/partition_spec_set.py b/metricflow-semantics/metricflow_semantics/specs/partition_spec_set.py new file mode 100644 index 0000000000..51b598b764 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/partition_spec_set.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Tuple + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass + +from metricflow_semantics.specs.spec_classes import DimensionSpec, TimeDimensionSpec + + +@dataclass(frozen=True) +class PartitionSpecSet(SerializableDataclass): + """Grouping of the linkable specs.""" + + dimension_specs: Tuple[DimensionSpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py b/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py index fa23af1990..34b44c1267 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py @@ -10,13 +10,12 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, - LinkableSpecSet, TimeDimensionSpec, TimeDimensionSpecComparisonKey, TimeDimensionSpecField, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type class BaseTimeGrainPattern(SpecPattern): @@ -66,7 +65,7 @@ def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[InstanceSpe return other_specs + tuple(BaseTimeGrainPattern(only_apply_for_metric_time=False).match(metric_time_specs)) - spec_set = LinkableSpecSet.from_specs(InstanceSpecSet.from_specs(candidate_specs).linkable_specs) + spec_set = group_specs_by_type(candidate_specs) spec_key_to_grains: Dict[TimeDimensionSpecComparisonKey, Set[TimeGranularity]] = defaultdict(set) spec_key_to_specs: Dict[TimeDimensionSpecComparisonKey, List[TimeDimensionSpec]] = defaultdict(list) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py index 5f152a07c4..5d5d9892a5 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py @@ -12,7 +12,8 @@ from typing_extensions import override from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern -from metricflow_semantics.specs.spec_classes import InstanceSpec, InstanceSpecSet, LinkableInstanceSpec +from metricflow_semantics.specs.spec_classes import InstanceSpec, LinkableInstanceSpec +from metricflow_semantics.specs.spec_set import group_specs_by_type logger = logging.getLogger(__name__) @@ -109,7 +110,7 @@ def _match_entity_links(self, candidate_specs: Sequence[LinkableInstanceSpec]) - @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - filtered_candidate_specs = InstanceSpecSet.from_specs(candidate_specs).linkable_specs + filtered_candidate_specs = group_specs_by_type(candidate_specs).linkable_specs # Checks that EntityLinkPatternParameterSetField is valid wrt to the parameter set. # Entity links could be a partial match, so it's handled separately. diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py index 0bbe31cbe4..346b39234c 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py @@ -9,9 +9,9 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, MetricSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type @dataclass(frozen=True) @@ -22,7 +22,7 @@ class MetricSpecPattern(SpecPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[MetricSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return tuple( metric_name for metric_name in spec_set.metric_specs if metric_name.reference == self.metric_reference ) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py index ff0337de10..4ef943a86d 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py @@ -8,10 +8,9 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, - LinkableSpecSet, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type class MetricTimePattern(SpecPattern): @@ -23,7 +22,7 @@ class MetricTimePattern(SpecPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[TimeDimensionSpec]: - spec_set = LinkableSpecSet.from_specs(InstanceSpecSet.from_specs(candidate_specs).linkable_specs) + spec_set = group_specs_by_type(candidate_specs) return tuple( time_dimension_spec for time_dimension_spec in spec_set.time_dimension_specs diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py b/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py index 5a30db5cff..e451948f5a 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py @@ -7,9 +7,9 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type class NoneDatePartPattern(SpecPattern): @@ -21,7 +21,7 @@ class NoneDatePartPattern(SpecPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: specs_to_return: List[LinkableInstanceSpec] = [] - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) for time_dimension_spec in spec_set.time_dimension_specs: if time_dimension_spec.date_part is None: specs_to_return.append(time_dimension_spec) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py index cec6ef7859..082a5b556a 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py @@ -17,7 +17,8 @@ EntityLinkPatternParameterSet, ParameterSetField, ) -from metricflow_semantics.specs.spec_classes import InstanceSpec, InstanceSpecSet, LinkableInstanceSpec +from metricflow_semantics.specs.spec_classes import InstanceSpec, LinkableInstanceSpec +from metricflow_semantics.specs.spec_set import group_specs_by_type @dataclass(frozen=True) @@ -29,7 +30,7 @@ class DimensionPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) filtered_specs: Sequence[LinkableInstanceSpec] = spec_set.dimension_specs + spec_set.time_dimension_specs return super().match(filtered_specs) @@ -58,7 +59,7 @@ class TimeDimensionPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return super().match(spec_set.time_dimension_specs) @staticmethod @@ -99,7 +100,7 @@ class EntityPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return super().match(spec_set.entity_specs) @staticmethod @@ -122,7 +123,7 @@ class GroupByMetricPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return super().match(spec_set.group_by_metric_specs) @staticmethod diff --git a/metricflow-semantics/metricflow_semantics/specs/query_spec.py b/metricflow-semantics/metricflow_semantics/specs/query_spec.py new file mode 100644 index 0000000000..be42f19bc1 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/query_spec.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional, Tuple + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass +from dbt_semantic_interfaces.protocols import WhereFilterIntersection + +from metricflow_semantics.filters.time_constraint import TimeRangeConstraint +from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( + FilterSpecResolutionLookUp, +) +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + GroupByMetricSpec, + MetricSpec, + OrderBySpec, + TimeDimensionSpec, +) + + +@dataclass(frozen=True) +class MetricFlowQuerySpec(SerializableDataclass): + """Specs needed for running a query.""" + + metric_specs: Tuple[MetricSpec, ...] = () + dimension_specs: Tuple[DimensionSpec, ...] = () + entity_specs: Tuple[EntitySpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () + group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () + order_by_specs: Tuple[OrderBySpec, ...] = () + time_range_constraint: Optional[TimeRangeConstraint] = None + limit: Optional[int] = None + filter_intersection: Optional[WhereFilterIntersection] = None + filter_spec_resolution_lookup: FilterSpecResolutionLookUp = FilterSpecResolutionLookUp.empty_instance() + min_max_only: bool = False + + @property + def linkable_specs(self) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet( + dimension_specs=self.dimension_specs, + time_dimension_specs=self.time_dimension_specs, + entity_specs=self.entity_specs, + group_by_metric_specs=self.group_by_metric_specs, + ) + + def with_time_range_constraint(self, time_range_constraint: Optional[TimeRangeConstraint]) -> MetricFlowQuerySpec: + """Return a query spec that's the same as self but with a different time_range_constraint.""" + return MetricFlowQuerySpec( + metric_specs=self.metric_specs, + dimension_specs=self.dimension_specs, + entity_specs=self.entity_specs, + time_dimension_specs=self.time_dimension_specs, + group_by_metric_specs=self.group_by_metric_specs, + order_by_specs=self.order_by_specs, + time_range_constraint=time_range_constraint, + limit=self.limit, + filter_intersection=self.filter_intersection, + filter_spec_resolution_lookup=self.filter_spec_resolution_lookup, + ) diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py index 573d48c6cb..acf114b07d 100644 --- a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py +++ b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py @@ -11,18 +11,17 @@ from __future__ import annotations -import itertools import logging from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum from hashlib import sha1 -from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, Sequence, Tuple, TypeVar, Union +from typing import Any, Generic, List, Optional, Sequence, Tuple, TypeVar, Union from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass from dbt_semantic_interfaces.implementations.metric import PydanticMetricTimeWindow from dbt_semantic_interfaces.naming.keywords import DUNDER, METRIC_TIME_ELEMENT_NAME -from dbt_semantic_interfaces.protocols import MetricTimeWindow, WhereFilterIntersection +from dbt_semantic_interfaces.protocols import MetricTimeWindow from dbt_semantic_interfaces.references import ( DimensionReference, EntityReference, @@ -36,20 +35,15 @@ from typing_extensions import override from metricflow_semantics.aggregation_properties import AggregationState +from metricflow_semantics.collection_helpers.dedupe import ordered_dedupe from metricflow_semantics.collection_helpers.merger import Mergeable -from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName -from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( - FilterSpecResolutionLookUp, -) from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow_semantics.sql.sql_column_type import SqlColumnType from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow_semantics.visitor import VisitorOutputT -if TYPE_CHECKING: - from metricflow_semantics.model.semantics.metric_lookup import MetricLookup - from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup +logger = logging.getLogger(__name__) def hash_items(items: Sequence[SqlColumnType]) -> str: @@ -122,12 +116,6 @@ def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT """See Visitable.""" raise NotImplementedError() - @property - @abstractmethod - def as_spec_set(self) -> InstanceSpecSet: - """Return this as the one item in a InstanceSpecSet.""" - raise NotImplementedError - SelfTypeT = TypeVar("SelfTypeT", bound="LinkableInstanceSpec") @@ -150,11 +138,6 @@ def from_name(name: str, agg_type: Optional[AggregationType] = None) -> Metadata def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_metadata_spec(self) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(metadata_specs=(self,)) - @dataclass(frozen=True) class LinkableInstanceSpec(InstanceSpec, ABC): @@ -233,65 +216,10 @@ def __hash__(self) -> int: # noqa: D105 def reference(self) -> EntityReference: # noqa: D102 return EntityReference(element_name=self.element_name) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(entity_specs=(self,)) - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_entity_spec(self) -@dataclass(frozen=True) -class GroupByMetricSpec(LinkableInstanceSpec, SerializableDataclass): - """Metric used in group by or where filter.""" - - @property - def without_first_entity_link(self) -> GroupByMetricSpec: # noqa: D102 - assert len(self.entity_links) > 0, f"Spec does not have any entity links: {self}" - return GroupByMetricSpec(element_name=self.element_name, entity_links=self.entity_links[1:]) - - @property - def without_entity_links(self) -> GroupByMetricSpec: # noqa: D102 - return GroupByMetricSpec(element_name=self.element_name, entity_links=()) - - @staticmethod - def from_name(name: str) -> GroupByMetricSpec: # noqa: D102 - structured_name = StructuredLinkableSpecName.from_name(name) - return GroupByMetricSpec( - entity_links=tuple(EntityReference(idl) for idl in structured_name.entity_link_names), - element_name=structured_name.element_name, - ) - - def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 - if not isinstance(other, GroupByMetricSpec): - return False - return self.element_name == other.element_name and self.entity_links == other.entity_links - - def __hash__(self) -> int: # noqa: D105 - return hash((self.element_name, self.entity_links)) - - @property - def reference(self) -> MetricReference: # noqa: D102 - return MetricReference(element_name=self.element_name) - - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(group_by_metric_specs=(self,)) - - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 - return visitor.visit_group_by_metric_spec(self) - - @property - def query_spec_for_source_node(self) -> MetricFlowQuerySpec: - """Query spec that can be used to build a source node for this spec in the DFP.""" - return MetricFlowQuerySpec( - metric_specs=(MetricSpec(element_name=self.element_name),), - entity_specs=tuple(EntitySpec.from_name(entity_link.element_name) for entity_link in self.entity_links), - ) - - @dataclass(frozen=True) class LinklessEntitySpec(EntitySpec, SerializableDataclass): """Similar to EntitySpec, but requires that it doesn't have entity links.""" @@ -348,11 +276,6 @@ def from_name(name: str) -> DimensionSpec: def reference(self) -> DimensionReference: # noqa: D102 return DimensionReference(element_name=self.element_name) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(dimension_specs=(self,)) - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_dimension_spec(self) @@ -474,11 +397,6 @@ def from_reference(reference: TimeDimensionReference) -> TimeDimensionSpec: """Initialize from a time dimension reference instance.""" return TimeDimensionSpec(entity_links=(), element_name=reference.element_name) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(time_dimension_specs=(self,)) - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_time_dimension_spec(self) @@ -559,13 +477,9 @@ def bucket_hash(self) -> str: return hash_items(values) @property - def linkable_specs(self) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=(), - time_dimension_specs=(TimeDimensionSpec.from_name(self.name),), - entity_specs=tuple( - LinklessEntitySpec.from_element_name(entity_name) for entity_name in self.window_groupings - ), + def linkable_specs(self) -> Sequence[LinkableInstanceSpec]: # noqa: D102 + return (TimeDimensionSpec.from_name(self.name),) + tuple( + LinklessEntitySpec.from_element_name(entity_name) for entity_name in self.window_groupings ) def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 @@ -601,11 +515,6 @@ def reference(self) -> MeasureReference: # noqa: D102 def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_measure_spec(self) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(measure_specs=(self,)) - @dataclass(frozen=True) class MetricSpec(InstanceSpec): # noqa: D101 @@ -632,11 +541,6 @@ def from_reference(reference: MetricReference) -> MetricSpec: def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_metric_spec(self) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(metric_specs=(self,)) - @property def reference(self) -> MetricReference: """Return the reference object that is used for referencing the associated metric in the manifest.""" @@ -699,318 +603,6 @@ class OrderBySpec(SerializableDataclass): # noqa: D101 descending: bool -@dataclass(frozen=True) -class FilterSpec(SerializableDataclass): # noqa: D101 - expr: str - elements: Tuple[InstanceSpec, ...] - - -@dataclass(frozen=True) -class LinkableSpecSet(Mergeable, SerializableDataclass): - """Groups linkable specs.""" - - dimension_specs: Tuple[DimensionSpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - entity_specs: Tuple[EntitySpec, ...] = () - group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () - - @property - def contains_metric_time(self) -> bool: - """Returns true if this set contains a spec referring to metric time at any grain.""" - return len(self.metric_time_specs) > 0 - - def included_agg_time_dimension_specs_for_metric( - self, metric_reference: MetricReference, metric_lookup: MetricLookup - ) -> List[TimeDimensionSpec]: - """Get the time dims included that are valid agg time dimensions for the specified metric.""" - queried_metric_time_specs = list(self.metric_time_specs) - - valid_agg_time_dimensions = metric_lookup.get_valid_agg_time_dimensions_for_metric(metric_reference) - queried_agg_time_dimension_specs = ( - list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) - + queried_metric_time_specs - ) - - return queried_agg_time_dimension_specs - - def included_agg_time_dimension_specs_for_measure( - self, measure_reference: MeasureReference, semantic_model_lookup: SemanticModelLookup - ) -> List[TimeDimensionSpec]: - """Get the time dims included that are valid agg time dimensions for the specified measure.""" - queried_metric_time_specs = list(self.metric_time_specs) - - valid_agg_time_dimensions = semantic_model_lookup.get_agg_time_dimension_specs_for_measure(measure_reference) - queried_agg_time_dimension_specs = ( - list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) - + queried_metric_time_specs - ) - - return queried_agg_time_dimension_specs - - @property - def metric_time_specs(self) -> Sequence[TimeDimensionSpec]: - """Returns any specs referring to metric time at any grain.""" - return tuple( - time_dimension_spec - for time_dimension_spec in self.time_dimension_specs - if time_dimension_spec.is_metric_time - ) - - @property - def as_tuple(self) -> Tuple[LinkableInstanceSpec, ...]: # noqa: D102 - return tuple( - itertools.chain( - self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs - ) - ) - - @override - def merge(self, other: LinkableSpecSet) -> LinkableSpecSet: - return LinkableSpecSet( - dimension_specs=self.dimension_specs + other.dimension_specs, - time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, - entity_specs=self.entity_specs + other.entity_specs, - group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, - ) - - @override - @classmethod - def empty_instance(cls) -> LinkableSpecSet: - return LinkableSpecSet() - - def dedupe(self) -> LinkableSpecSet: # noqa: D102 - # Use dictionaries to dedupe as it preserves insertion order. - - dimension_spec_dict: Dict[DimensionSpec, None] = {} - for dimension_spec in self.dimension_specs: - dimension_spec_dict[dimension_spec] = None - - time_dimension_spec_dict: Dict[TimeDimensionSpec, None] = {} - for time_dimension_spec in self.time_dimension_specs: - time_dimension_spec_dict[time_dimension_spec] = None - - entity_spec_dict: Dict[EntitySpec, None] = {} - for entity_spec in self.entity_specs: - entity_spec_dict[entity_spec] = None - - group_by_metric_spec_dict: Dict[GroupByMetricSpec, None] = {} - for group_by_metric in self.group_by_metric_specs: - group_by_metric_spec_dict[group_by_metric] = None - - return LinkableSpecSet( - dimension_specs=tuple(dimension_spec_dict.keys()), - time_dimension_specs=tuple(time_dimension_spec_dict.keys()), - entity_specs=tuple(entity_spec_dict.keys()), - group_by_metric_specs=tuple(group_by_metric_spec_dict.keys()), - ) - - def is_subset_of(self, other_set: LinkableSpecSet) -> bool: # noqa: D102 - return set(self.as_tuple).issubset(set(other_set.as_tuple)) - - @property - def as_spec_set(self) -> InstanceSpecSet: # noqa: D102 - return InstanceSpecSet( - dimension_specs=self.dimension_specs, - time_dimension_specs=self.time_dimension_specs, - entity_specs=self.entity_specs, - group_by_metric_specs=self.group_by_metric_specs, - ) - - def difference(self, other: LinkableSpecSet) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=tuple(set(self.dimension_specs) - set(other.dimension_specs)), - time_dimension_specs=tuple(set(self.time_dimension_specs) - set(other.time_dimension_specs)), - entity_specs=tuple(set(self.entity_specs) - set(other.entity_specs)), - group_by_metric_specs=tuple(set(self.group_by_metric_specs) - set(other.group_by_metric_specs)), - ) - - def __len__(self) -> int: # noqa: D105 - return len(self.dimension_specs) + len(self.time_dimension_specs) + len(self.entity_specs) - - @staticmethod - def from_specs(specs: Sequence[LinkableInstanceSpec]) -> LinkableSpecSet: # noqa: D102 - instance_spec_set = InstanceSpecSet.from_specs(specs) - return LinkableSpecSet( - dimension_specs=instance_spec_set.dimension_specs, - time_dimension_specs=instance_spec_set.time_dimension_specs, - entity_specs=instance_spec_set.entity_specs, - group_by_metric_specs=instance_spec_set.group_by_metric_specs, - ) - - -@dataclass(frozen=True) -class MetricFlowQuerySpec(SerializableDataclass): - """Specs needed for running a query.""" - - metric_specs: Tuple[MetricSpec, ...] = () - dimension_specs: Tuple[DimensionSpec, ...] = () - entity_specs: Tuple[EntitySpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () - order_by_specs: Tuple[OrderBySpec, ...] = () - time_range_constraint: Optional[TimeRangeConstraint] = None - limit: Optional[int] = None - filter_intersection: Optional[WhereFilterIntersection] = None - filter_spec_resolution_lookup: FilterSpecResolutionLookUp = FilterSpecResolutionLookUp.empty_instance() - min_max_only: bool = False - - @property - def linkable_specs(self) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=self.dimension_specs, - time_dimension_specs=self.time_dimension_specs, - entity_specs=self.entity_specs, - group_by_metric_specs=self.group_by_metric_specs, - ) - - def with_time_range_constraint(self, time_range_constraint: Optional[TimeRangeConstraint]) -> MetricFlowQuerySpec: - """Return a query spec that's the same as self but with a different time_range_constraint.""" - return MetricFlowQuerySpec( - metric_specs=self.metric_specs, - dimension_specs=self.dimension_specs, - entity_specs=self.entity_specs, - time_dimension_specs=self.time_dimension_specs, - group_by_metric_specs=self.group_by_metric_specs, - order_by_specs=self.order_by_specs, - time_range_constraint=time_range_constraint, - limit=self.limit, - filter_intersection=self.filter_intersection, - filter_spec_resolution_lookup=self.filter_spec_resolution_lookup, - ) - - -TransformOutputT = TypeVar("TransformOutputT") - - -class InstanceSpecSetTransform(Generic[TransformOutputT], ABC): - """Function to use for transforming spec sets.""" - - @abstractmethod - def transform(self, spec_set: InstanceSpecSet) -> TransformOutputT: # noqa: D102 - pass - - -@dataclass(frozen=True) -class InstanceSpecSet(Mergeable, SerializableDataclass): - """Consolidates all specs used in an instance set.""" - - metric_specs: Tuple[MetricSpec, ...] = () - measure_specs: Tuple[MeasureSpec, ...] = () - dimension_specs: Tuple[DimensionSpec, ...] = () - entity_specs: Tuple[EntitySpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () - metadata_specs: Tuple[MetadataSpec, ...] = () - - @override - def merge(self, other: InstanceSpecSet) -> InstanceSpecSet: - return InstanceSpecSet( - metric_specs=self.metric_specs + other.metric_specs, - measure_specs=self.measure_specs + other.measure_specs, - dimension_specs=self.dimension_specs + other.dimension_specs, - entity_specs=self.entity_specs + other.entity_specs, - group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, - time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, - metadata_specs=self.metadata_specs + other.metadata_specs, - ) - - @override - @classmethod - def empty_instance(cls) -> InstanceSpecSet: - return InstanceSpecSet() - - def dedupe(self) -> InstanceSpecSet: - """De-duplicates repeated elements. - - TBD: Have merge de-duplicate instead. - """ - metric_specs_deduped = [] - for metric_spec in self.metric_specs: - if metric_spec not in metric_specs_deduped: - metric_specs_deduped.append(metric_spec) - - measure_specs_deduped = [] - for measure_spec in self.measure_specs: - if measure_spec not in measure_specs_deduped: - measure_specs_deduped.append(measure_spec) - - dimension_specs_deduped = [] - for dimension_spec in self.dimension_specs: - if dimension_spec not in dimension_specs_deduped: - dimension_specs_deduped.append(dimension_spec) - - time_dimension_specs_deduped = [] - for time_dimension_spec in self.time_dimension_specs: - if time_dimension_spec not in time_dimension_specs_deduped: - time_dimension_specs_deduped.append(time_dimension_spec) - - entity_specs_deduped = [] - for entity_spec in self.entity_specs: - if entity_spec not in entity_specs_deduped: - entity_specs_deduped.append(entity_spec) - - group_by_metric_specs_deduped = [] - for group_by_metric_spec in self.group_by_metric_specs: - if group_by_metric_spec not in group_by_metric_specs_deduped: - group_by_metric_specs_deduped.append(group_by_metric_spec) - - return InstanceSpecSet( - metric_specs=tuple(metric_specs_deduped), - measure_specs=tuple(measure_specs_deduped), - dimension_specs=tuple(dimension_specs_deduped), - time_dimension_specs=tuple(time_dimension_specs_deduped), - entity_specs=tuple(entity_specs_deduped), - group_by_metric_specs=tuple(group_by_metric_specs_deduped), - ) - - @property - def linkable_specs(self) -> Sequence[LinkableInstanceSpec]: - """All linkable specs in this set.""" - return list( - itertools.chain( - self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs - ) - ) - - @property - def all_specs(self) -> Sequence[InstanceSpec]: # noqa: D102 - return tuple( - itertools.chain( - self.measure_specs, - self.dimension_specs, - self.time_dimension_specs, - self.entity_specs, - self.group_by_metric_specs, - self.metric_specs, - self.metadata_specs, - ) - ) - - def transform( # noqa: D102 - self, transform_function: InstanceSpecSetTransform[TransformOutputT] - ) -> TransformOutputT: # noqa: D102 - return transform_function.transform(self) - - @staticmethod - def from_specs(specs: Sequence[InstanceSpec]) -> InstanceSpecSet: # noqa: D102 - return InstanceSpecSet.merge_iterable(spec.as_spec_set for spec in specs) - - -@dataclass(frozen=True) -class PartitionSpecSet(SerializableDataclass): - """Grouping of the linkable specs.""" - - dimension_specs: Tuple[DimensionSpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - - -logger = logging.getLogger(__name__) - - -class WhereFilterResolutionException(Exception): # noqa: D101 - pass - - @dataclass(frozen=True) class WhereFilterSpec(Mergeable, SerializableDataclass): """Similar to the WhereFilter, but with the where_sql_template rendered and used elements extracted. @@ -1038,7 +630,7 @@ class WhereFilterSpec(Mergeable, SerializableDataclass): # quoted identifiers later. where_sql: str bind_parameters: SqlBindParameters - linkable_spec_set: LinkableSpecSet + linkable_specs: Tuple[LinkableInstanceSpec, ...] def merge(self, other: WhereFilterSpec) -> WhereFilterSpec: # noqa: D102 if self == WhereFilterSpec.empty_instance(): @@ -1053,7 +645,7 @@ def merge(self, other: WhereFilterSpec) -> WhereFilterSpec: # noqa: D102 return WhereFilterSpec( where_sql=f"({self.where_sql}) AND ({other.where_sql})", bind_parameters=self.bind_parameters.combine(other.bind_parameters), - linkable_spec_set=self.linkable_spec_set.merge(other.linkable_spec_set).dedupe(), + linkable_specs=ordered_dedupe(self.linkable_specs, other.linkable_specs), ) @classmethod @@ -1066,7 +658,7 @@ def empty_instance(cls) -> WhereFilterSpec: return WhereFilterSpec( where_sql="TRUE", bind_parameters=SqlBindParameters(), - linkable_spec_set=LinkableSpecSet(), + linkable_specs=(), ) @@ -1085,3 +677,40 @@ class JoinToTimeSpineDescription: join_type: SqlJoinType offset_window: Optional[MetricTimeWindow] offset_to_grain: Optional[TimeGranularity] + + +@dataclass(frozen=True) +class GroupByMetricSpec(LinkableInstanceSpec, SerializableDataclass): + """Metric used in group by or where filter.""" + + @property + def without_first_entity_link(self) -> GroupByMetricSpec: # noqa: D102 + assert len(self.entity_links) > 0, f"Spec does not have any entity links: {self}" + return GroupByMetricSpec(element_name=self.element_name, entity_links=self.entity_links[1:]) + + @property + def without_entity_links(self) -> GroupByMetricSpec: # noqa: D102 + return GroupByMetricSpec(element_name=self.element_name, entity_links=()) + + @staticmethod + def from_name(name: str) -> GroupByMetricSpec: # noqa: D102 + structured_name = StructuredLinkableSpecName.from_name(name) + return GroupByMetricSpec( + entity_links=tuple(EntityReference(idl) for idl in structured_name.entity_link_names), + element_name=structured_name.element_name, + ) + + def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 + if not isinstance(other, GroupByMetricSpec): + return False + return self.element_name == other.element_name and self.entity_links == other.entity_links + + def __hash__(self) -> int: # noqa: D105 + return hash((self.element_name, self.entity_links)) + + @property + def reference(self) -> MetricReference: # noqa: D102 + return MetricReference(element_name=self.element_name) + + def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 + return visitor.visit_group_by_metric_spec(self) diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_set.py b/metricflow-semantics/metricflow_semantics/specs/spec_set.py new file mode 100644 index 0000000000..8839af492d --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/spec_set.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import dataclasses +import itertools +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Generic, List, Sequence, Tuple, TypeVar + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass +from typing_extensions import override + +from metricflow_semantics.collection_helpers.merger import Mergeable +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + GroupByMetricSpec, + InstanceSpec, + InstanceSpecVisitor, + LinkableInstanceSpec, + MeasureSpec, + MetadataSpec, + MetricSpec, + TimeDimensionSpec, +) + + +@dataclass(frozen=True) +class InstanceSpecSet(Mergeable, SerializableDataclass): + """Consolidates all specs used in an instance set.""" + + metric_specs: Tuple[MetricSpec, ...] = () + measure_specs: Tuple[MeasureSpec, ...] = () + dimension_specs: Tuple[DimensionSpec, ...] = () + entity_specs: Tuple[EntitySpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () + group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () + metadata_specs: Tuple[MetadataSpec, ...] = () + + @override + def merge(self, other: InstanceSpecSet) -> InstanceSpecSet: + return InstanceSpecSet( + metric_specs=self.metric_specs + other.metric_specs, + measure_specs=self.measure_specs + other.measure_specs, + dimension_specs=self.dimension_specs + other.dimension_specs, + entity_specs=self.entity_specs + other.entity_specs, + group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, + time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, + metadata_specs=self.metadata_specs + other.metadata_specs, + ) + + @override + @classmethod + def empty_instance(cls) -> InstanceSpecSet: + return InstanceSpecSet() + + def dedupe(self) -> InstanceSpecSet: + """De-duplicates repeated elements. + + TBD: Have merge de-duplicate instead. + """ + metric_specs_deduped = [] + for metric_spec in self.metric_specs: + if metric_spec not in metric_specs_deduped: + metric_specs_deduped.append(metric_spec) + + measure_specs_deduped = [] + for measure_spec in self.measure_specs: + if measure_spec not in measure_specs_deduped: + measure_specs_deduped.append(measure_spec) + + dimension_specs_deduped = [] + for dimension_spec in self.dimension_specs: + if dimension_spec not in dimension_specs_deduped: + dimension_specs_deduped.append(dimension_spec) + + time_dimension_specs_deduped = [] + for time_dimension_spec in self.time_dimension_specs: + if time_dimension_spec not in time_dimension_specs_deduped: + time_dimension_specs_deduped.append(time_dimension_spec) + + entity_specs_deduped = [] + for entity_spec in self.entity_specs: + if entity_spec not in entity_specs_deduped: + entity_specs_deduped.append(entity_spec) + + group_by_metric_specs_deduped = [] + for group_by_metric_spec in self.group_by_metric_specs: + if group_by_metric_spec not in group_by_metric_specs_deduped: + group_by_metric_specs_deduped.append(group_by_metric_spec) + + return InstanceSpecSet( + metric_specs=tuple(metric_specs_deduped), + measure_specs=tuple(measure_specs_deduped), + dimension_specs=tuple(dimension_specs_deduped), + time_dimension_specs=tuple(time_dimension_specs_deduped), + entity_specs=tuple(entity_specs_deduped), + group_by_metric_specs=tuple(group_by_metric_specs_deduped), + ) + + @property + def linkable_specs(self) -> Sequence[LinkableInstanceSpec]: + """All linkable specs in this set.""" + return list( + itertools.chain( + self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs + ) + ) + + @property + def all_specs(self) -> Sequence[InstanceSpec]: # noqa: D102 + return tuple( + itertools.chain( + self.measure_specs, + self.dimension_specs, + self.time_dimension_specs, + self.entity_specs, + self.group_by_metric_specs, + self.metric_specs, + self.metadata_specs, + ) + ) + + def transform( # noqa: D102 + self, transform_function: InstanceSpecSetTransform[TransformOutputT] + ) -> TransformOutputT: + return transform_function.transform(self) + + @property + def metric_time_specs(self) -> Sequence[TimeDimensionSpec]: + """Returns any specs referring to metric time at any grain.""" + return tuple( + time_dimension_spec + for time_dimension_spec in self.time_dimension_specs + if time_dimension_spec.is_metric_time + ) + + @staticmethod + def create_from_specs(specs: Sequence[InstanceSpec]) -> InstanceSpecSet: # noqa: D102 + return group_specs_by_type(specs) + + +TransformOutputT = TypeVar("TransformOutputT") + + +class InstanceSpecSetTransform(Generic[TransformOutputT], ABC): + """Function to use for transforming spec sets.""" + + @abstractmethod + def transform(self, spec_set: InstanceSpecSet) -> TransformOutputT: # noqa: D102 + pass + + +@dataclass +class _GroupSpecByTypeVisitor(InstanceSpecVisitor[None]): + """Groups a spec by type into an `InstanceSpecSet`.""" + + metric_specs: List[MetricSpec] = dataclasses.field(default_factory=list) + measure_specs: List[MeasureSpec] = dataclasses.field(default_factory=list) + dimension_specs: List[DimensionSpec] = dataclasses.field(default_factory=list) + entity_specs: List[EntitySpec] = dataclasses.field(default_factory=list) + time_dimension_specs: List[TimeDimensionSpec] = dataclasses.field(default_factory=list) + group_by_metric_specs: List[GroupByMetricSpec] = dataclasses.field(default_factory=list) + metadata_specs: List[MetadataSpec] = dataclasses.field(default_factory=list) + + @override + def visit_measure_spec(self, measure_spec: MeasureSpec) -> None: + self.measure_specs.append(measure_spec) + + @override + def visit_dimension_spec(self, dimension_spec: DimensionSpec) -> None: + self.dimension_specs.append(dimension_spec) + + @override + def visit_time_dimension_spec(self, time_dimension_spec: TimeDimensionSpec) -> None: + self.time_dimension_specs.append(time_dimension_spec) + + @override + def visit_entity_spec(self, entity_spec: EntitySpec) -> None: + self.entity_specs.append(entity_spec) + + @override + def visit_group_by_metric_spec(self, group_by_metric_spec: GroupByMetricSpec) -> None: + self.group_by_metric_specs.append(group_by_metric_spec) + + @override + def visit_metric_spec(self, metric_spec: MetricSpec) -> None: + self.metric_specs.append(metric_spec) + + @override + def visit_metadata_spec(self, metadata_spec: MetadataSpec) -> None: + self.metadata_specs.append(metadata_spec) + + +def group_specs_by_type(specs: Sequence[InstanceSpec]) -> InstanceSpecSet: + """Groups a sequence of specs by type.""" + grouper = _GroupSpecByTypeVisitor() + for spec in specs: + spec.accept(grouper) + + return InstanceSpecSet( + metric_specs=tuple(grouper.metric_specs), + measure_specs=tuple(grouper.measure_specs), + dimension_specs=tuple(grouper.dimension_specs), + entity_specs=tuple(grouper.entity_specs), + time_dimension_specs=tuple(grouper.time_dimension_specs), + group_by_metric_specs=tuple(grouper.group_by_metric_specs), + metadata_specs=tuple(grouper.metadata_specs), + ) + + +def group_spec_by_type(spec: InstanceSpec) -> InstanceSpecSet: + """Similar to group_specs_by_type() but for a single spec.""" + return group_specs_by_type((spec,)) diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py b/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py index 991cacafc2..a36aa0c2b0 100644 --- a/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py +++ b/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py @@ -2,7 +2,7 @@ from typing import Set -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, InstanceSpecSetTransform +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform class ToElementNameSet(InstanceSpecSetTransform[Set[str]]): diff --git a/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py b/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py index d56676d912..fe82785d7c 100644 --- a/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py +++ b/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py @@ -13,7 +13,7 @@ ) from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver from metricflow_semantics.specs.rendered_spec_tracker import RenderedSpecTracker -from metricflow_semantics.specs.spec_classes import LinkableSpecSet, WhereFilterSpec +from metricflow_semantics.specs.spec_classes import WhereFilterSpec from metricflow_semantics.specs.where_filter_dimension import WhereFilterDimensionFactory from metricflow_semantics.specs.where_filter_entity import WhereFilterEntityFactory from metricflow_semantics.specs.where_filter_metric import WhereFilterMetricFactory @@ -103,7 +103,7 @@ def create_from_where_filter_intersection( # noqa: D102 WhereFilterSpec( where_sql=where_sql, bind_parameters=SqlBindParameters(), - linkable_spec_set=LinkableSpecSet.from_specs(rendered_spec_tracker.rendered_specs), + linkable_specs=tuple(rendered_spec_tracker.rendered_specs), ) ) diff --git a/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py b/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py index 4354c6eed4..32c5fbd178 100644 --- a/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py +++ b/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py @@ -16,7 +16,8 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableSpecSet +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet +from metricflow_semantics.specs.spec_set import InstanceSpecSet logger = logging.getLogger(__name__) diff --git a/metricflow-semantics/tests_metricflow_semantics/api/__init__.py b/metricflow-semantics/tests_metricflow_semantics/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/tests_metricflow_semantics/api/v0_1/__init__.py b/metricflow-semantics/tests_metricflow_semantics/api/v0_1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/tests_metricflow_semantics/api/v0_1/test_saved_query_dependency_resolver.py b/metricflow-semantics/tests_metricflow_semantics/api/v0_1/test_saved_query_dependency_resolver.py new file mode 100644 index 0000000000..d8514db27e --- /dev/null +++ b/metricflow-semantics/tests_metricflow_semantics/api/v0_1/test_saved_query_dependency_resolver.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import pytest +from dbt_semantic_interfaces.implementations.semantic_manifest import ( + PydanticSemanticManifest, +) +from dbt_semantic_interfaces.references import ( + SemanticModelReference, +) +from metricflow_semantics.api.v0_1.saved_query_dependency_resolver import SavedQueryDependencyResolver + + +@pytest.fixture(scope="session") +def resolver( # noqa: D103 + simple_semantic_manifest: PydanticSemanticManifest, +) -> SavedQueryDependencyResolver: + return SavedQueryDependencyResolver(simple_semantic_manifest) + + +def test_saved_query_dependency_resolver(resolver: SavedQueryDependencyResolver) -> None: # noqa: D103 + dependency_set = resolver.resolve_dependencies("p0_booking") + assert tuple(dependency_set.semantic_model_references) == ( + SemanticModelReference(semantic_model_name="bookings_source"), + SemanticModelReference(semantic_model_name="listings_latest"), + ) diff --git a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py index 925e5ed8b7..2dd2acafd8 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py @@ -11,6 +11,7 @@ import itertools +import pytest from dbt_semantic_interfaces.protocols.dimension import DimensionType from dbt_semantic_interfaces.references import ( DimensionReference, @@ -20,14 +21,22 @@ TimeDimensionReference, ) from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element import ( + ElementPathKey, LinkableDimension, - LinkableElementProperty, + LinkableElementType, LinkableEntity, LinkableMetric, SemanticModelJoinPathElement, ) from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet +from metricflow_semantics.specs.patterns.entity_link_pattern import ( + EntityLinkPattern, + EntityLinkPatternParameterSet, + ParameterSetField, +) +from metricflow_semantics.specs.spec_classes import TimeDimensionSpec from more_itertools import bucket AMBIGUOUS_NAME = "ambiguous" @@ -475,3 +484,137 @@ def test_only_unique_path_keys() -> None: _base_entity.path_key: (_base_entity,) }, "Got unexpected value for unique entity sets!" assert unique_path_keys.path_key_to_linkable_metrics == dict(), "Found unexpected unique values for metric sets!" + + +@pytest.fixture(scope="session") +def linkable_set() -> LinkableElementSet: # noqa: D103 + entity_0 = EntityReference("entity_0") + entity_0_source = SemanticModelReference("entity_0_source") + entity_1 = EntityReference("entity_1") + entity_1_source = SemanticModelReference("entity_1_source") + entity_2 = EntityReference("entity_2") + entity_2_source = SemanticModelReference("entity_2_source") + entity_3 = EntityReference("entity_3") + entity_3_source = SemanticModelReference("entity_3_source") + + return LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="dimension_element", + entity_links=(entity_0,), + element_type=LinkableElementType.DIMENSION, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("dimension_source"), + element_name="dimension_element", + dimension_type=DimensionType.CATEGORICAL, + entity_links=(entity_0,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_0_source, + join_on_entity=entity_0, + ), + ), + properties=frozenset(), + time_granularity=None, + date_part=None, + ), + ), + ElementPathKey( + element_name="time_dimension_element", + entity_links=(entity_1,), + element_type=LinkableElementType.TIME_DIMENSION, + time_granularity=TimeGranularity.DAY, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("time_dimension_source"), + element_name="time_dimension_element", + dimension_type=DimensionType.TIME, + entity_links=(entity_1,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_1_source, + join_on_entity=entity_1, + ), + ), + properties=frozenset(), + time_granularity=TimeGranularity.DAY, + date_part=None, + ), + ), + }, + path_key_to_linkable_entities={ + ElementPathKey( + element_name="entity_element", + entity_links=(entity_2,), + element_type=LinkableElementType.ENTITY, + ): ( + LinkableEntity( + semantic_model_origin=SemanticModelReference("entity_source"), + element_name="entity_element", + entity_links=(entity_2,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_2_source, + join_on_entity=entity_2, + ), + ), + properties=frozenset(), + ), + ) + }, + path_key_to_linkable_metrics={ + ElementPathKey( + element_name="metric_element", + entity_links=(entity_3,), + element_type=LinkableElementType.METRIC, + ): ( + LinkableMetric( + join_by_semantic_model=SemanticModelReference("metric_source"), + element_name="metric_element", + entity_links=(entity_3,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_3_source, + join_on_entity=entity_3, + ), + ), + properties=frozenset(), + ), + ) + }, + ) + + +def test_derived_semantic_models(linkable_set: LinkableElementSet) -> None: + """Tests that the semantic models in the element set are returned via `derived_from_semantic_models`.""" + assert tuple(linkable_set.derived_from_semantic_models) == ( + SemanticModelReference(semantic_model_name="dimension_source"), + SemanticModelReference(semantic_model_name="entity_0_source"), + SemanticModelReference(semantic_model_name="entity_1_source"), + SemanticModelReference(semantic_model_name="entity_2_source"), + SemanticModelReference(semantic_model_name="entity_3_source"), + SemanticModelReference(semantic_model_name="entity_source"), + SemanticModelReference(semantic_model_name="metric_source"), + SemanticModelReference(semantic_model_name="time_dimension_source"), + ) + + +def test_filter_by_pattern(linkable_set: LinkableElementSet) -> None: + """Tests that the specs produced by the set are properly filtered by spec patterns.""" + spec_pattern = EntityLinkPattern( + EntityLinkPatternParameterSet( + fields_to_compare=(ParameterSetField.ENTITY_LINKS,), + element_name=None, + entity_links=(EntityReference("entity_1"),), + ) + ) + + assert tuple(linkable_set.filter_by_spec_patterns((spec_pattern,)).specs) == ( + TimeDimensionSpec( + element_name="time_dimension_element", + entity_links=(EntityReference("entity_1"),), + time_granularity=TimeGranularity.DAY, + date_part=None, + ), + ) diff --git a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py index cce2e574c7..bc7c201a1e 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py @@ -10,9 +10,9 @@ MetricReference, SemanticModelReference, ) +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.model.semantics.linkable_element import ( - LinkableElementProperty, SemanticModelJoinPath, SemanticModelJoinPathElement, ) @@ -20,6 +20,7 @@ ValidLinkableSpecResolver, ) from metricflow_semantics.model.semantics.semantic_model_join_evaluator import MAX_JOIN_HOPS +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.snapshot_helpers import ( assert_linkable_element_set_snapshot_equal, @@ -193,15 +194,16 @@ def test_linkable_element_set_as_spec_set( double up on the .as_spec_set calls here. Yes, this is lazy. No, I don't care to make another helper to do snapshot comparisons on LinkableSpecSets. """ - linkable_spec_set = simple_model_spec_resolver.get_linkable_element_set_for_measure( - MeasureReference(element_name="listings"), - with_any_of=LinkableElementProperty.all_properties(), - without_any_of=frozenset({}), - ).as_spec_set - + linkable_spec_set = InstanceSpecSet.create_from_specs( + simple_model_spec_resolver.get_linkable_element_set_for_measure( + MeasureReference(element_name="listings"), + with_any_of=LinkableElementProperty.all_properties(), + without_any_of=frozenset({}), + ).specs + ) assert_spec_set_snapshot_equal( request=request, mf_test_configuration=mf_test_configuration, set_id="set0", - spec_set=linkable_spec_set.as_spec_set, + spec_set=linkable_spec_set, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py b/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py index 69f8c5ec0c..b0649ba89f 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py @@ -6,7 +6,7 @@ from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest from dbt_semantic_interfaces.references import EntityReference, MeasureReference, MetricReference -from metricflow_semantics.model.semantics.linkable_element import LinkableElementProperty +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.metric_lookup import MetricLookup from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration @@ -79,7 +79,7 @@ def test_local_linked_elements_for_metric( # noqa: D103 with_any_property=frozenset({LinkableElementProperty.LOCAL_LINKED}), without_any_property=frozenset({LinkableElementProperty.DERIVED_TIME_GRANULARITY}), ) - sorted_specs = sorted(linkable_elements.as_spec_set.as_tuple, key=lambda x: x.qualified_name) + sorted_specs = sorted(linkable_elements.specs, key=lambda x: x.qualified_name) assert_object_snapshot_equal( request=request, mf_test_configuration=mf_test_configuration, @@ -140,7 +140,7 @@ def test_linkable_elements_for_no_metrics_query( LinkableElementProperty.DERIVED_TIME_GRANULARITY, } ) - sorted_specs = sorted(linkable_elements.as_spec_set.as_tuple, key=lambda x: x.qualified_name) + sorted_specs = sorted(linkable_elements.specs, key=lambda x: x.qualified_name) assert_object_snapshot_equal( request=request, mf_test_configuration=mf_test_configuration, diff --git a/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py b/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py index 5e234f104c..de5bf4b61e 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py @@ -19,10 +19,20 @@ DimensionReference, EntityReference, MetricReference, + SemanticModelReference, TimeDimensionReference, ) +from dbt_semantic_interfaces.type_enums import DimensionType from dbt_semantic_interfaces.type_enums.date_part import DatePart from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.model.semantics.linkable_element import ( + ElementPathKey, + LinkableDimension, + LinkableElementType, + LinkableEntity, + LinkableMetric, +) +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_location import WhereFilterLocation from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( @@ -34,12 +44,12 @@ from metricflow_semantics.query.group_by_item.resolution_path import MetricFlowQueryResolutionPath from metricflow_semantics.query.issues.issues_base import MetricFlowQueryResolutionIssueSet from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, GroupByMetricSpec, LinkableInstanceSpec, - LinkableSpecSet, TimeDimensionSpec, WhereFilterSpec, ) @@ -51,7 +61,9 @@ def create_spec_lookup( - call_parameter_set: CallParameterSet, resolved_spec: LinkableInstanceSpec + call_parameter_set: CallParameterSet, + resolved_spec: LinkableInstanceSpec, + resolved_linkable_element_set: LinkableElementSet, ) -> FilterSpecResolutionLookUp: """Create a FilterSpecResolutionLookUp where the call_parameter_set maps to resolved_spec.""" return FilterSpecResolutionLookUp( @@ -63,7 +75,7 @@ def create_spec_lookup( ), filter_location_path=MetricFlowQueryResolutionPath.empty_instance(), where_filter_intersection=create_where_filter_intersection("Dimension('dummy__dimension')"), - resolved_spec=resolved_spec, + resolved_linkable_element_set=resolved_linkable_element_set, issue_set=MetricFlowQueryResolutionIssueSet.empty_instance(), spec_pattern=ObjectBuilderNamingScheme().spec_pattern("Dimension('dummy__dimension')"), object_builder_str="Dimension('dummy__dimension')", @@ -89,6 +101,28 @@ def test_dimension_in_filter( # noqa: D103 dimension_reference=DimensionReference("country_latest"), ), resolved_spec=DimensionSpec(element_name="country_latest", entity_links=(EntityReference("listing"),)), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="country_latest", + element_type=LinkableElementType.DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=None, + date_part=None, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.CATEGORICAL, + element_name="country_latest", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + time_granularity=None, + date_part=None, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, @@ -97,7 +131,7 @@ def test_dimension_in_filter( # noqa: D103 assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] assert where_filter_spec.where_sql == "listing__country_latest = 'US'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=( DimensionSpec(element_name="country_latest", entity_links=(EntityReference(element_name="listing"),)), ), @@ -115,29 +149,51 @@ def test_dimension_in_filter_with_grain( # noqa: D103 spec_resolution_lookup=create_spec_lookup( call_parameter_set=TimeDimensionCallParameterSet( entity_path=(EntityReference("listing"),), - time_dimension_reference=TimeDimensionReference("country_latest"), + time_dimension_reference=TimeDimensionReference("created_at"), time_granularity=TimeGranularity.WEEK, ), resolved_spec=TimeDimensionSpec( - element_name="country_latest", + element_name="created_at", entity_links=(EntityReference("listing"),), time_granularity=TimeGranularity.WEEK, ), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="created_at", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.WEEK, + date_part=None, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("listings_source"), + dimension_type=DimensionType.TIME, + element_name="created_at", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.WEEK, + date_part=None, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, filter_intersection=create_where_filter_intersection( - "{{ Dimension('listing__country_latest').grain('WEEK') }} = 'US'" + "{{ Dimension('listing__created_at').grain('WEEK') }} = 'US'" ), ) assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] - assert where_filter_spec.where_sql == "listing__country_latest__week = 'US'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert where_filter_spec.where_sql == "listing__created_at__week = 'US'" + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( - element_name="country_latest", + element_name="created_at", entity_links=(EntityReference(element_name="listing"),), time_granularity=TimeGranularity.WEEK, ), @@ -163,6 +219,28 @@ def test_time_dimension_in_filter( # noqa: D103 entity_links=(EntityReference("listing"),), time_granularity=TimeGranularity.MONTH, ), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="created_at", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.MONTH, + date_part=None, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("listings_source"), + dimension_type=DimensionType.CATEGORICAL, + element_name="created_at", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.MONTH, + date_part=None, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, @@ -173,7 +251,7 @@ def test_time_dimension_in_filter( # noqa: D103 assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] assert where_filter_spec.where_sql == "listing__created_at__month = '2020-01-01'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -204,6 +282,28 @@ def test_date_part_in_filter( # noqa: D103 time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR, ), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="metric_time", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.TIME, + element_name="metric_time", + entity_links=(), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, @@ -214,7 +314,7 @@ def test_date_part_in_filter( # noqa: D103 assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] assert where_filter_spec.where_sql == "metric_time__extract_year = '2020'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -248,11 +348,27 @@ def resolved_spec_lookup() -> FilterSpecResolutionLookUp: where_filter_intersection=create_where_filter_intersection( "TimeDimension('metric_time', 'week', 'year')" ), - resolved_spec=TimeDimensionSpec( - element_name="metric_time", - entity_links=(), - time_granularity=TimeGranularity.WEEK, - date_part=DatePart.YEAR, + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="metric_time", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(), + time_granularity=TimeGranularity.WEEK, + date_part=DatePart.YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.TIME, + element_name="metric_time", + entity_links=(), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.WEEK, + date_part=DatePart.YEAR, + ), + ) + } ), spec_pattern=ObjectBuilderNamingScheme().spec_pattern("Dimension('dummy__dimension')"), issue_set=MetricFlowQueryResolutionIssueSet.empty_instance(), @@ -284,7 +400,7 @@ def test_date_part_and_grain_in_filter( # noqa: D103 ).create_from_where_filter(EXAMPLE_FILTER_LOCATION, where_filter) assert where_filter_spec.where_sql == "metric_time__extract_year = '2020'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -321,7 +437,7 @@ def test_date_part_less_than_grain_in_filter( # noqa: D103 ).create_from_where_filter(EXAMPLE_FILTER_LOCATION, where_filter) assert where_filter_spec.where_sql == "metric_time__extract_day = '2020'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -352,11 +468,30 @@ def test_entity_in_filter( # noqa: D103 entity_reference=EntityReference("user"), ), resolved_spec=EntitySpec(element_name="user", entity_links=(EntityReference("listing"),)), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_entities={ + ElementPathKey( + element_name="user", + element_type=LinkableElementType.ENTITY, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ): ( + LinkableEntity( + semantic_model_origin=SemanticModelReference("bookings"), + element_name="user", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + ), + ) + } + ), ), ).create_from_where_filter(filter_location=EXAMPLE_FILTER_LOCATION, where_filter=where_filter) assert where_filter_spec.where_sql == "listing__user == 'example_user_id'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=(), entity_specs=(EntitySpec(element_name="user", entity_links=(EntityReference(element_name="listing"),)),), @@ -379,11 +514,30 @@ def test_metric_in_filter( # noqa: D103 metric_reference=MetricReference("bookings"), ), resolved_spec=group_by_metric_spec, + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_metrics={ + ElementPathKey( + element_name="bookings", + element_type=LinkableElementType.METRIC, + entity_links=(EntityReference("listing"),), + time_granularity=None, + date_part=None, + ): ( + LinkableMetric( + join_by_semantic_model=SemanticModelReference("bookings"), + element_name="bookings", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + ), + ) + } + ), ), ).create_from_where_filter(filter_location=EXAMPLE_FILTER_LOCATION, where_filter=where_filter) assert where_filter_spec.where_sql == "listing__bookings > 2" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=(), entity_specs=(), @@ -411,11 +565,27 @@ def get_spec(dimension: str) -> WhereFilterSpec: ), filter_location_path=MetricFlowQueryResolutionPath(()), where_filter_intersection=PydanticWhereFilterIntersection(where_filters=[where_filter]), - resolved_spec=TimeDimensionSpec( - element_name=METRIC_TIME_ELEMENT_NAME, - entity_links=(), - time_granularity=TimeGranularity.WEEK, - date_part=DatePart.YEAR, + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name=METRIC_TIME_ELEMENT_NAME, + element_type=LinkableElementType.DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.TIME, + element_name=METRIC_TIME_ELEMENT_NAME, + entity_links=(), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.WEEK, + date_part=DatePart.YEAR, + ), + ) + } ), spec_pattern=ObjectBuilderNamingScheme().spec_pattern("Dimension('dummy__dimension')"), issue_set=MetricFlowQueryResolutionIssueSet.empty_instance(), diff --git a/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py b/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py index ffb7af27ff..ad35fa2748 100644 --- a/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py +++ b/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py @@ -8,7 +8,8 @@ from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.query.group_by_item.group_by_item_resolver import GroupByItemResolver from metricflow_semantics.query.group_by_item.resolution_dag.dag import GroupByItemResolutionDag -from metricflow_semantics.specs.spec_classes import LinkableSpecSet +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.snapshot_helpers import assert_linkable_spec_set_snapshot_equal @@ -36,5 +37,5 @@ def test_available_group_by_items( # noqa: D103 request=request, mf_test_configuration=mf_test_configuration, set_id="set0", - spec_set=LinkableSpecSet.from_specs(result.specs), + spec_set=LinkableSpecSet.create_from_spec_set(group_specs_by_type(result.specs)), ) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt index 4f335a25f2..a151d9b75c 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoCommonItemsInParents( @@ -11,11 +12,24 @@ GroupByItemResolution( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -36,11 +50,24 @@ GroupByItemResolution( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt index ca5af91d22..9778252906 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoCommonItemsInParents( @@ -8,11 +9,24 @@ GroupByItemResolution( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -31,11 +45,24 @@ GroupByItemResolution( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt index d3552b1753..057b0dfe18 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt @@ -1 +1,19 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=DAY)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=DAY), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=DAY, + ): ( + LinkableDimension( + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=DAY, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt index a0fcdd41f3..21234f22f4 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoMatchingItemsForMeasure( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt index 4eac17bbba..a3774cbe89 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoMatchingItemsForMeasure( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt index e9dded7b67..e4e1f4ce43 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -60,7 +78,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt index 1c8462ad59..063f27b837 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt @@ -22,6 +22,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -40,11 +41,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -65,11 +79,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt index d64f856af1..8e6b7a0ca8 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt @@ -22,6 +22,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -41,11 +42,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -68,11 +82,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt index ed4de4e885..60b59c8d79 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt index 65513b2d5f..cc155e70a5 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt index db27d0f522..7d77ed1031 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt index 7d961e2d45..58383f8a08 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt @@ -22,6 +22,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -40,11 +41,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -65,11 +79,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt index c9fd95b464..df8b92b453 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt index d8f991770d..2cc7ddb651 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt @@ -25,6 +25,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -42,11 +43,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -65,11 +79,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt index d6cbeec091..08165c1164 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt @@ -25,7 +25,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt index b8c3f57c16..30917c4241 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/test_specs.py b/metricflow-semantics/tests_metricflow_semantics/test_specs.py index 41ffb13ba7..8413e1c20f 100644 --- a/metricflow-semantics/tests_metricflow_semantics/test_specs.py +++ b/metricflow-semantics/tests_metricflow_semantics/test_specs.py @@ -10,13 +10,13 @@ EntitySpec, GroupByMetricSpec, InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, LinklessEntitySpec, MeasureSpec, MetricSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet @pytest.fixture diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py index 6f87ce0636..c48457ae52 100644 --- a/metricflow/dataflow/builder/dataflow_plan_builder.py +++ b/metricflow/dataflow/builder/dataflow_plan_builder.py @@ -34,18 +34,18 @@ FilterSpecResolutionLookUp, ) from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( ConstantPropertySpec, CumulativeMeasureDescription, EntitySpec, - InstanceSpecSet, + GroupByMetricSpec, JoinToTimeSpineDescription, LinkableInstanceSpec, - LinkableSpecSet, LinklessEntitySpec, MeasureSpec, MetadataSpec, - MetricFlowQuerySpec, MetricInputMeasureSpec, MetricSpec, NonAdditiveDimensionSpec, @@ -53,6 +53,7 @@ TimeDimensionSpec, WhereFilterSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet, group_specs_by_type from metricflow_semantics.specs.where_filter_transform import WhereSpecFactory from metricflow_semantics.sql.sql_join_type import SqlJoinType @@ -308,7 +309,7 @@ def _build_aggregated_conversion_node( ) filtered_unaggregated_base_node = FilterElementsNode( parent_node=unaggregated_base_measure_node, - include_specs=InstanceSpecSet.from_specs(required_local_specs) + include_specs=group_specs_by_type(required_local_specs) .merge(base_required_linkable_specs.as_spec_set) .dedupe(), ) @@ -812,6 +813,14 @@ def _build_measure_spec_properties(self, measure_specs: Sequence[MeasureSpec]) - non_additive_dimension_spec=non_additive_dimension_spec, ) + def _query_spec_for_source_node(self, group_by_metric_spec: GroupByMetricSpec) -> MetricFlowQuerySpec: + return MetricFlowQuerySpec( + metric_specs=(MetricSpec(element_name=group_by_metric_spec.element_name),), + entity_specs=tuple( + EntitySpec.from_name(entity_link.element_name) for entity_link in group_by_metric_spec.entity_links + ), + ) + def _find_dataflow_recipe( self, linkable_spec_set: LinkableSpecSet, @@ -827,7 +836,7 @@ def _find_dataflow_recipe( # MetricGroupBy source nodes could be extremely large (and potentially slow). candidate_nodes_for_right_side_of_join += [ self._build_query_output_node( - query_spec=group_by_metric_spec.query_spec_for_source_node, for_group_by_source_node=True + query_spec=self._query_spec_for_source_node(group_by_metric_spec), for_group_by_source_node=True ) for group_by_metric_spec in linkable_spec_set.group_by_metric_specs ] @@ -1208,9 +1217,11 @@ def __get_required_and_extraneous_linkable_specs( """ linkable_spec_sets_to_merge: List[LinkableSpecSet] = [] for filter_spec in filter_specs: - linkable_spec_sets_to_merge.append(filter_spec.linkable_spec_set) + linkable_spec_sets_to_merge.append(LinkableSpecSet.create_from_specs(filter_spec.linkable_specs)) if non_additive_dimension_spec: - linkable_spec_sets_to_merge.append(non_additive_dimension_spec.linkable_specs) + linkable_spec_sets_to_merge.append( + LinkableSpecSet.create_from_specs(non_additive_dimension_spec.linkable_specs) + ) extraneous_linkable_specs = LinkableSpecSet.merge_iterable(linkable_spec_sets_to_merge).dedupe() required_linkable_specs = queried_linkable_specs.merge(extraneous_linkable_specs).dedupe() @@ -1343,7 +1354,7 @@ def _build_aggregated_measure_from_measure_source_node( filtered_measure_source_node = FilterElementsNode( parent_node=join_to_time_spine_node or time_range_node or measure_recipe.source_node, include_specs=InstanceSpecSet(measure_specs=(measure_spec,)).merge( - InstanceSpecSet.from_specs(measure_recipe.required_local_linkable_specs), + group_specs_by_type(measure_recipe.required_local_linkable_specs), ), ) diff --git a/metricflow/dataflow/builder/node_evaluator.py b/metricflow/dataflow/builder/node_evaluator.py index 858f320059..7e6b36e066 100644 --- a/metricflow/dataflow/builder/node_evaluator.py +++ b/metricflow/dataflow/builder/node_evaluator.py @@ -27,11 +27,10 @@ from metricflow_semantics.model.semantics.semantic_model_join_evaluator import SemanticModelJoinEvaluator from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.specs.spec_classes import ( - InstanceSpecSet, LinkableInstanceSpec, - LinkableSpecSet, LinklessEntitySpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow.dataflow.builder.node_data_set import DataflowPlanNodeOutputDataSetResolver @@ -120,7 +119,7 @@ def join_description(self) -> JoinDescription: ] ) filtered_node_to_join = FilterElementsNode( - parent_node=self.node_to_join, include_specs=InstanceSpecSet.from_specs(include_specs) + parent_node=self.node_to_join, include_specs=group_specs_by_type(include_specs) ) return JoinDescription( @@ -201,7 +200,7 @@ def _find_joinable_candidate_nodes_that_can_satisfy_linkable_specs( for right_node in self._nodes_available_for_joins: # If right node is time spine source node, use cross join. if right_node == self._time_spine_node: - needed_metric_time_specs = LinkableSpecSet.from_specs(needed_linkable_specs).metric_time_specs + needed_metric_time_specs = group_specs_by_type(needed_linkable_specs).metric_time_specs candidates_for_join.append( JoinLinkableInstancesRecipe( node_to_join=right_node, diff --git a/metricflow/dataflow/builder/partitions.py b/metricflow/dataflow/builder/partitions.py index 0d65e7f76f..510a9fa4ec 100644 --- a/metricflow/dataflow/builder/partitions.py +++ b/metricflow/dataflow/builder/partitions.py @@ -5,12 +5,12 @@ from typing import List, Sequence, Tuple from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup +from metricflow_semantics.specs.partition_spec_set import PartitionSpecSet from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - InstanceSpecSet, - PartitionSpecSet, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow.dataset.dataset_classes import DataSet diff --git a/metricflow/dataflow/nodes/filter_elements.py b/metricflow/dataflow/nodes/filter_elements.py index 3d110e7e9f..a20ef5c146 100644 --- a/metricflow/dataflow/nodes/filter_elements.py +++ b/metricflow/dataflow/nodes/filter_elements.py @@ -5,7 +5,7 @@ from metricflow_semantics.dag.id_prefix import IdPrefix, StaticIdPrefix from metricflow_semantics.dag.mf_dag import DisplayedProperty from metricflow_semantics.mf_logging.pretty_print import mf_pformat -from metricflow_semantics.specs.spec_classes import InstanceSpecSet +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.visitor import VisitorOutputT from metricflow.dataflow.dataflow_plan import BaseOutput, DataflowPlanNode, DataflowPlanNodeVisitor diff --git a/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py b/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py index e5b3c7da47..3aedbedf71 100644 --- a/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py +++ b/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py @@ -1,6 +1,6 @@ from __future__ import annotations -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, InstanceSpecSetTransform +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform class MatchingLinkableSpecsTransform(InstanceSpecSetTransform[bool]): diff --git a/metricflow/engine/metricflow_engine.py b/metricflow/engine/metricflow_engine.py index eb0e011623..60612b8d8a 100644 --- a/metricflow/engine/metricflow_engine.py +++ b/metricflow/engine/metricflow_engine.py @@ -17,10 +17,10 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.model.semantics.linkable_element import ( LinkableDimension, - LinkableElementProperty, ) from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName @@ -31,7 +31,8 @@ from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver from metricflow_semantics.specs.query_param_implementations import SavedQueryParameter -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, MetricFlowQuerySpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.time.time_source import TimeSource from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder @@ -469,7 +470,7 @@ def _create_execution_plan(self, mf_query_request: MetricFlowQueryRequest) -> Me time_constraint_end=mf_query_request.time_constraint_end, order_by_names=mf_query_request.order_by_names, order_by_parameters=mf_query_request.order_by, - ) + ).query_spec else: query_spec = self._query_parser.parse_and_validate_query( metric_names=mf_query_request.metric_names, diff --git a/metricflow/plan_conversion/dataflow_to_sql.py b/metricflow/plan_conversion/dataflow_to_sql.py index c4cacfd692..772dae2ea4 100644 --- a/metricflow/plan_conversion/dataflow_to_sql.py +++ b/metricflow/plan_conversion/dataflow_to_sql.py @@ -32,12 +32,12 @@ ) from metricflow_semantics.specs.spec_classes import ( GroupByMetricSpec, - InstanceSpecSet, MeasureSpec, MetadataSpec, MetricSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow_semantics.time.time_constants import ISO8601_PYTHON_FORMAT @@ -874,7 +874,7 @@ def visit_where_constraint_node(self, node: WhereConstraintNode) -> SqlDataSet: column_associations_in_where_sql: Sequence[ColumnAssociation] = CreateColumnAssociations( column_association_resolver=self._column_association_resolver - ).transform(spec_set=node.where.linkable_spec_set.as_spec_set) + ).transform(spec_set=InstanceSpecSet.create_from_specs(node.where.linkable_specs)) return SqlDataSet( instance_set=output_instance_set, diff --git a/metricflow/plan_conversion/instance_converters.py b/metricflow/plan_conversion/instance_converters.py index 1139d07831..39fab0d316 100644 --- a/metricflow/plan_conversion/instance_converters.py +++ b/metricflow/plan_conversion/instance_converters.py @@ -35,13 +35,13 @@ EntitySpec, GroupByMetricSpec, InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, LinklessEntitySpec, MeasureSpec, MetricInputMeasureSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from more_itertools import bucket from metricflow.dataflow.nodes.join_to_base import ValidityWindowJoinDescription diff --git a/metricflow/plan_conversion/node_processor.py b/metricflow/plan_conversion/node_processor.py index f58756542d..8499ca47b7 100644 --- a/metricflow/plan_conversion/node_processor.py +++ b/metricflow/plan_conversion/node_processor.py @@ -9,7 +9,8 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantics.semantic_model_join_evaluator import MAX_JOIN_HOPS from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec, LinklessEntitySpec +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec, LinklessEntitySpec +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.specs.spec_set_transforms import ToElementNameSet from metricflow_semantics.sql.sql_join_type import SqlJoinType @@ -223,7 +224,7 @@ def _get_candidates_nodes_for_multi_hop( specs = data_set_of_second_node_that_can_be_joined.instance_set.spec_set filtered_joinable_node = FilterElementsNode( parent_node=second_node_that_could_be_joined, - include_specs=InstanceSpecSet.from_specs( + include_specs=group_specs_by_type( specs.dimension_specs + specs.entity_specs + specs.time_dimension_specs ), ) diff --git a/metricflow/plan_conversion/spec_transforms.py b/metricflow/plan_conversion/spec_transforms.py index 9cdab81984..b2e86ac0c7 100644 --- a/metricflow/plan_conversion/spec_transforms.py +++ b/metricflow/plan_conversion/spec_transforms.py @@ -3,10 +3,7 @@ from typing import List, Sequence from metricflow_semantics.specs.column_assoc import ColumnAssociation, ColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import ( - InstanceSpecSet, - InstanceSpecSetTransform, -) +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform from metricflow.plan_conversion.select_column_gen import SelectColumnSet from metricflow.plan_conversion.sql_expression_builders import make_coalesced_expr diff --git a/metricflow/validation/data_warehouse_model_validator.py b/metricflow/validation/data_warehouse_model_validator.py index 7d43e34e11..99a35829c2 100644 --- a/metricflow/validation/data_warehouse_model_validator.py +++ b/metricflow/validation/data_warehouse_model_validator.py @@ -30,7 +30,8 @@ ) from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec, MeasureSpec +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec, MeasureSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow.dataflow.builder.node_data_set import DataflowPlanNodeOutputDataSetResolver diff --git a/scripts/ci_tests/metricflow_package_test.py b/scripts/ci_tests/metricflow_package_test.py index 0d85c3f53a..bcc1a5f627 100644 --- a/scripts/ci_tests/metricflow_package_test.py +++ b/scripts/ci_tests/metricflow_package_test.py @@ -13,10 +13,10 @@ from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, ) diff --git a/tests_metricflow/dataflow/builder/test_cyclic_join.py b/tests_metricflow/dataflow/builder/test_cyclic_join.py index 10eb9758b9..07c2a13c16 100644 --- a/tests_metricflow/dataflow/builder/test_cyclic_join.py +++ b/tests_metricflow/dataflow/builder/test_cyclic_join.py @@ -6,9 +6,9 @@ import pytest from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.references import EntityReference +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py b/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py index 24c60296e7..61541e08f1 100644 --- a/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py +++ b/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py @@ -12,10 +12,10 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, OrderBySpec, TimeDimensionSpec, diff --git a/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py b/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py index 56aca18e6c..88f959e50c 100644 --- a/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py +++ b/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py @@ -6,7 +6,8 @@ from _pytest.fixtures import FixtureRequest from metricflow_semantics.dag.id_prefix import StaticIdPrefix from metricflow_semantics.dag.mf_dag import DagId -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, MeasureSpec +from metricflow_semantics.specs.spec_classes import MeasureSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.snapshot_helpers import assert_plan_snapshot_text_equal diff --git a/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py b/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py index 1cb33ee108..0a6789cb41 100644 --- a/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py +++ b/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py @@ -8,10 +8,10 @@ from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/examples/test_node_sql.py b/tests_metricflow/examples/test_node_sql.py index cd6e517563..d3c96fffc6 100644 --- a/tests_metricflow/examples/test_node_sql.py +++ b/tests_metricflow/examples/test_node_sql.py @@ -8,7 +8,8 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, TimeDimensionReference, TimeDimensionSpec +from metricflow_semantics.specs.spec_classes import TimeDimensionReference, TimeDimensionSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow.dataflow.builder.node_data_set import DataflowPlanNodeOutputDataSetResolver from metricflow.dataflow.nodes.filter_elements import FilterElementsNode diff --git a/tests_metricflow/fixtures/setup_fixtures.py b/tests_metricflow/fixtures/setup_fixtures.py index 3d441d9c83..c346d2c821 100644 --- a/tests_metricflow/fixtures/setup_fixtures.py +++ b/tests_metricflow/fixtures/setup_fixtures.py @@ -24,9 +24,6 @@ logger = logging.getLogger(__name__) -# from metricflow.test.time.configurable_time_source import ConfigurableTimeSource - - DISPLAY_GRAPHS_CLI_FLAG = "--display-graphs" USE_PERSISTENT_SOURCE_SCHEMA_CLI_FLAG = "--use-persistent-source-schema" diff --git a/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py b/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py index fc3138738a..2531bcd295 100644 --- a/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py +++ b/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py @@ -4,9 +4,9 @@ from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py b/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py index a7771e4c97..f0fcf7b7ea 100644 --- a/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py +++ b/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py @@ -6,7 +6,8 @@ from dbt_semantic_interfaces.references import EntityReference from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import DimensionSpec, MetricFlowQuerySpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import DimensionSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder diff --git a/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py b/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py index ae53da1509..065e19eb02 100644 --- a/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py +++ b/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py @@ -5,7 +5,8 @@ import pytest from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.references import TimeDimensionReference -from metricflow_semantics.specs.spec_classes import MetricFlowQuerySpec, MetricSpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import MetricSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.metric_time_dimension import MTD_SPEC_DAY diff --git a/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py b/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py index fbd9f7eb10..c07ade0bb6 100644 --- a/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py +++ b/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py @@ -5,7 +5,8 @@ from metricflow_semantics.instances import InstanceSet from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, MeasureSpec, MetricInputMeasureSpec +from metricflow_semantics.specs.spec_classes import MeasureSpec, MetricInputMeasureSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow.plan_conversion.instance_converters import ( CreateSelectColumnsWithMeasuresAggregated, diff --git a/tests_metricflow/plan_conversion/test_dataflow_to_execution.py b/tests_metricflow/plan_conversion/test_dataflow_to_execution.py index 925c7b22e7..5197fe1d63 100644 --- a/tests_metricflow/plan_conversion/test_dataflow_to_execution.py +++ b/tests_metricflow/plan_conversion/test_dataflow_to_execution.py @@ -4,10 +4,10 @@ from _pytest.fixtures import FixtureRequest from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py b/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py index a007d72765..5c1e4c51bf 100644 --- a/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py +++ b/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py @@ -13,13 +13,11 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - InstanceSpecSet, - LinkableSpecSet, LinklessEntitySpec, MeasureSpec, - MetricFlowQuerySpec, MetricInputMeasureSpec, MetricSpec, NonAdditiveDimensionSpec, @@ -27,6 +25,7 @@ TimeDimensionSpec, WhereFilterSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration @@ -192,14 +191,12 @@ def test_filter_with_where_constraint_node( where_constraint=WhereFilterSpec( where_sql="booking__ds__day = '2020-01-01'", bind_parameters=SqlBindParameters(), - linkable_spec_set=LinkableSpecSet( - time_dimension_specs=( - TimeDimensionSpec( - element_name="ds", - entity_links=(EntityReference(element_name="booking"),), - time_granularity=TimeGranularity.DAY, - ), - ) + linkable_specs=( + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="booking"),), + time_granularity=TimeGranularity.DAY, + ), ), ), ) diff --git a/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py b/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py index a96e1f532c..f13355485d 100644 --- a/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py +++ b/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py @@ -13,7 +13,8 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import EntityReference, MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import EntityReference, MetricSpec, TimeDimensionSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.metric_time_dimension import MTD_SPEC_MONTH diff --git a/tests_metricflow/query_rendering/test_derived_metric_rendering.py b/tests_metricflow/query_rendering/test_derived_metric_rendering.py index 509ee6c253..284e39f999 100644 --- a/tests_metricflow/query_rendering/test_derived_metric_rendering.py +++ b/tests_metricflow/query_rendering/test_derived_metric_rendering.py @@ -12,8 +12,8 @@ from metricflow_semantics.naming.dunder_scheme import DunderNamingScheme from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py b/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py index ae903803c6..f820b8bbe4 100644 --- a/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py +++ b/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py @@ -12,9 +12,9 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from metricflow_semantics.query.query_parser import MetricFlowQueryParser +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py b/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py index b4bfd33a7c..b40622a7b0 100644 --- a/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py +++ b/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py @@ -10,8 +10,8 @@ from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.type_enums.date_part import DatePart from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/query_rendering/test_metric_time_without_metrics.py b/tests_metricflow/query_rendering/test_metric_time_without_metrics.py index 55f32370e5..e28988d435 100644 --- a/tests_metricflow/query_rendering/test_metric_time_without_metrics.py +++ b/tests_metricflow/query_rendering/test_metric_time_without_metrics.py @@ -7,7 +7,8 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from metricflow_semantics.filters.time_constraint import TimeRangeConstraint -from metricflow_semantics.specs.spec_classes import DimensionSpec, MetricFlowQuerySpec, TimeDimensionSpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import DimensionSpec, TimeDimensionSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.metric_time_dimension import MTD_SPEC_DAY diff --git a/tests_metricflow/query_rendering/test_query_rendering.py b/tests_metricflow/query_rendering/test_query_rendering.py index fe32116bfb..b07e349dd9 100644 --- a/tests_metricflow/query_rendering/test_query_rendering.py +++ b/tests_metricflow/query_rendering/test_query_rendering.py @@ -18,9 +18,9 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/query_rendering/test_time_spine_join_rendering.py b/tests_metricflow/query_rendering/test_time_spine_join_rendering.py index 58a07c9f2e..774e0095d8 100644 --- a/tests_metricflow/query_rendering/test_time_spine_join_rendering.py +++ b/tests_metricflow/query_rendering/test_time_spine_join_rendering.py @@ -11,8 +11,8 @@ import pytest from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml index bf661d834b..eb2573ef1e 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml @@ -26,23 +26,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml index da62fd9e41..d74f407e48 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml @@ -31,23 +31,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml index 55f773dbc8..68a39bcd80 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -31,19 +26,12 @@ - - - - - - - - - - - - - + + + + + + @@ -68,12 +56,10 @@ - - - - - - + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml index ebd82f88a6..e4e3677e4f 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml @@ -12,28 +12,22 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -46,23 +40,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + @@ -135,28 +123,22 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -169,23 +151,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml index 01e049e74c..7b35eae487 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml @@ -12,29 +12,23 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + @@ -47,23 +41,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml index 1f539c3a57..75a683ec01 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml @@ -17,8 +17,17 @@ - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml index f0a5328a55..41bf9e5c26 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml @@ -5,16 +5,22 @@ - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -26,8 +32,17 @@ - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml index 489ec19a70..1a30add9bb 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -33,12 +28,10 @@ - - - - - - + + + + @@ -58,19 +51,12 @@ - - - - - - - - - - - - - + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml index 4e94e82589..944c59cc46 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -36,12 +31,10 @@ - - - - - - + + + + @@ -64,12 +57,10 @@ - - - - - - + + + + @@ -120,12 +111,10 @@ - - - - - - + + + + @@ -147,12 +136,10 @@ - - - - - - + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml index b134b228aa..0955aebc17 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml @@ -5,28 +5,22 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -43,23 +37,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml index b3ba6c0a1e..2eaabb15d5 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -39,19 +34,12 @@ - - - - - - - - - - - - - + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml index 7a9e744ad3..07b2aa8b7a 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml @@ -5,51 +5,39 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml index 2171124c84..6b6d49f158 100644 --- a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml +++ b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml @@ -47,23 +47,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml index 0ea28d8b15..f625d9c134 100644 --- a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml +++ b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml @@ -47,23 +47,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + +