From 9d73ca5d4f6b29250ba1b5f1e5515f73150ddfba Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Thu, 25 Apr 2024 15:43:13 -0700 Subject: [PATCH 01/14] Move `LinkableElementProperty.` This was moved to address circular-dependency issues. --- .../model/linkable_element_property.py | 43 +++++++++++++++++++ .../model/semantics/linkable_element.py | 40 +---------------- .../model/semantics/linkable_element_set.py | 2 +- .../model/semantics/linkable_spec_resolver.py | 2 +- .../model/semantics/metric_lookup.py | 2 +- .../candidate_push_down/push_down_visitor.py | 2 +- .../semantics/test_linkable_element_set.py | 2 +- .../semantics/test_linkable_spec_resolver.py | 2 +- .../model/test_semantic_model_container.py | 2 +- metricflow/engine/metricflow_engine.py | 2 +- 10 files changed, 52 insertions(+), 47 deletions(-) create mode 100644 metricflow-semantics/metricflow_semantics/model/linkable_element_property.py diff --git a/metricflow-semantics/metricflow_semantics/model/linkable_element_property.py b/metricflow-semantics/metricflow_semantics/model/linkable_element_property.py new file mode 100644 index 0000000000..668be83974 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/model/linkable_element_property.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from enum import Enum +from typing import FrozenSet + + +class LinkableElementProperty(Enum): + """The properties associated with a valid linkable element. + + Local means an element that is defined within the same semantic model as the measure. This definition is used + throughout the related classes. + """ + + # A local element as per above definition. + LOCAL = "local" + # A local dimension that is prefixed with a local primary entity. + LOCAL_LINKED = "local_linked" + # An element that was joined to the measure semantic model by an entity. + JOINED = "joined" + # An element that was joined to the measure semantic model by joining multiple semantic models. + MULTI_HOP = "multi_hop" + # A time dimension that is a version of a time dimension in a semantic model, but at a different granularity. + DERIVED_TIME_GRANULARITY = "derived_time_granularity" + # Refers to an entity, not a dimension. + ENTITY = "entity" + # See metric_time in DataSet + METRIC_TIME = "metric_time" + # Refers to a metric, not a dimension. + METRIC = "metric" + + @staticmethod + def all_properties() -> FrozenSet[LinkableElementProperty]: # noqa: D102 + return frozenset( + { + LinkableElementProperty.LOCAL, + LinkableElementProperty.LOCAL_LINKED, + LinkableElementProperty.JOINED, + LinkableElementProperty.MULTI_HOP, + LinkableElementProperty.DERIVED_TIME_GRANULARITY, + LinkableElementProperty.METRIC_TIME, + LinkableElementProperty.METRIC, + } + ) diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py index 7167a784b1..aafde71315 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py @@ -10,6 +10,7 @@ from dbt_semantic_interfaces.type_enums.date_part import DatePart from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.specs.spec_classes import EntityReference @@ -40,45 +41,6 @@ def is_dimension_type(self) -> bool: return assert_values_exhausted(element_type) -class LinkableElementProperty(Enum): - """The properties associated with a valid linkable element. - - Local means an element that is defined within the same semantic model as the measure. This definition is used - throughout the related classes. - """ - - # A local element as per above definition. - LOCAL = "local" - # A local dimension that is prefixed with a local primary entity. - LOCAL_LINKED = "local_linked" - # An element that was joined to the measure semantic model by an entity. - JOINED = "joined" - # An element that was joined to the measure semantic model by joining multiple semantic models. - MULTI_HOP = "multi_hop" - # A time dimension that is a version of a time dimension in a semantic model, but at a different granularity. - DERIVED_TIME_GRANULARITY = "derived_time_granularity" - # Refers to an entity, not a dimension. - ENTITY = "entity" - # See metric_time in DataSet - METRIC_TIME = "metric_time" - # Refers to a metric, not a dimension. - METRIC = "metric" - - @staticmethod - def all_properties() -> FrozenSet[LinkableElementProperty]: # noqa: D102 - return frozenset( - { - LinkableElementProperty.LOCAL, - LinkableElementProperty.LOCAL_LINKED, - LinkableElementProperty.JOINED, - LinkableElementProperty.MULTI_HOP, - LinkableElementProperty.DERIVED_TIME_GRANULARITY, - LinkableElementProperty.METRIC_TIME, - LinkableElementProperty.METRIC, - } - ) - - @dataclass(frozen=True) class ElementPathKey: """A key that can uniquely identify an element and the joins used to realize the element.""" diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py index 3f3d6d09e5..820043b983 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py @@ -4,10 +4,10 @@ from dataclasses import dataclass, field from typing import Dict, FrozenSet, List, Sequence, Set, Tuple +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element import ( ElementPathKey, LinkableDimension, - LinkableElementProperty, LinkableElementType, LinkableEntity, LinkableMetric, diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py index ce81cf8699..171c1897e0 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py @@ -22,10 +22,10 @@ from metricflow_semantics.errors.error_classes import UnknownMetricLinkingError from metricflow_semantics.mf_logging.pretty_print import mf_pformat +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element import ( ElementPathKey, LinkableDimension, - LinkableElementProperty, LinkableElementType, LinkableEntity, LinkableMetric, diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py b/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py index 0809a811ba..51c95ab88c 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/metric_lookup.py @@ -9,7 +9,7 @@ from dbt_semantic_interfaces.references import MeasureReference, MetricReference from metricflow_semantics.errors.error_classes import DuplicateMetricError, MetricNotFoundError, NonExistentMeasureError -from metricflow_semantics.model.semantics.linkable_element import LinkableElementProperty +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.model.semantics.linkable_spec_resolver import ( ValidLinkableSpecResolver, diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py index 7abca90066..bdc7e5933d 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py @@ -11,8 +11,8 @@ from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat, mf_pformat_many +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup -from metricflow_semantics.model.semantics.linkable_element import LinkableElementProperty from metricflow_semantics.query.group_by_item.candidate_push_down.group_by_item_candidate import GroupByItemCandidateSet from metricflow_semantics.query.group_by_item.resolution_dag.resolution_nodes.base_node import ( GroupByItemResolutionNode, diff --git a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py index 925e5ed8b7..f352052e67 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py @@ -20,9 +20,9 @@ TimeDimensionReference, ) from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element import ( LinkableDimension, - LinkableElementProperty, LinkableEntity, LinkableMetric, SemanticModelJoinPathElement, diff --git a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py index cce2e574c7..a67a0cfaee 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py @@ -10,9 +10,9 @@ MetricReference, SemanticModelReference, ) +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.model.semantics.linkable_element import ( - LinkableElementProperty, SemanticModelJoinPath, SemanticModelJoinPathElement, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py b/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py index 69f8c5ec0c..3eb4aee33f 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py @@ -6,7 +6,7 @@ from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest from dbt_semantic_interfaces.references import EntityReference, MeasureReference, MetricReference -from metricflow_semantics.model.semantics.linkable_element import LinkableElementProperty +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.metric_lookup import MetricLookup from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/metricflow/engine/metricflow_engine.py b/metricflow/engine/metricflow_engine.py index eb0e011623..543f02b84f 100644 --- a/metricflow/engine/metricflow_engine.py +++ b/metricflow/engine/metricflow_engine.py @@ -17,10 +17,10 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.model.semantics.linkable_element import ( LinkableDimension, - LinkableElementProperty, ) from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName From 0fa7db773ae6385d6d4fec6cc782509153e2cf70 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Thu, 25 Apr 2024 15:51:53 -0700 Subject: [PATCH 02/14] Move query-related specs to separate file. This was moved to address circular-dependency issues. Note: `GroupByMetricSpec` had to be moved back in a later commit. --- .../metricflow_semantics/instances.py | 2 +- .../model/semantics/linkable_element_set.py | 2 +- .../query/query_parser.py | 2 +- .../query/query_resolution.py | 2 +- .../query/query_resolver.py | 2 +- .../dunder_column_association_resolver.py | 2 +- .../specs/group_by_metric_spec.py | 69 +++++++++++++ .../metricflow_semantics/specs/query_spec.py | 65 ++++++++++++ .../specs/spec_classes.py | 98 +------------------ .../model/test_where_filter_spec.py | 2 +- .../naming/conftest.py | 2 +- .../test_object_builder_naming_scheme.py | 2 +- .../patterns/test_entity_link_pattern.py | 2 +- .../specs/patterns/test_typed_patterns.py | 2 +- .../tests_metricflow_semantics/test_specs.py | 2 +- .../dataflow/builder/dataflow_plan_builder.py | 2 +- metricflow/engine/metricflow_engine.py | 3 +- metricflow/plan_conversion/dataflow_to_sql.py | 2 +- .../plan_conversion/instance_converters.py | 2 +- scripts/ci_tests/metricflow_package_test.py | 2 +- .../dataflow/builder/test_cyclic_join.py | 2 +- .../builder/test_dataflow_plan_builder.py | 2 +- .../source_scan/test_source_scan_optimizer.py | 2 +- .../test_conversion_metrics_to_sql.py | 2 +- .../test_distinct_values_to_sql.py | 3 +- .../test_metric_time_dimension_to_sql.py | 3 +- .../test_dataflow_to_execution.py | 2 +- .../test_dataflow_to_sql_plan.py | 2 +- .../test_cumulative_metric_rendering.py | 3 +- .../test_derived_metric_rendering.py | 2 +- .../test_fill_nulls_with_rendering.py | 2 +- .../test_granularity_date_part_rendering.py | 2 +- .../test_metric_time_without_metrics.py | 3 +- .../query_rendering/test_query_rendering.py | 2 +- .../test_time_spine_join_rendering.py | 2 +- 35 files changed, 173 insertions(+), 128 deletions(-) create mode 100644 metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py create mode 100644 metricflow-semantics/metricflow_semantics/specs/query_spec.py diff --git a/metricflow-semantics/metricflow_semantics/instances.py b/metricflow-semantics/metricflow_semantics/instances.py index 2e6c4499c3..99f24e33fa 100644 --- a/metricflow-semantics/metricflow_semantics/instances.py +++ b/metricflow-semantics/metricflow_semantics/instances.py @@ -11,10 +11,10 @@ from metricflow_semantics.aggregation_properties import AggregationState from metricflow_semantics.specs.column_assoc import ColumnAssociation +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, InstanceSpec, InstanceSpecSet, MeasureSpec, diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py index 820043b983..449db647af 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py @@ -12,10 +12,10 @@ LinkableEntity, LinkableMetric, ) +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, LinkableSpecSet, TimeDimensionSpec, ) diff --git a/metricflow-semantics/metricflow_semantics/query/query_parser.py b/metricflow-semantics/metricflow_semantics/query/query_parser.py index 63486319ce..d92b53ef2b 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_parser.py +++ b/metricflow-semantics/metricflow_semantics/query/query_parser.py @@ -54,10 +54,10 @@ from metricflow_semantics.specs.patterns.base_time_grain import BaseTimeGrainPattern from metricflow_semantics.specs.patterns.metric_time_pattern import MetricTimePattern from metricflow_semantics.specs.patterns.none_date_part import NoneDatePartPattern +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( InstanceSpec, InstanceSpecSet, - MetricFlowQuerySpec, TimeDimensionSpec, ) from metricflow_semantics.time.time_granularity import ( diff --git a/metricflow-semantics/metricflow_semantics/query/query_resolution.py b/metricflow-semantics/metricflow_semantics/query/query_resolution.py index f715c048fd..5111495975 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_resolution.py +++ b/metricflow-semantics/metricflow_semantics/query/query_resolution.py @@ -13,7 +13,7 @@ from metricflow_semantics.query.group_by_item.resolution_dag.dag import GroupByItemResolutionDag from metricflow_semantics.query.issues.issues_base import MetricFlowQueryResolutionIssueSet from metricflow_semantics.query.resolver_inputs.base_resolver_inputs import MetricFlowQueryResolverInput -from metricflow_semantics.specs.spec_classes import MetricFlowQuerySpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec @dataclass(frozen=True) diff --git a/metricflow-semantics/metricflow_semantics/query/query_resolver.py b/metricflow-semantics/metricflow_semantics/query/query_resolver.py index 45f4e0ff39..ac5b49dbdf 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_resolver.py +++ b/metricflow-semantics/metricflow_semantics/query/query_resolver.py @@ -52,11 +52,11 @@ from metricflow_semantics.query.suggestion_generator import QueryItemSuggestionGenerator from metricflow_semantics.query.validation_rules.query_validator import PostResolutionQueryValidator from metricflow_semantics.specs.patterns.match_list_pattern import MatchListSpecPattern +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( InstanceSpec, LinkableInstanceSpec, LinkableSpecSet, - MetricFlowQuerySpec, MetricSpec, OrderBySpec, ) diff --git a/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py b/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py index d217067646..6351a4b572 100644 --- a/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py +++ b/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py @@ -7,10 +7,10 @@ ColumnAssociationResolver, SingleColumnCorrelationKey, ) +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, InstanceSpec, InstanceSpecVisitor, MeasureSpec, diff --git a/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py b/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py new file mode 100644 index 0000000000..f6758e8a03 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass +from dbt_semantic_interfaces.references import EntityReference, MetricReference +from typing_extensions import override + +from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import ( + EntitySpec, + InstanceSpecSet, + InstanceSpecVisitor, + LinkableInstanceSpec, + MetricSpec, +) +from metricflow_semantics.visitor import VisitorOutputT + + +@dataclass(frozen=True) +class GroupByMetricSpec(LinkableInstanceSpec, SerializableDataclass): + """Metric used in group by or where filter.""" + + @property + def without_first_entity_link(self) -> GroupByMetricSpec: # noqa: D102 + assert len(self.entity_links) > 0, f"Spec does not have any entity links: {self}" + return GroupByMetricSpec(element_name=self.element_name, entity_links=self.entity_links[1:]) + + @property + def without_entity_links(self) -> GroupByMetricSpec: # noqa: D102 + return GroupByMetricSpec(element_name=self.element_name, entity_links=()) + + @staticmethod + def from_name(name: str) -> GroupByMetricSpec: # noqa: D102 + structured_name = StructuredLinkableSpecName.from_name(name) + return GroupByMetricSpec( + entity_links=tuple(EntityReference(idl) for idl in structured_name.entity_link_names), + element_name=structured_name.element_name, + ) + + def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 + if not isinstance(other, GroupByMetricSpec): + return False + return self.element_name == other.element_name and self.entity_links == other.entity_links + + def __hash__(self) -> int: # noqa: D105 + return hash((self.element_name, self.entity_links)) + + @property + def reference(self) -> MetricReference: # noqa: D102 + return MetricReference(element_name=self.element_name) + + @property + @override + def as_spec_set(self) -> InstanceSpecSet: + return InstanceSpecSet(group_by_metric_specs=(self,)) + + def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 + return visitor.visit_group_by_metric_spec(self) + + @property + def query_spec_for_source_node(self) -> MetricFlowQuerySpec: + """Query spec that can be used to build a source node for this spec in the DFP.""" + return MetricFlowQuerySpec( + metric_specs=(MetricSpec(element_name=self.element_name),), + entity_specs=tuple(EntitySpec.from_name(entity_link.element_name) for entity_link in self.entity_links), + ) diff --git a/metricflow-semantics/metricflow_semantics/specs/query_spec.py b/metricflow-semantics/metricflow_semantics/specs/query_spec.py new file mode 100644 index 0000000000..7098aacbb1 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/query_spec.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import typing +from dataclasses import dataclass +from typing import Optional, Tuple + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass +from dbt_semantic_interfaces.protocols import WhereFilterIntersection + +from metricflow_semantics.filters.time_constraint import TimeRangeConstraint +from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( + FilterSpecResolutionLookUp, +) +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + LinkableSpecSet, + MetricSpec, + OrderBySpec, + TimeDimensionSpec, +) + +if typing.TYPE_CHECKING: + from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec + + +@dataclass(frozen=True) +class MetricFlowQuerySpec(SerializableDataclass): + """Specs needed for running a query.""" + + metric_specs: Tuple[MetricSpec, ...] = () + dimension_specs: Tuple[DimensionSpec, ...] = () + entity_specs: Tuple[EntitySpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () + group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () + order_by_specs: Tuple[OrderBySpec, ...] = () + time_range_constraint: Optional[TimeRangeConstraint] = None + limit: Optional[int] = None + filter_intersection: Optional[WhereFilterIntersection] = None + filter_spec_resolution_lookup: FilterSpecResolutionLookUp = FilterSpecResolutionLookUp.empty_instance() + min_max_only: bool = False + + @property + def linkable_specs(self) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet( + dimension_specs=self.dimension_specs, + time_dimension_specs=self.time_dimension_specs, + entity_specs=self.entity_specs, + group_by_metric_specs=self.group_by_metric_specs, + ) + + def with_time_range_constraint(self, time_range_constraint: Optional[TimeRangeConstraint]) -> MetricFlowQuerySpec: + """Return a query spec that's the same as self but with a different time_range_constraint.""" + return MetricFlowQuerySpec( + metric_specs=self.metric_specs, + dimension_specs=self.dimension_specs, + entity_specs=self.entity_specs, + time_dimension_specs=self.time_dimension_specs, + group_by_metric_specs=self.group_by_metric_specs, + order_by_specs=self.order_by_specs, + time_range_constraint=time_range_constraint, + limit=self.limit, + filter_intersection=self.filter_intersection, + filter_spec_resolution_lookup=self.filter_spec_resolution_lookup, + ) diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py index 573d48c6cb..4cf1f58a6c 100644 --- a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py +++ b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py @@ -22,7 +22,7 @@ from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass from dbt_semantic_interfaces.implementations.metric import PydanticMetricTimeWindow from dbt_semantic_interfaces.naming.keywords import DUNDER, METRIC_TIME_ELEMENT_NAME -from dbt_semantic_interfaces.protocols import MetricTimeWindow, WhereFilterIntersection +from dbt_semantic_interfaces.protocols import MetricTimeWindow from dbt_semantic_interfaces.references import ( DimensionReference, EntityReference, @@ -37,11 +37,7 @@ from metricflow_semantics.aggregation_properties import AggregationState from metricflow_semantics.collection_helpers.merger import Mergeable -from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName -from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( - FilterSpecResolutionLookUp, -) from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow_semantics.sql.sql_column_type import SqlColumnType from metricflow_semantics.sql.sql_join_type import SqlJoinType @@ -50,6 +46,7 @@ if TYPE_CHECKING: from metricflow_semantics.model.semantics.metric_lookup import MetricLookup from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup + from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec def hash_items(items: Sequence[SqlColumnType]) -> str: @@ -242,56 +239,6 @@ def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT return visitor.visit_entity_spec(self) -@dataclass(frozen=True) -class GroupByMetricSpec(LinkableInstanceSpec, SerializableDataclass): - """Metric used in group by or where filter.""" - - @property - def without_first_entity_link(self) -> GroupByMetricSpec: # noqa: D102 - assert len(self.entity_links) > 0, f"Spec does not have any entity links: {self}" - return GroupByMetricSpec(element_name=self.element_name, entity_links=self.entity_links[1:]) - - @property - def without_entity_links(self) -> GroupByMetricSpec: # noqa: D102 - return GroupByMetricSpec(element_name=self.element_name, entity_links=()) - - @staticmethod - def from_name(name: str) -> GroupByMetricSpec: # noqa: D102 - structured_name = StructuredLinkableSpecName.from_name(name) - return GroupByMetricSpec( - entity_links=tuple(EntityReference(idl) for idl in structured_name.entity_link_names), - element_name=structured_name.element_name, - ) - - def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 - if not isinstance(other, GroupByMetricSpec): - return False - return self.element_name == other.element_name and self.entity_links == other.entity_links - - def __hash__(self) -> int: # noqa: D105 - return hash((self.element_name, self.entity_links)) - - @property - def reference(self) -> MetricReference: # noqa: D102 - return MetricReference(element_name=self.element_name) - - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(group_by_metric_specs=(self,)) - - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 - return visitor.visit_group_by_metric_spec(self) - - @property - def query_spec_for_source_node(self) -> MetricFlowQuerySpec: - """Query spec that can be used to build a source node for this spec in the DFP.""" - return MetricFlowQuerySpec( - metric_specs=(MetricSpec(element_name=self.element_name),), - entity_specs=tuple(EntitySpec.from_name(entity_link.element_name) for entity_link in self.entity_links), - ) - - @dataclass(frozen=True) class LinklessEntitySpec(EntitySpec, SerializableDataclass): """Similar to EntitySpec, but requires that it doesn't have entity links.""" @@ -838,47 +785,6 @@ def from_specs(specs: Sequence[LinkableInstanceSpec]) -> LinkableSpecSet: # noq ) -@dataclass(frozen=True) -class MetricFlowQuerySpec(SerializableDataclass): - """Specs needed for running a query.""" - - metric_specs: Tuple[MetricSpec, ...] = () - dimension_specs: Tuple[DimensionSpec, ...] = () - entity_specs: Tuple[EntitySpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () - order_by_specs: Tuple[OrderBySpec, ...] = () - time_range_constraint: Optional[TimeRangeConstraint] = None - limit: Optional[int] = None - filter_intersection: Optional[WhereFilterIntersection] = None - filter_spec_resolution_lookup: FilterSpecResolutionLookUp = FilterSpecResolutionLookUp.empty_instance() - min_max_only: bool = False - - @property - def linkable_specs(self) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=self.dimension_specs, - time_dimension_specs=self.time_dimension_specs, - entity_specs=self.entity_specs, - group_by_metric_specs=self.group_by_metric_specs, - ) - - def with_time_range_constraint(self, time_range_constraint: Optional[TimeRangeConstraint]) -> MetricFlowQuerySpec: - """Return a query spec that's the same as self but with a different time_range_constraint.""" - return MetricFlowQuerySpec( - metric_specs=self.metric_specs, - dimension_specs=self.dimension_specs, - entity_specs=self.entity_specs, - time_dimension_specs=self.time_dimension_specs, - group_by_metric_specs=self.group_by_metric_specs, - order_by_specs=self.order_by_specs, - time_range_constraint=time_range_constraint, - limit=self.limit, - filter_intersection=self.filter_intersection, - filter_spec_resolution_lookup=self.filter_spec_resolution_lookup, - ) - - TransformOutputT = TypeVar("TransformOutputT") diff --git a/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py b/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py index 5e234f104c..6e7d7cc42b 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py @@ -34,10 +34,10 @@ from metricflow_semantics.query.group_by_item.resolution_path import MetricFlowQueryResolutionPath from metricflow_semantics.query.issues.issues_base import MetricFlowQueryResolutionIssueSet from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, LinkableInstanceSpec, LinkableSpecSet, TimeDimensionSpec, diff --git a/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py b/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py index 3dac120add..2f89474db0 100644 --- a/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py +++ b/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py @@ -6,10 +6,10 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py b/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py index d5abdfcb27..58de22b339 100644 --- a/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py +++ b/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py @@ -7,10 +7,10 @@ from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py index cfb1c4da16..af5d270e11 100644 --- a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py +++ b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py @@ -9,6 +9,7 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.patterns.entity_link_pattern import ( EntityLinkPattern, EntityLinkPatternParameterSet, @@ -17,7 +18,6 @@ from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py index 4c16d8f016..0bdb611568 100644 --- a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py +++ b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py @@ -18,6 +18,7 @@ ) from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.patterns.typed_patterns import ( DimensionPattern, EntityPattern, @@ -27,7 +28,6 @@ from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/test_specs.py b/metricflow-semantics/tests_metricflow_semantics/test_specs.py index 41ffb13ba7..eac5e6a951 100644 --- a/metricflow-semantics/tests_metricflow_semantics/test_specs.py +++ b/metricflow-semantics/tests_metricflow_semantics/test_specs.py @@ -4,11 +4,11 @@ import pytest from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, EntitySpec, - GroupByMetricSpec, InstanceSpec, InstanceSpecSet, LinkableInstanceSpec, diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py index 6f87ce0636..009e823c50 100644 --- a/metricflow/dataflow/builder/dataflow_plan_builder.py +++ b/metricflow/dataflow/builder/dataflow_plan_builder.py @@ -34,6 +34,7 @@ FilterSpecResolutionLookUp, ) from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( ConstantPropertySpec, CumulativeMeasureDescription, @@ -45,7 +46,6 @@ LinklessEntitySpec, MeasureSpec, MetadataSpec, - MetricFlowQuerySpec, MetricInputMeasureSpec, MetricSpec, NonAdditiveDimensionSpec, diff --git a/metricflow/engine/metricflow_engine.py b/metricflow/engine/metricflow_engine.py index 543f02b84f..a3c12652c1 100644 --- a/metricflow/engine/metricflow_engine.py +++ b/metricflow/engine/metricflow_engine.py @@ -31,7 +31,8 @@ from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver from metricflow_semantics.specs.query_param_implementations import SavedQueryParameter -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, MetricFlowQuerySpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import InstanceSpecSet from metricflow_semantics.time.time_source import TimeSource from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder diff --git a/metricflow/plan_conversion/dataflow_to_sql.py b/metricflow/plan_conversion/dataflow_to_sql.py index c4cacfd692..c8278444d0 100644 --- a/metricflow/plan_conversion/dataflow_to_sql.py +++ b/metricflow/plan_conversion/dataflow_to_sql.py @@ -30,8 +30,8 @@ ColumnAssociationResolver, SingleColumnCorrelationKey, ) +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( - GroupByMetricSpec, InstanceSpecSet, MeasureSpec, MetadataSpec, diff --git a/metricflow/plan_conversion/instance_converters.py b/metricflow/plan_conversion/instance_converters.py index 1139d07831..51987abe0f 100644 --- a/metricflow/plan_conversion/instance_converters.py +++ b/metricflow/plan_conversion/instance_converters.py @@ -29,11 +29,11 @@ from metricflow_semantics.model.semantics.metric_lookup import MetricLookup from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, EntitySpec, - GroupByMetricSpec, InstanceSpec, InstanceSpecSet, LinkableInstanceSpec, diff --git a/scripts/ci_tests/metricflow_package_test.py b/scripts/ci_tests/metricflow_package_test.py index 0d85c3f53a..bcc1a5f627 100644 --- a/scripts/ci_tests/metricflow_package_test.py +++ b/scripts/ci_tests/metricflow_package_test.py @@ -13,10 +13,10 @@ from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, ) diff --git a/tests_metricflow/dataflow/builder/test_cyclic_join.py b/tests_metricflow/dataflow/builder/test_cyclic_join.py index 10eb9758b9..07c2a13c16 100644 --- a/tests_metricflow/dataflow/builder/test_cyclic_join.py +++ b/tests_metricflow/dataflow/builder/test_cyclic_join.py @@ -6,9 +6,9 @@ import pytest from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.references import EntityReference +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py b/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py index 24c60296e7..61541e08f1 100644 --- a/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py +++ b/tests_metricflow/dataflow/builder/test_dataflow_plan_builder.py @@ -12,10 +12,10 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, OrderBySpec, TimeDimensionSpec, diff --git a/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py b/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py index 1cb33ee108..0a6789cb41 100644 --- a/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py +++ b/tests_metricflow/dataflow/optimizer/source_scan/test_source_scan_optimizer.py @@ -8,10 +8,10 @@ from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py b/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py index fc3138738a..2531bcd295 100644 --- a/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py +++ b/tests_metricflow/plan_conversion/dataflow_to_sql/test_conversion_metrics_to_sql.py @@ -4,9 +4,9 @@ from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py b/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py index a7771e4c97..f0fcf7b7ea 100644 --- a/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py +++ b/tests_metricflow/plan_conversion/dataflow_to_sql/test_distinct_values_to_sql.py @@ -6,7 +6,8 @@ from dbt_semantic_interfaces.references import EntityReference from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import DimensionSpec, MetricFlowQuerySpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import DimensionSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder diff --git a/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py b/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py index ae53da1509..065e19eb02 100644 --- a/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py +++ b/tests_metricflow/plan_conversion/dataflow_to_sql/test_metric_time_dimension_to_sql.py @@ -5,7 +5,8 @@ import pytest from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.references import TimeDimensionReference -from metricflow_semantics.specs.spec_classes import MetricFlowQuerySpec, MetricSpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import MetricSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.metric_time_dimension import MTD_SPEC_DAY diff --git a/tests_metricflow/plan_conversion/test_dataflow_to_execution.py b/tests_metricflow/plan_conversion/test_dataflow_to_execution.py index 925c7b22e7..5197fe1d63 100644 --- a/tests_metricflow/plan_conversion/test_dataflow_to_execution.py +++ b/tests_metricflow/plan_conversion/test_dataflow_to_execution.py @@ -4,10 +4,10 @@ from _pytest.fixtures import FixtureRequest from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py b/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py index a007d72765..73cee1d92c 100644 --- a/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py +++ b/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py @@ -13,13 +13,13 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, InstanceSpecSet, LinkableSpecSet, LinklessEntitySpec, MeasureSpec, - MetricFlowQuerySpec, MetricInputMeasureSpec, MetricSpec, NonAdditiveDimensionSpec, diff --git a/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py b/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py index a96e1f532c..f13355485d 100644 --- a/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py +++ b/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py @@ -13,7 +13,8 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import EntityReference, MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import EntityReference, MetricSpec, TimeDimensionSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.metric_time_dimension import MTD_SPEC_MONTH diff --git a/tests_metricflow/query_rendering/test_derived_metric_rendering.py b/tests_metricflow/query_rendering/test_derived_metric_rendering.py index 509ee6c253..284e39f999 100644 --- a/tests_metricflow/query_rendering/test_derived_metric_rendering.py +++ b/tests_metricflow/query_rendering/test_derived_metric_rendering.py @@ -12,8 +12,8 @@ from metricflow_semantics.naming.dunder_scheme import DunderNamingScheme from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py b/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py index ae903803c6..f820b8bbe4 100644 --- a/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py +++ b/tests_metricflow/query_rendering/test_fill_nulls_with_rendering.py @@ -12,9 +12,9 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from metricflow_semantics.query.query_parser import MetricFlowQueryParser +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py b/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py index b4bfd33a7c..b40622a7b0 100644 --- a/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py +++ b/tests_metricflow/query_rendering/test_granularity_date_part_rendering.py @@ -10,8 +10,8 @@ from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.type_enums.date_part import DatePart from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration diff --git a/tests_metricflow/query_rendering/test_metric_time_without_metrics.py b/tests_metricflow/query_rendering/test_metric_time_without_metrics.py index 55f32370e5..e28988d435 100644 --- a/tests_metricflow/query_rendering/test_metric_time_without_metrics.py +++ b/tests_metricflow/query_rendering/test_metric_time_without_metrics.py @@ -7,7 +7,8 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from metricflow_semantics.filters.time_constraint import TimeRangeConstraint -from metricflow_semantics.specs.spec_classes import DimensionSpec, MetricFlowQuerySpec, TimeDimensionSpec +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec +from metricflow_semantics.specs.spec_classes import DimensionSpec, TimeDimensionSpec from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.metric_time_dimension import MTD_SPEC_DAY diff --git a/tests_metricflow/query_rendering/test_query_rendering.py b/tests_metricflow/query_rendering/test_query_rendering.py index fe32116bfb..b07e349dd9 100644 --- a/tests_metricflow/query_rendering/test_query_rendering.py +++ b/tests_metricflow/query_rendering/test_query_rendering.py @@ -18,9 +18,9 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.query.query_parser import MetricFlowQueryParser from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - MetricFlowQuerySpec, MetricSpec, TimeDimensionSpec, ) diff --git a/tests_metricflow/query_rendering/test_time_spine_join_rendering.py b/tests_metricflow/query_rendering/test_time_spine_join_rendering.py index 58a07c9f2e..774e0095d8 100644 --- a/tests_metricflow/query_rendering/test_time_spine_join_rendering.py +++ b/tests_metricflow/query_rendering/test_time_spine_join_rendering.py @@ -11,8 +11,8 @@ import pytest from _pytest.fixtures import FixtureRequest from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( - MetricFlowQuerySpec, MetricSpec, ) from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration From c270b8b269e6b23d573f55eb7efd0c199d525cfe Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Thu, 25 Apr 2024 21:11:14 -0700 Subject: [PATCH 03/14] Remove `MetricFlowQuerySpec` dependency from `GroupByMetricSpec`. --- .../specs/group_by_metric_spec.py | 11 ----------- .../metricflow_semantics/specs/query_spec.py | 5 +---- .../metricflow_semantics/specs/spec_classes.py | 2 +- metricflow/dataflow/builder/dataflow_plan_builder.py | 11 ++++++++++- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py b/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py index f6758e8a03..63a6bd198e 100644 --- a/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py +++ b/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py @@ -8,13 +8,10 @@ from typing_extensions import override from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName -from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( - EntitySpec, InstanceSpecSet, InstanceSpecVisitor, LinkableInstanceSpec, - MetricSpec, ) from metricflow_semantics.visitor import VisitorOutputT @@ -59,11 +56,3 @@ def as_spec_set(self) -> InstanceSpecSet: def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_group_by_metric_spec(self) - - @property - def query_spec_for_source_node(self) -> MetricFlowQuerySpec: - """Query spec that can be used to build a source node for this spec in the DFP.""" - return MetricFlowQuerySpec( - metric_specs=(MetricSpec(element_name=self.element_name),), - entity_specs=tuple(EntitySpec.from_name(entity_link.element_name) for entity_link in self.entity_links), - ) diff --git a/metricflow-semantics/metricflow_semantics/specs/query_spec.py b/metricflow-semantics/metricflow_semantics/specs/query_spec.py index 7098aacbb1..14ff1f7860 100644 --- a/metricflow-semantics/metricflow_semantics/specs/query_spec.py +++ b/metricflow-semantics/metricflow_semantics/specs/query_spec.py @@ -1,6 +1,5 @@ from __future__ import annotations -import typing from dataclasses import dataclass from typing import Optional, Tuple @@ -11,6 +10,7 @@ from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( FilterSpecResolutionLookUp, ) +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, @@ -20,9 +20,6 @@ TimeDimensionSpec, ) -if typing.TYPE_CHECKING: - from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec - @dataclass(frozen=True) class MetricFlowQuerySpec(SerializableDataclass): diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py index 4cf1f58a6c..6fd726f15b 100644 --- a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py +++ b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py @@ -38,6 +38,7 @@ from metricflow_semantics.aggregation_properties import AggregationState from metricflow_semantics.collection_helpers.merger import Mergeable from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow_semantics.sql.sql_column_type import SqlColumnType from metricflow_semantics.sql.sql_join_type import SqlJoinType @@ -46,7 +47,6 @@ if TYPE_CHECKING: from metricflow_semantics.model.semantics.metric_lookup import MetricLookup from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup - from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec def hash_items(items: Sequence[SqlColumnType]) -> str: diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py index 009e823c50..405f33cf8f 100644 --- a/metricflow/dataflow/builder/dataflow_plan_builder.py +++ b/metricflow/dataflow/builder/dataflow_plan_builder.py @@ -34,6 +34,7 @@ FilterSpecResolutionLookUp, ) from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver +from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( ConstantPropertySpec, @@ -812,6 +813,14 @@ def _build_measure_spec_properties(self, measure_specs: Sequence[MeasureSpec]) - non_additive_dimension_spec=non_additive_dimension_spec, ) + def _query_spec_for_source_node(self, group_by_metric_spec: GroupByMetricSpec) -> MetricFlowQuerySpec: + return MetricFlowQuerySpec( + metric_specs=(MetricSpec(element_name=group_by_metric_spec.element_name),), + entity_specs=tuple( + EntitySpec.from_name(entity_link.element_name) for entity_link in group_by_metric_spec.entity_links + ), + ) + def _find_dataflow_recipe( self, linkable_spec_set: LinkableSpecSet, @@ -827,7 +836,7 @@ def _find_dataflow_recipe( # MetricGroupBy source nodes could be extremely large (and potentially slow). candidate_nodes_for_right_side_of_join += [ self._build_query_output_node( - query_spec=group_by_metric_spec.query_spec_for_source_node, for_group_by_source_node=True + query_spec=self._query_spec_for_source_node(group_by_metric_spec), for_group_by_source_node=True ) for group_by_metric_spec in linkable_spec_set.group_by_metric_specs ] From 43671745fd619fea55c822c093fb0ed1273e9255 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 02:41:11 -0700 Subject: [PATCH 04/14] Change `MetricFlowQueryParser.parse_and_validate_saved_query()` to return a result object. --- .../query/query_parser.py | 35 ++++++++++++++----- metricflow/engine/metricflow_engine.py | 2 +- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/query/query_parser.py b/metricflow-semantics/metricflow_semantics/query/query_parser.py index d92b53ef2b..c88f9929fc 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_parser.py +++ b/metricflow-semantics/metricflow_semantics/query/query_parser.py @@ -2,6 +2,7 @@ import datetime import logging +from dataclasses import dataclass from typing import List, Optional, Sequence, Tuple, Union import pandas as pd @@ -11,6 +12,7 @@ ) from dbt_semantic_interfaces.protocols import SavedQuery from dbt_semantic_interfaces.protocols.where_filter import WhereFilter +from dbt_semantic_interfaces.references import SemanticModelReference from dbt_semantic_interfaces.type_enums import TimeGranularity from metricflow_semantics.assert_one_arg import assert_at_most_one_arg_set @@ -18,7 +20,6 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat -from metricflow_semantics.mf_logging.runtime import log_runtime from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.naming.dunder_scheme import DunderNamingScheme from metricflow_semantics.naming.metric_scheme import MetricNamingScheme @@ -101,7 +102,7 @@ def parse_and_validate_saved_query( time_constraint_end: Optional[datetime.datetime], order_by_names: Optional[Sequence[str]], order_by_parameters: Optional[Sequence[OrderByQueryParameter]], - ) -> MetricFlowQuerySpec: + ) -> ParseQueryResult: """Parse and validate a query using parameters from a pre-defined / saved query. Additional parameters act in conjunction with the parameters in the saved query. @@ -115,15 +116,19 @@ def parse_and_validate_saved_query( if where_filter is not None: where_filters.append(where_filter) - return self.parse_and_validate_query( + return self._parse_and_validate_query( metric_names=saved_query.query_params.metrics, + metrics=None, group_by_names=saved_query.query_params.group_by, + group_by=None, where_constraint=merge_to_single_where_filter(PydanticWhereFilterIntersection(where_filters=where_filters)), + where_constraint_str=None, time_constraint_start=time_constraint_start, time_constraint_end=time_constraint_end, limit=limit, order_by_names=order_by_names, order_by=order_by_parameters, + min_max_only=False, ) def _get_saved_query(self, saved_query_parameter: SavedQueryParameter) -> SavedQuery: @@ -333,9 +338,9 @@ def parse_and_validate_query( order_by_names=order_by_names, order_by=order_by, min_max_only=min_max_only, - ) + ).query_spec - @log_runtime() + # @log_runtime() def _parse_and_validate_query( self, metric_names: Optional[Sequence[str]], @@ -350,7 +355,7 @@ def _parse_and_validate_query( order_by_names: Optional[Sequence[str]], order_by: Optional[Sequence[OrderByQueryParameter]], min_max_only: bool, - ) -> MetricFlowQuerySpec: + ) -> ParseQueryResult: # TODO: validate min_max_only - can only be called for non-metric queries assert_at_most_one_arg_set(metric_names=metric_names, metrics=metrics) assert_at_most_one_arg_set(group_by_names=group_by_names, group_by=group_by) @@ -508,6 +513,20 @@ def _parse_and_validate_query( ) logger.info(f"Time constraint after adjustment is: {time_constraint}") - return query_spec.with_time_range_constraint(time_constraint) + return ParseQueryResult( + query_spec=query_spec.with_time_range_constraint(time_constraint), + queried_semantic_models=query_resolution.queried_semantic_models, + ) + + return ParseQueryResult( + query_spec=query_spec, + queried_semantic_models=query_resolution.queried_semantic_models, + ) + + +@dataclass(frozen=True) +class ParseQueryResult: + """Result of parsing a MetricFlow query.""" - return query_spec + query_spec: MetricFlowQuerySpec + queried_semantic_models: Tuple[SemanticModelReference, ...] diff --git a/metricflow/engine/metricflow_engine.py b/metricflow/engine/metricflow_engine.py index a3c12652c1..915a17ec81 100644 --- a/metricflow/engine/metricflow_engine.py +++ b/metricflow/engine/metricflow_engine.py @@ -470,7 +470,7 @@ def _create_execution_plan(self, mf_query_request: MetricFlowQueryRequest) -> Me time_constraint_end=mf_query_request.time_constraint_end, order_by_names=mf_query_request.order_by_names, order_by_parameters=mf_query_request.order_by, - ) + ).query_spec else: query_spec = self._query_parser.parse_and_validate_query( metric_names=mf_query_request.metric_names, From 796b3673cd59176d684b64432697f3cbce88cf01 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 14:11:56 -0700 Subject: [PATCH 05/14] Split set classes from `spec_classes.py` into separate files. This was moved to address circular-dependency issues. Some methods were removed / changed to help support the split. The general issue was that because the set classes had bidirectional dependences with the spec classes, circular dependencies could be easily formed when imported by other modules. --- .../collection_helpers/dedupe.py | 15 + .../dunder_column_association_resolver.py | 2 +- .../specs/group_by_metric_spec.py | 58 --- .../specs/linkable_spec_set.py | 161 ++++++++ .../specs/partition_spec_set.py | 16 + .../specs/patterns/base_time_grain.py | 5 +- .../specs/patterns/entity_link_pattern.py | 5 +- .../specs/patterns/metric_pattern.py | 4 +- .../specs/patterns/metric_time_pattern.py | 5 +- .../specs/patterns/none_date_part.py | 4 +- .../specs/patterns/typed_patterns.py | 11 +- .../metricflow_semantics/specs/query_spec.py | 4 +- .../specs/spec_classes.py | 369 +++--------------- .../metricflow_semantics/specs/spec_set.py | 213 ++++++++++ .../specs/spec_set_transforms.py | 2 +- .../specs/where_filter_transform.py | 4 +- 16 files changed, 474 insertions(+), 404 deletions(-) create mode 100644 metricflow-semantics/metricflow_semantics/collection_helpers/dedupe.py delete mode 100644 metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py create mode 100644 metricflow-semantics/metricflow_semantics/specs/linkable_spec_set.py create mode 100644 metricflow-semantics/metricflow_semantics/specs/partition_spec_set.py create mode 100644 metricflow-semantics/metricflow_semantics/specs/spec_set.py diff --git a/metricflow-semantics/metricflow_semantics/collection_helpers/dedupe.py b/metricflow-semantics/metricflow_semantics/collection_helpers/dedupe.py new file mode 100644 index 0000000000..1cc5b46091 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/collection_helpers/dedupe.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from typing import Dict, Iterable, Tuple, TypeVar + +IterableT = TypeVar("IterableT") + + +def ordered_dedupe(*iterables: Iterable[IterableT]) -> Tuple[IterableT, ...]: + """De-duplicates the items in the iterables while preserving the order.""" + ordered_results: Dict[IterableT, None] = {} + for iterable in iterables: + for item in iterable: + ordered_results[item] = None + + return tuple(ordered_results.keys()) diff --git a/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py b/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py index 6351a4b572..d217067646 100644 --- a/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py +++ b/metricflow-semantics/metricflow_semantics/specs/dunder_column_association_resolver.py @@ -7,10 +7,10 @@ ColumnAssociationResolver, SingleColumnCorrelationKey, ) -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, + GroupByMetricSpec, InstanceSpec, InstanceSpecVisitor, MeasureSpec, diff --git a/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py b/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py deleted file mode 100644 index 63a6bd198e..0000000000 --- a/metricflow-semantics/metricflow_semantics/specs/group_by_metric_spec.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any - -from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass -from dbt_semantic_interfaces.references import EntityReference, MetricReference -from typing_extensions import override - -from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName -from metricflow_semantics.specs.spec_classes import ( - InstanceSpecSet, - InstanceSpecVisitor, - LinkableInstanceSpec, -) -from metricflow_semantics.visitor import VisitorOutputT - - -@dataclass(frozen=True) -class GroupByMetricSpec(LinkableInstanceSpec, SerializableDataclass): - """Metric used in group by or where filter.""" - - @property - def without_first_entity_link(self) -> GroupByMetricSpec: # noqa: D102 - assert len(self.entity_links) > 0, f"Spec does not have any entity links: {self}" - return GroupByMetricSpec(element_name=self.element_name, entity_links=self.entity_links[1:]) - - @property - def without_entity_links(self) -> GroupByMetricSpec: # noqa: D102 - return GroupByMetricSpec(element_name=self.element_name, entity_links=()) - - @staticmethod - def from_name(name: str) -> GroupByMetricSpec: # noqa: D102 - structured_name = StructuredLinkableSpecName.from_name(name) - return GroupByMetricSpec( - entity_links=tuple(EntityReference(idl) for idl in structured_name.entity_link_names), - element_name=structured_name.element_name, - ) - - def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 - if not isinstance(other, GroupByMetricSpec): - return False - return self.element_name == other.element_name and self.entity_links == other.entity_links - - def __hash__(self) -> int: # noqa: D105 - return hash((self.element_name, self.entity_links)) - - @property - def reference(self) -> MetricReference: # noqa: D102 - return MetricReference(element_name=self.element_name) - - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(group_by_metric_specs=(self,)) - - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 - return visitor.visit_group_by_metric_spec(self) diff --git a/metricflow-semantics/metricflow_semantics/specs/linkable_spec_set.py b/metricflow-semantics/metricflow_semantics/specs/linkable_spec_set.py new file mode 100644 index 0000000000..236172d8ab --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/linkable_spec_set.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import itertools +import typing +from dataclasses import dataclass +from typing import Dict, List, Sequence, Tuple + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass +from dbt_semantic_interfaces.references import MeasureReference, MetricReference +from typing_extensions import override + +from metricflow_semantics.collection_helpers.merger import Mergeable +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + GroupByMetricSpec, + InstanceSpec, + LinkableInstanceSpec, + TimeDimensionSpec, +) +from metricflow_semantics.specs.spec_set import InstanceSpecSet + +if typing.TYPE_CHECKING: + from metricflow_semantics.model.semantics.metric_lookup import MetricLookup + from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup + + +@dataclass(frozen=True) +class LinkableSpecSet(Mergeable, SerializableDataclass): + """Groups linkable specs.""" + + dimension_specs: Tuple[DimensionSpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () + entity_specs: Tuple[EntitySpec, ...] = () + group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () + + @property + def contains_metric_time(self) -> bool: + """Returns true if this set contains a spec referring to metric time at any grain.""" + return len(self.metric_time_specs) > 0 + + def included_agg_time_dimension_specs_for_metric( + self, metric_reference: MetricReference, metric_lookup: MetricLookup + ) -> List[TimeDimensionSpec]: + """Get the time dims included that are valid agg time dimensions for the specified metric.""" + queried_metric_time_specs = list(self.metric_time_specs) + + valid_agg_time_dimensions = metric_lookup.get_valid_agg_time_dimensions_for_metric(metric_reference) + queried_agg_time_dimension_specs = ( + list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) + + queried_metric_time_specs + ) + + return queried_agg_time_dimension_specs + + def included_agg_time_dimension_specs_for_measure( + self, measure_reference: MeasureReference, semantic_model_lookup: SemanticModelLookup + ) -> List[TimeDimensionSpec]: + """Get the time dims included that are valid agg time dimensions for the specified measure.""" + queried_metric_time_specs = list(self.metric_time_specs) + + valid_agg_time_dimensions = semantic_model_lookup.get_agg_time_dimension_specs_for_measure(measure_reference) + queried_agg_time_dimension_specs = ( + list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) + + queried_metric_time_specs + ) + + return queried_agg_time_dimension_specs + + @property + def metric_time_specs(self) -> Sequence[TimeDimensionSpec]: + """Returns any specs referring to metric time at any grain.""" + return tuple( + time_dimension_spec + for time_dimension_spec in self.time_dimension_specs + if time_dimension_spec.is_metric_time + ) + + @property + def as_tuple(self) -> Tuple[LinkableInstanceSpec, ...]: # noqa: D102 + return tuple( + itertools.chain( + self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs + ) + ) + + @override + def merge(self, other: LinkableSpecSet) -> LinkableSpecSet: + return LinkableSpecSet( + dimension_specs=self.dimension_specs + other.dimension_specs, + time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, + entity_specs=self.entity_specs + other.entity_specs, + group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, + ) + + @override + @classmethod + def empty_instance(cls) -> LinkableSpecSet: + return LinkableSpecSet() + + def dedupe(self) -> LinkableSpecSet: # noqa: D102 + # Use dictionaries to dedupe as it preserves insertion order. + + dimension_spec_dict: Dict[DimensionSpec, None] = {} + for dimension_spec in self.dimension_specs: + dimension_spec_dict[dimension_spec] = None + + time_dimension_spec_dict: Dict[TimeDimensionSpec, None] = {} + for time_dimension_spec in self.time_dimension_specs: + time_dimension_spec_dict[time_dimension_spec] = None + + entity_spec_dict: Dict[EntitySpec, None] = {} + for entity_spec in self.entity_specs: + entity_spec_dict[entity_spec] = None + + group_by_metric_spec_dict: Dict[GroupByMetricSpec, None] = {} + for group_by_metric in self.group_by_metric_specs: + group_by_metric_spec_dict[group_by_metric] = None + + return LinkableSpecSet( + dimension_specs=tuple(dimension_spec_dict.keys()), + time_dimension_specs=tuple(time_dimension_spec_dict.keys()), + entity_specs=tuple(entity_spec_dict.keys()), + group_by_metric_specs=tuple(group_by_metric_spec_dict.keys()), + ) + + def is_subset_of(self, other_set: LinkableSpecSet) -> bool: # noqa: D102 + return set(self.as_tuple).issubset(set(other_set.as_tuple)) + + @property + def as_spec_set(self) -> InstanceSpecSet: # noqa: D102 + return InstanceSpecSet( + dimension_specs=self.dimension_specs, + time_dimension_specs=self.time_dimension_specs, + entity_specs=self.entity_specs, + group_by_metric_specs=self.group_by_metric_specs, + ) + + def difference(self, other: LinkableSpecSet) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet( + dimension_specs=tuple(set(self.dimension_specs) - set(other.dimension_specs)), + time_dimension_specs=tuple(set(self.time_dimension_specs) - set(other.time_dimension_specs)), + entity_specs=tuple(set(self.entity_specs) - set(other.entity_specs)), + group_by_metric_specs=tuple(set(self.group_by_metric_specs) - set(other.group_by_metric_specs)), + ) + + def __len__(self) -> int: # noqa: D105 + return len(self.dimension_specs) + len(self.time_dimension_specs) + len(self.entity_specs) + + @staticmethod + def create_from_spec_set(spec_set: InstanceSpecSet) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet( + dimension_specs=spec_set.dimension_specs, + time_dimension_specs=spec_set.time_dimension_specs, + entity_specs=spec_set.entity_specs, + group_by_metric_specs=spec_set.group_by_metric_specs, + ) + + @staticmethod + def create_from_specs(specs: Sequence[InstanceSpec]) -> LinkableSpecSet: # noqa: D102 + return LinkableSpecSet.create_from_spec_set(InstanceSpecSet.create_from_specs(specs)) diff --git a/metricflow-semantics/metricflow_semantics/specs/partition_spec_set.py b/metricflow-semantics/metricflow_semantics/specs/partition_spec_set.py new file mode 100644 index 0000000000..51b598b764 --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/partition_spec_set.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Tuple + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass + +from metricflow_semantics.specs.spec_classes import DimensionSpec, TimeDimensionSpec + + +@dataclass(frozen=True) +class PartitionSpecSet(SerializableDataclass): + """Grouping of the linkable specs.""" + + dimension_specs: Tuple[DimensionSpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py b/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py index fa23af1990..34b44c1267 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/base_time_grain.py @@ -10,13 +10,12 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, - LinkableSpecSet, TimeDimensionSpec, TimeDimensionSpecComparisonKey, TimeDimensionSpecField, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type class BaseTimeGrainPattern(SpecPattern): @@ -66,7 +65,7 @@ def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[InstanceSpe return other_specs + tuple(BaseTimeGrainPattern(only_apply_for_metric_time=False).match(metric_time_specs)) - spec_set = LinkableSpecSet.from_specs(InstanceSpecSet.from_specs(candidate_specs).linkable_specs) + spec_set = group_specs_by_type(candidate_specs) spec_key_to_grains: Dict[TimeDimensionSpecComparisonKey, Set[TimeGranularity]] = defaultdict(set) spec_key_to_specs: Dict[TimeDimensionSpecComparisonKey, List[TimeDimensionSpec]] = defaultdict(list) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py index 5f152a07c4..5d5d9892a5 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py @@ -12,7 +12,8 @@ from typing_extensions import override from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern -from metricflow_semantics.specs.spec_classes import InstanceSpec, InstanceSpecSet, LinkableInstanceSpec +from metricflow_semantics.specs.spec_classes import InstanceSpec, LinkableInstanceSpec +from metricflow_semantics.specs.spec_set import group_specs_by_type logger = logging.getLogger(__name__) @@ -109,7 +110,7 @@ def _match_entity_links(self, candidate_specs: Sequence[LinkableInstanceSpec]) - @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - filtered_candidate_specs = InstanceSpecSet.from_specs(candidate_specs).linkable_specs + filtered_candidate_specs = group_specs_by_type(candidate_specs).linkable_specs # Checks that EntityLinkPatternParameterSetField is valid wrt to the parameter set. # Entity links could be a partial match, so it's handled separately. diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py index 0bbe31cbe4..346b39234c 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_pattern.py @@ -9,9 +9,9 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, MetricSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type @dataclass(frozen=True) @@ -22,7 +22,7 @@ class MetricSpecPattern(SpecPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[MetricSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return tuple( metric_name for metric_name in spec_set.metric_specs if metric_name.reference == self.metric_reference ) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py index ff0337de10..4ef943a86d 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/metric_time_pattern.py @@ -8,10 +8,9 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, - LinkableSpecSet, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type class MetricTimePattern(SpecPattern): @@ -23,7 +22,7 @@ class MetricTimePattern(SpecPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[TimeDimensionSpec]: - spec_set = LinkableSpecSet.from_specs(InstanceSpecSet.from_specs(candidate_specs).linkable_specs) + spec_set = group_specs_by_type(candidate_specs) return tuple( time_dimension_spec for time_dimension_spec in spec_set.time_dimension_specs diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py b/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py index 5a30db5cff..e451948f5a 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/none_date_part.py @@ -7,9 +7,9 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type class NoneDatePartPattern(SpecPattern): @@ -21,7 +21,7 @@ class NoneDatePartPattern(SpecPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: specs_to_return: List[LinkableInstanceSpec] = [] - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) for time_dimension_spec in spec_set.time_dimension_specs: if time_dimension_spec.date_part is None: specs_to_return.append(time_dimension_spec) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py index cec6ef7859..082a5b556a 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py @@ -17,7 +17,8 @@ EntityLinkPatternParameterSet, ParameterSetField, ) -from metricflow_semantics.specs.spec_classes import InstanceSpec, InstanceSpecSet, LinkableInstanceSpec +from metricflow_semantics.specs.spec_classes import InstanceSpec, LinkableInstanceSpec +from metricflow_semantics.specs.spec_set import group_specs_by_type @dataclass(frozen=True) @@ -29,7 +30,7 @@ class DimensionPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) filtered_specs: Sequence[LinkableInstanceSpec] = spec_set.dimension_specs + spec_set.time_dimension_specs return super().match(filtered_specs) @@ -58,7 +59,7 @@ class TimeDimensionPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return super().match(spec_set.time_dimension_specs) @staticmethod @@ -99,7 +100,7 @@ class EntityPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return super().match(spec_set.entity_specs) @staticmethod @@ -122,7 +123,7 @@ class GroupByMetricPattern(EntityLinkPattern): @override def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableInstanceSpec]: - spec_set = InstanceSpecSet.from_specs(candidate_specs) + spec_set = group_specs_by_type(candidate_specs) return super().match(spec_set.group_by_metric_specs) @staticmethod diff --git a/metricflow-semantics/metricflow_semantics/specs/query_spec.py b/metricflow-semantics/metricflow_semantics/specs/query_spec.py index 14ff1f7860..be42f19bc1 100644 --- a/metricflow-semantics/metricflow_semantics/specs/query_spec.py +++ b/metricflow-semantics/metricflow_semantics/specs/query_spec.py @@ -10,11 +10,11 @@ from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( FilterSpecResolutionLookUp, ) -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - LinkableSpecSet, + GroupByMetricSpec, MetricSpec, OrderBySpec, TimeDimensionSpec, diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py index 6fd726f15b..acf114b07d 100644 --- a/metricflow-semantics/metricflow_semantics/specs/spec_classes.py +++ b/metricflow-semantics/metricflow_semantics/specs/spec_classes.py @@ -11,13 +11,12 @@ from __future__ import annotations -import itertools import logging from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum from hashlib import sha1 -from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, Sequence, Tuple, TypeVar, Union +from typing import Any, Generic, List, Optional, Sequence, Tuple, TypeVar, Union from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass from dbt_semantic_interfaces.implementations.metric import PydanticMetricTimeWindow @@ -36,17 +35,15 @@ from typing_extensions import override from metricflow_semantics.aggregation_properties import AggregationState +from metricflow_semantics.collection_helpers.dedupe import ordered_dedupe from metricflow_semantics.collection_helpers.merger import Mergeable from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow_semantics.sql.sql_column_type import SqlColumnType from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow_semantics.visitor import VisitorOutputT -if TYPE_CHECKING: - from metricflow_semantics.model.semantics.metric_lookup import MetricLookup - from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup +logger = logging.getLogger(__name__) def hash_items(items: Sequence[SqlColumnType]) -> str: @@ -119,12 +116,6 @@ def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT """See Visitable.""" raise NotImplementedError() - @property - @abstractmethod - def as_spec_set(self) -> InstanceSpecSet: - """Return this as the one item in a InstanceSpecSet.""" - raise NotImplementedError - SelfTypeT = TypeVar("SelfTypeT", bound="LinkableInstanceSpec") @@ -147,11 +138,6 @@ def from_name(name: str, agg_type: Optional[AggregationType] = None) -> Metadata def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_metadata_spec(self) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(metadata_specs=(self,)) - @dataclass(frozen=True) class LinkableInstanceSpec(InstanceSpec, ABC): @@ -230,11 +216,6 @@ def __hash__(self) -> int: # noqa: D105 def reference(self) -> EntityReference: # noqa: D102 return EntityReference(element_name=self.element_name) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(entity_specs=(self,)) - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_entity_spec(self) @@ -295,11 +276,6 @@ def from_name(name: str) -> DimensionSpec: def reference(self) -> DimensionReference: # noqa: D102 return DimensionReference(element_name=self.element_name) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(dimension_specs=(self,)) - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_dimension_spec(self) @@ -421,11 +397,6 @@ def from_reference(reference: TimeDimensionReference) -> TimeDimensionSpec: """Initialize from a time dimension reference instance.""" return TimeDimensionSpec(entity_links=(), element_name=reference.element_name) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(time_dimension_specs=(self,)) - def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_time_dimension_spec(self) @@ -506,13 +477,9 @@ def bucket_hash(self) -> str: return hash_items(values) @property - def linkable_specs(self) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=(), - time_dimension_specs=(TimeDimensionSpec.from_name(self.name),), - entity_specs=tuple( - LinklessEntitySpec.from_element_name(entity_name) for entity_name in self.window_groupings - ), + def linkable_specs(self) -> Sequence[LinkableInstanceSpec]: # noqa: D102 + return (TimeDimensionSpec.from_name(self.name),) + tuple( + LinklessEntitySpec.from_element_name(entity_name) for entity_name in self.window_groupings ) def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 @@ -548,11 +515,6 @@ def reference(self) -> MeasureReference: # noqa: D102 def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_measure_spec(self) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(measure_specs=(self,)) - @dataclass(frozen=True) class MetricSpec(InstanceSpec): # noqa: D101 @@ -579,11 +541,6 @@ def from_reference(reference: MetricReference) -> MetricSpec: def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 return visitor.visit_metric_spec(self) - @property - @override - def as_spec_set(self) -> InstanceSpecSet: - return InstanceSpecSet(metric_specs=(self,)) - @property def reference(self) -> MetricReference: """Return the reference object that is used for referencing the associated metric in the manifest.""" @@ -646,277 +603,6 @@ class OrderBySpec(SerializableDataclass): # noqa: D101 descending: bool -@dataclass(frozen=True) -class FilterSpec(SerializableDataclass): # noqa: D101 - expr: str - elements: Tuple[InstanceSpec, ...] - - -@dataclass(frozen=True) -class LinkableSpecSet(Mergeable, SerializableDataclass): - """Groups linkable specs.""" - - dimension_specs: Tuple[DimensionSpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - entity_specs: Tuple[EntitySpec, ...] = () - group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () - - @property - def contains_metric_time(self) -> bool: - """Returns true if this set contains a spec referring to metric time at any grain.""" - return len(self.metric_time_specs) > 0 - - def included_agg_time_dimension_specs_for_metric( - self, metric_reference: MetricReference, metric_lookup: MetricLookup - ) -> List[TimeDimensionSpec]: - """Get the time dims included that are valid agg time dimensions for the specified metric.""" - queried_metric_time_specs = list(self.metric_time_specs) - - valid_agg_time_dimensions = metric_lookup.get_valid_agg_time_dimensions_for_metric(metric_reference) - queried_agg_time_dimension_specs = ( - list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) - + queried_metric_time_specs - ) - - return queried_agg_time_dimension_specs - - def included_agg_time_dimension_specs_for_measure( - self, measure_reference: MeasureReference, semantic_model_lookup: SemanticModelLookup - ) -> List[TimeDimensionSpec]: - """Get the time dims included that are valid agg time dimensions for the specified measure.""" - queried_metric_time_specs = list(self.metric_time_specs) - - valid_agg_time_dimensions = semantic_model_lookup.get_agg_time_dimension_specs_for_measure(measure_reference) - queried_agg_time_dimension_specs = ( - list(set(self.time_dimension_specs).intersection(set(valid_agg_time_dimensions))) - + queried_metric_time_specs - ) - - return queried_agg_time_dimension_specs - - @property - def metric_time_specs(self) -> Sequence[TimeDimensionSpec]: - """Returns any specs referring to metric time at any grain.""" - return tuple( - time_dimension_spec - for time_dimension_spec in self.time_dimension_specs - if time_dimension_spec.is_metric_time - ) - - @property - def as_tuple(self) -> Tuple[LinkableInstanceSpec, ...]: # noqa: D102 - return tuple( - itertools.chain( - self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs - ) - ) - - @override - def merge(self, other: LinkableSpecSet) -> LinkableSpecSet: - return LinkableSpecSet( - dimension_specs=self.dimension_specs + other.dimension_specs, - time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, - entity_specs=self.entity_specs + other.entity_specs, - group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, - ) - - @override - @classmethod - def empty_instance(cls) -> LinkableSpecSet: - return LinkableSpecSet() - - def dedupe(self) -> LinkableSpecSet: # noqa: D102 - # Use dictionaries to dedupe as it preserves insertion order. - - dimension_spec_dict: Dict[DimensionSpec, None] = {} - for dimension_spec in self.dimension_specs: - dimension_spec_dict[dimension_spec] = None - - time_dimension_spec_dict: Dict[TimeDimensionSpec, None] = {} - for time_dimension_spec in self.time_dimension_specs: - time_dimension_spec_dict[time_dimension_spec] = None - - entity_spec_dict: Dict[EntitySpec, None] = {} - for entity_spec in self.entity_specs: - entity_spec_dict[entity_spec] = None - - group_by_metric_spec_dict: Dict[GroupByMetricSpec, None] = {} - for group_by_metric in self.group_by_metric_specs: - group_by_metric_spec_dict[group_by_metric] = None - - return LinkableSpecSet( - dimension_specs=tuple(dimension_spec_dict.keys()), - time_dimension_specs=tuple(time_dimension_spec_dict.keys()), - entity_specs=tuple(entity_spec_dict.keys()), - group_by_metric_specs=tuple(group_by_metric_spec_dict.keys()), - ) - - def is_subset_of(self, other_set: LinkableSpecSet) -> bool: # noqa: D102 - return set(self.as_tuple).issubset(set(other_set.as_tuple)) - - @property - def as_spec_set(self) -> InstanceSpecSet: # noqa: D102 - return InstanceSpecSet( - dimension_specs=self.dimension_specs, - time_dimension_specs=self.time_dimension_specs, - entity_specs=self.entity_specs, - group_by_metric_specs=self.group_by_metric_specs, - ) - - def difference(self, other: LinkableSpecSet) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=tuple(set(self.dimension_specs) - set(other.dimension_specs)), - time_dimension_specs=tuple(set(self.time_dimension_specs) - set(other.time_dimension_specs)), - entity_specs=tuple(set(self.entity_specs) - set(other.entity_specs)), - group_by_metric_specs=tuple(set(self.group_by_metric_specs) - set(other.group_by_metric_specs)), - ) - - def __len__(self) -> int: # noqa: D105 - return len(self.dimension_specs) + len(self.time_dimension_specs) + len(self.entity_specs) - - @staticmethod - def from_specs(specs: Sequence[LinkableInstanceSpec]) -> LinkableSpecSet: # noqa: D102 - instance_spec_set = InstanceSpecSet.from_specs(specs) - return LinkableSpecSet( - dimension_specs=instance_spec_set.dimension_specs, - time_dimension_specs=instance_spec_set.time_dimension_specs, - entity_specs=instance_spec_set.entity_specs, - group_by_metric_specs=instance_spec_set.group_by_metric_specs, - ) - - -TransformOutputT = TypeVar("TransformOutputT") - - -class InstanceSpecSetTransform(Generic[TransformOutputT], ABC): - """Function to use for transforming spec sets.""" - - @abstractmethod - def transform(self, spec_set: InstanceSpecSet) -> TransformOutputT: # noqa: D102 - pass - - -@dataclass(frozen=True) -class InstanceSpecSet(Mergeable, SerializableDataclass): - """Consolidates all specs used in an instance set.""" - - metric_specs: Tuple[MetricSpec, ...] = () - measure_specs: Tuple[MeasureSpec, ...] = () - dimension_specs: Tuple[DimensionSpec, ...] = () - entity_specs: Tuple[EntitySpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () - metadata_specs: Tuple[MetadataSpec, ...] = () - - @override - def merge(self, other: InstanceSpecSet) -> InstanceSpecSet: - return InstanceSpecSet( - metric_specs=self.metric_specs + other.metric_specs, - measure_specs=self.measure_specs + other.measure_specs, - dimension_specs=self.dimension_specs + other.dimension_specs, - entity_specs=self.entity_specs + other.entity_specs, - group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, - time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, - metadata_specs=self.metadata_specs + other.metadata_specs, - ) - - @override - @classmethod - def empty_instance(cls) -> InstanceSpecSet: - return InstanceSpecSet() - - def dedupe(self) -> InstanceSpecSet: - """De-duplicates repeated elements. - - TBD: Have merge de-duplicate instead. - """ - metric_specs_deduped = [] - for metric_spec in self.metric_specs: - if metric_spec not in metric_specs_deduped: - metric_specs_deduped.append(metric_spec) - - measure_specs_deduped = [] - for measure_spec in self.measure_specs: - if measure_spec not in measure_specs_deduped: - measure_specs_deduped.append(measure_spec) - - dimension_specs_deduped = [] - for dimension_spec in self.dimension_specs: - if dimension_spec not in dimension_specs_deduped: - dimension_specs_deduped.append(dimension_spec) - - time_dimension_specs_deduped = [] - for time_dimension_spec in self.time_dimension_specs: - if time_dimension_spec not in time_dimension_specs_deduped: - time_dimension_specs_deduped.append(time_dimension_spec) - - entity_specs_deduped = [] - for entity_spec in self.entity_specs: - if entity_spec not in entity_specs_deduped: - entity_specs_deduped.append(entity_spec) - - group_by_metric_specs_deduped = [] - for group_by_metric_spec in self.group_by_metric_specs: - if group_by_metric_spec not in group_by_metric_specs_deduped: - group_by_metric_specs_deduped.append(group_by_metric_spec) - - return InstanceSpecSet( - metric_specs=tuple(metric_specs_deduped), - measure_specs=tuple(measure_specs_deduped), - dimension_specs=tuple(dimension_specs_deduped), - time_dimension_specs=tuple(time_dimension_specs_deduped), - entity_specs=tuple(entity_specs_deduped), - group_by_metric_specs=tuple(group_by_metric_specs_deduped), - ) - - @property - def linkable_specs(self) -> Sequence[LinkableInstanceSpec]: - """All linkable specs in this set.""" - return list( - itertools.chain( - self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs - ) - ) - - @property - def all_specs(self) -> Sequence[InstanceSpec]: # noqa: D102 - return tuple( - itertools.chain( - self.measure_specs, - self.dimension_specs, - self.time_dimension_specs, - self.entity_specs, - self.group_by_metric_specs, - self.metric_specs, - self.metadata_specs, - ) - ) - - def transform( # noqa: D102 - self, transform_function: InstanceSpecSetTransform[TransformOutputT] - ) -> TransformOutputT: # noqa: D102 - return transform_function.transform(self) - - @staticmethod - def from_specs(specs: Sequence[InstanceSpec]) -> InstanceSpecSet: # noqa: D102 - return InstanceSpecSet.merge_iterable(spec.as_spec_set for spec in specs) - - -@dataclass(frozen=True) -class PartitionSpecSet(SerializableDataclass): - """Grouping of the linkable specs.""" - - dimension_specs: Tuple[DimensionSpec, ...] = () - time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () - - -logger = logging.getLogger(__name__) - - -class WhereFilterResolutionException(Exception): # noqa: D101 - pass - - @dataclass(frozen=True) class WhereFilterSpec(Mergeable, SerializableDataclass): """Similar to the WhereFilter, but with the where_sql_template rendered and used elements extracted. @@ -944,7 +630,7 @@ class WhereFilterSpec(Mergeable, SerializableDataclass): # quoted identifiers later. where_sql: str bind_parameters: SqlBindParameters - linkable_spec_set: LinkableSpecSet + linkable_specs: Tuple[LinkableInstanceSpec, ...] def merge(self, other: WhereFilterSpec) -> WhereFilterSpec: # noqa: D102 if self == WhereFilterSpec.empty_instance(): @@ -959,7 +645,7 @@ def merge(self, other: WhereFilterSpec) -> WhereFilterSpec: # noqa: D102 return WhereFilterSpec( where_sql=f"({self.where_sql}) AND ({other.where_sql})", bind_parameters=self.bind_parameters.combine(other.bind_parameters), - linkable_spec_set=self.linkable_spec_set.merge(other.linkable_spec_set).dedupe(), + linkable_specs=ordered_dedupe(self.linkable_specs, other.linkable_specs), ) @classmethod @@ -972,7 +658,7 @@ def empty_instance(cls) -> WhereFilterSpec: return WhereFilterSpec( where_sql="TRUE", bind_parameters=SqlBindParameters(), - linkable_spec_set=LinkableSpecSet(), + linkable_specs=(), ) @@ -991,3 +677,40 @@ class JoinToTimeSpineDescription: join_type: SqlJoinType offset_window: Optional[MetricTimeWindow] offset_to_grain: Optional[TimeGranularity] + + +@dataclass(frozen=True) +class GroupByMetricSpec(LinkableInstanceSpec, SerializableDataclass): + """Metric used in group by or where filter.""" + + @property + def without_first_entity_link(self) -> GroupByMetricSpec: # noqa: D102 + assert len(self.entity_links) > 0, f"Spec does not have any entity links: {self}" + return GroupByMetricSpec(element_name=self.element_name, entity_links=self.entity_links[1:]) + + @property + def without_entity_links(self) -> GroupByMetricSpec: # noqa: D102 + return GroupByMetricSpec(element_name=self.element_name, entity_links=()) + + @staticmethod + def from_name(name: str) -> GroupByMetricSpec: # noqa: D102 + structured_name = StructuredLinkableSpecName.from_name(name) + return GroupByMetricSpec( + entity_links=tuple(EntityReference(idl) for idl in structured_name.entity_link_names), + element_name=structured_name.element_name, + ) + + def __eq__(self, other: Any) -> bool: # type: ignore[misc] # noqa: D105 + if not isinstance(other, GroupByMetricSpec): + return False + return self.element_name == other.element_name and self.entity_links == other.entity_links + + def __hash__(self) -> int: # noqa: D105 + return hash((self.element_name, self.entity_links)) + + @property + def reference(self) -> MetricReference: # noqa: D102 + return MetricReference(element_name=self.element_name) + + def accept(self, visitor: InstanceSpecVisitor[VisitorOutputT]) -> VisitorOutputT: # noqa: D102 + return visitor.visit_group_by_metric_spec(self) diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_set.py b/metricflow-semantics/metricflow_semantics/specs/spec_set.py new file mode 100644 index 0000000000..8839af492d --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/specs/spec_set.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import dataclasses +import itertools +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Generic, List, Sequence, Tuple, TypeVar + +from dbt_semantic_interfaces.dataclass_serialization import SerializableDataclass +from typing_extensions import override + +from metricflow_semantics.collection_helpers.merger import Mergeable +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + GroupByMetricSpec, + InstanceSpec, + InstanceSpecVisitor, + LinkableInstanceSpec, + MeasureSpec, + MetadataSpec, + MetricSpec, + TimeDimensionSpec, +) + + +@dataclass(frozen=True) +class InstanceSpecSet(Mergeable, SerializableDataclass): + """Consolidates all specs used in an instance set.""" + + metric_specs: Tuple[MetricSpec, ...] = () + measure_specs: Tuple[MeasureSpec, ...] = () + dimension_specs: Tuple[DimensionSpec, ...] = () + entity_specs: Tuple[EntitySpec, ...] = () + time_dimension_specs: Tuple[TimeDimensionSpec, ...] = () + group_by_metric_specs: Tuple[GroupByMetricSpec, ...] = () + metadata_specs: Tuple[MetadataSpec, ...] = () + + @override + def merge(self, other: InstanceSpecSet) -> InstanceSpecSet: + return InstanceSpecSet( + metric_specs=self.metric_specs + other.metric_specs, + measure_specs=self.measure_specs + other.measure_specs, + dimension_specs=self.dimension_specs + other.dimension_specs, + entity_specs=self.entity_specs + other.entity_specs, + group_by_metric_specs=self.group_by_metric_specs + other.group_by_metric_specs, + time_dimension_specs=self.time_dimension_specs + other.time_dimension_specs, + metadata_specs=self.metadata_specs + other.metadata_specs, + ) + + @override + @classmethod + def empty_instance(cls) -> InstanceSpecSet: + return InstanceSpecSet() + + def dedupe(self) -> InstanceSpecSet: + """De-duplicates repeated elements. + + TBD: Have merge de-duplicate instead. + """ + metric_specs_deduped = [] + for metric_spec in self.metric_specs: + if metric_spec not in metric_specs_deduped: + metric_specs_deduped.append(metric_spec) + + measure_specs_deduped = [] + for measure_spec in self.measure_specs: + if measure_spec not in measure_specs_deduped: + measure_specs_deduped.append(measure_spec) + + dimension_specs_deduped = [] + for dimension_spec in self.dimension_specs: + if dimension_spec not in dimension_specs_deduped: + dimension_specs_deduped.append(dimension_spec) + + time_dimension_specs_deduped = [] + for time_dimension_spec in self.time_dimension_specs: + if time_dimension_spec not in time_dimension_specs_deduped: + time_dimension_specs_deduped.append(time_dimension_spec) + + entity_specs_deduped = [] + for entity_spec in self.entity_specs: + if entity_spec not in entity_specs_deduped: + entity_specs_deduped.append(entity_spec) + + group_by_metric_specs_deduped = [] + for group_by_metric_spec in self.group_by_metric_specs: + if group_by_metric_spec not in group_by_metric_specs_deduped: + group_by_metric_specs_deduped.append(group_by_metric_spec) + + return InstanceSpecSet( + metric_specs=tuple(metric_specs_deduped), + measure_specs=tuple(measure_specs_deduped), + dimension_specs=tuple(dimension_specs_deduped), + time_dimension_specs=tuple(time_dimension_specs_deduped), + entity_specs=tuple(entity_specs_deduped), + group_by_metric_specs=tuple(group_by_metric_specs_deduped), + ) + + @property + def linkable_specs(self) -> Sequence[LinkableInstanceSpec]: + """All linkable specs in this set.""" + return list( + itertools.chain( + self.dimension_specs, self.time_dimension_specs, self.entity_specs, self.group_by_metric_specs + ) + ) + + @property + def all_specs(self) -> Sequence[InstanceSpec]: # noqa: D102 + return tuple( + itertools.chain( + self.measure_specs, + self.dimension_specs, + self.time_dimension_specs, + self.entity_specs, + self.group_by_metric_specs, + self.metric_specs, + self.metadata_specs, + ) + ) + + def transform( # noqa: D102 + self, transform_function: InstanceSpecSetTransform[TransformOutputT] + ) -> TransformOutputT: + return transform_function.transform(self) + + @property + def metric_time_specs(self) -> Sequence[TimeDimensionSpec]: + """Returns any specs referring to metric time at any grain.""" + return tuple( + time_dimension_spec + for time_dimension_spec in self.time_dimension_specs + if time_dimension_spec.is_metric_time + ) + + @staticmethod + def create_from_specs(specs: Sequence[InstanceSpec]) -> InstanceSpecSet: # noqa: D102 + return group_specs_by_type(specs) + + +TransformOutputT = TypeVar("TransformOutputT") + + +class InstanceSpecSetTransform(Generic[TransformOutputT], ABC): + """Function to use for transforming spec sets.""" + + @abstractmethod + def transform(self, spec_set: InstanceSpecSet) -> TransformOutputT: # noqa: D102 + pass + + +@dataclass +class _GroupSpecByTypeVisitor(InstanceSpecVisitor[None]): + """Groups a spec by type into an `InstanceSpecSet`.""" + + metric_specs: List[MetricSpec] = dataclasses.field(default_factory=list) + measure_specs: List[MeasureSpec] = dataclasses.field(default_factory=list) + dimension_specs: List[DimensionSpec] = dataclasses.field(default_factory=list) + entity_specs: List[EntitySpec] = dataclasses.field(default_factory=list) + time_dimension_specs: List[TimeDimensionSpec] = dataclasses.field(default_factory=list) + group_by_metric_specs: List[GroupByMetricSpec] = dataclasses.field(default_factory=list) + metadata_specs: List[MetadataSpec] = dataclasses.field(default_factory=list) + + @override + def visit_measure_spec(self, measure_spec: MeasureSpec) -> None: + self.measure_specs.append(measure_spec) + + @override + def visit_dimension_spec(self, dimension_spec: DimensionSpec) -> None: + self.dimension_specs.append(dimension_spec) + + @override + def visit_time_dimension_spec(self, time_dimension_spec: TimeDimensionSpec) -> None: + self.time_dimension_specs.append(time_dimension_spec) + + @override + def visit_entity_spec(self, entity_spec: EntitySpec) -> None: + self.entity_specs.append(entity_spec) + + @override + def visit_group_by_metric_spec(self, group_by_metric_spec: GroupByMetricSpec) -> None: + self.group_by_metric_specs.append(group_by_metric_spec) + + @override + def visit_metric_spec(self, metric_spec: MetricSpec) -> None: + self.metric_specs.append(metric_spec) + + @override + def visit_metadata_spec(self, metadata_spec: MetadataSpec) -> None: + self.metadata_specs.append(metadata_spec) + + +def group_specs_by_type(specs: Sequence[InstanceSpec]) -> InstanceSpecSet: + """Groups a sequence of specs by type.""" + grouper = _GroupSpecByTypeVisitor() + for spec in specs: + spec.accept(grouper) + + return InstanceSpecSet( + metric_specs=tuple(grouper.metric_specs), + measure_specs=tuple(grouper.measure_specs), + dimension_specs=tuple(grouper.dimension_specs), + entity_specs=tuple(grouper.entity_specs), + time_dimension_specs=tuple(grouper.time_dimension_specs), + group_by_metric_specs=tuple(grouper.group_by_metric_specs), + metadata_specs=tuple(grouper.metadata_specs), + ) + + +def group_spec_by_type(spec: InstanceSpec) -> InstanceSpecSet: + """Similar to group_specs_by_type() but for a single spec.""" + return group_specs_by_type((spec,)) diff --git a/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py b/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py index 991cacafc2..a36aa0c2b0 100644 --- a/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py +++ b/metricflow-semantics/metricflow_semantics/specs/spec_set_transforms.py @@ -2,7 +2,7 @@ from typing import Set -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, InstanceSpecSetTransform +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform class ToElementNameSet(InstanceSpecSetTransform[Set[str]]): diff --git a/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py b/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py index d56676d912..fe82785d7c 100644 --- a/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py +++ b/metricflow-semantics/metricflow_semantics/specs/where_filter_transform.py @@ -13,7 +13,7 @@ ) from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver from metricflow_semantics.specs.rendered_spec_tracker import RenderedSpecTracker -from metricflow_semantics.specs.spec_classes import LinkableSpecSet, WhereFilterSpec +from metricflow_semantics.specs.spec_classes import WhereFilterSpec from metricflow_semantics.specs.where_filter_dimension import WhereFilterDimensionFactory from metricflow_semantics.specs.where_filter_entity import WhereFilterEntityFactory from metricflow_semantics.specs.where_filter_metric import WhereFilterMetricFactory @@ -103,7 +103,7 @@ def create_from_where_filter_intersection( # noqa: D102 WhereFilterSpec( where_sql=where_sql, bind_parameters=SqlBindParameters(), - linkable_spec_set=LinkableSpecSet.from_specs(rendered_spec_tracker.rendered_specs), + linkable_specs=tuple(rendered_spec_tracker.rendered_specs), ) ) From 0673c22b220c6f1fe4d798285f2e86f8e214fe63 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 14:14:50 -0700 Subject: [PATCH 06/14] Update `LinkableElementSet` to include semantic model and spec methods. `LinkableElementSet` will be used in group-by resolution to keep track of the semantic models that a query would read. --- .../model/semantic_model_derivation.py | 19 ++ .../model/semantics/linkable_element.py | 57 +++++- .../model/semantics/linkable_element_set.py | 181 ++++++++++++++---- 3 files changed, 210 insertions(+), 47 deletions(-) create mode 100644 metricflow-semantics/metricflow_semantics/model/semantic_model_derivation.py diff --git a/metricflow-semantics/metricflow_semantics/model/semantic_model_derivation.py b/metricflow-semantics/metricflow_semantics/model/semantic_model_derivation.py new file mode 100644 index 0000000000..ba541b68ba --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/model/semantic_model_derivation.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Sequence + +from dbt_semantic_interfaces.references import SemanticModelReference + + +class SemanticModelDerivation(ABC): + """Interface for an object that can be described as derived from a semantic model.""" + + @property + @abstractmethod + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + """The semantic models that this was derived from. + + The returned sequence should be ordered and not contain duplicates. + """ + raise NotImplementedError diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py index aafde71315..86dfb3669c 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py @@ -1,17 +1,27 @@ from __future__ import annotations +import logging +from abc import ABC from dataclasses import dataclass from enum import Enum -from typing import FrozenSet, Optional, Tuple +from typing import FrozenSet, Optional, Sequence, Tuple from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from dbt_semantic_interfaces.protocols.dimension import DimensionType -from dbt_semantic_interfaces.references import DimensionReference, MetricReference, SemanticModelReference +from dbt_semantic_interfaces.references import ( + DimensionReference, + EntityReference, + MetricReference, + SemanticModelReference, +) from dbt_semantic_interfaces.type_enums.date_part import DatePart from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from typing_extensions import override from metricflow_semantics.model.linkable_element_property import LinkableElementProperty -from metricflow_semantics.specs.spec_classes import EntityReference +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation + +logger = logging.getLogger(__name__) class LinkableElementType(Enum): @@ -72,8 +82,14 @@ class SemanticModelJoinPathElement: join_on_entity: EntityReference +class LinkableElement(SemanticModelDerivation, ABC): + """An entity / dimension that may have been joined by entities.""" + + pass + + @dataclass(frozen=True) -class LinkableDimension: +class LinkableDimension(LinkableElement): """Describes how a dimension can be realized by joining based on entity links.""" # The semantic model where this dimension was defined. @@ -105,9 +121,20 @@ def path_key(self) -> ElementPathKey: # noqa: D102 def reference(self) -> DimensionReference: # noqa: D102 return DimensionReference(element_name=self.element_name) + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references = set() + if self.semantic_model_origin: + semantic_model_references.add(self.semantic_model_origin) + for join_path_item in self.join_path: + semantic_model_references.add(join_path_item.semantic_model_reference) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + @dataclass(frozen=True) -class LinkableEntity: +class LinkableEntity(LinkableElement, SemanticModelDerivation): """Describes how an entity can be realized by joining based on entity links.""" # The semantic model where this entity was defined. @@ -127,9 +154,18 @@ def path_key(self) -> ElementPathKey: # noqa: D102 def reference(self) -> EntityReference: # noqa: D102 return EntityReference(element_name=self.element_name) + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references = {self.semantic_model_origin} + for join_path_item in self.join_path: + semantic_model_references.add(join_path_item.semantic_model_reference) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + @dataclass(frozen=True) -class LinkableMetric: +class LinkableMetric(LinkableElement, SemanticModelDerivation): """Describes how a metric can be realized by joining based on entity links.""" element_name: str @@ -149,6 +185,15 @@ def path_key(self) -> ElementPathKey: # noqa: D102 def reference(self) -> MetricReference: # noqa: D102 return MetricReference(element_name=self.element_name) + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references = {self.join_by_semantic_model} + for join_path_item in self.join_path: + semantic_model_references.add(join_path_item.semantic_model_reference) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + @dataclass(frozen=True) class SemanticModelJoinPath: diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py index 449db647af..4408d6f7a4 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py @@ -4,7 +4,12 @@ from dataclasses import dataclass, field from typing import Dict, FrozenSet, List, Sequence, Set, Tuple +from dbt_semantic_interfaces.enum_extension import assert_values_exhausted +from dbt_semantic_interfaces.references import SemanticModelReference +from typing_extensions import override + from metricflow_semantics.model.linkable_element_property import LinkableElementProperty +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation from metricflow_semantics.model.semantics.linkable_element import ( ElementPathKey, LinkableDimension, @@ -12,17 +17,19 @@ LinkableEntity, LinkableMetric, ) -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec +from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, - LinkableSpecSet, + GroupByMetricSpec, + InstanceSpec, + LinkableInstanceSpec, TimeDimensionSpec, ) @dataclass(frozen=True) -class LinkableElementSet: +class LinkableElementSet(SemanticModelDerivation): """Container class for storing all linkable elements for a metric. TODO: There are similarities with LinkableSpecSet - consider consolidation. @@ -61,6 +68,33 @@ def __post_init__(self) -> None: f"type! Mismatched elements: {mismatched_elements}" ) + # There shouldn't be a path key without any concrete items. Can be an issue as specs contained in this set are + # generated from the path keys. + for key, value in ( + tuple(self.path_key_to_linkable_dimensions.items()) + + tuple(self.path_key_to_linkable_entities.items()) + + tuple(self.path_key_to_linkable_metrics.items()) + ): + assert len(value) > 0, f"{key} is empty" + + # There shouldn't be any duplicate specs. + specs = self.specs + deduped_specs = set(specs) + assert len(deduped_specs) == len(specs) + assert len(deduped_specs) == ( + len(self.path_key_to_linkable_dimensions) + + len(self.path_key_to_linkable_entities) + + len(self.path_key_to_linkable_metrics) + ) + + # Check time dimensions have the grain set. + for path_key, linkable_dimensions in self.path_key_to_linkable_dimensions.items(): + if path_key.element_type is LinkableElementType.TIME_DIMENSION: + for linkable_dimension in linkable_dimensions: + assert ( + linkable_dimension.time_granularity is not None + ), f"{path_key} has a dimension without the time granularity set: {linkable_dimension}" + @staticmethod def merge_by_path_key(linkable_element_sets: Sequence[LinkableElementSet]) -> LinkableElementSet: """Combine multiple sets together by the path key. @@ -108,6 +142,8 @@ def intersection_by_path_key(linkable_element_sets: Sequence[LinkableElementSet] """ if len(linkable_element_sets) == 0: return LinkableElementSet() + elif len(linkable_element_sets) == 1: + return linkable_element_sets[0] # Find path keys that are common to all LinkableElementSets. dimension_path_keys: List[Set[ElementPathKey]] = [] @@ -120,7 +156,6 @@ def intersection_by_path_key(linkable_element_sets: Sequence[LinkableElementSet] common_linkable_dimension_path_keys = set.intersection(*dimension_path_keys) if dimension_path_keys else set() common_linkable_entity_path_keys = set.intersection(*entity_path_keys) if entity_path_keys else set() common_linkable_metric_path_keys = set.intersection(*metric_path_keys) if metric_path_keys else set() - # Create a new LinkableElementSet that only includes items where the path key is common to all sets. join_path_to_linkable_dimensions: Dict[ElementPathKey, Set[LinkableDimension]] = defaultdict(set) join_path_to_linkable_entities: Dict[ElementPathKey, Set[LinkableEntity]] = defaultdict(set) @@ -233,43 +268,6 @@ def filter( path_key_to_linkable_metrics=key_to_linkable_metrics, ) - @property - def as_spec_set(self) -> LinkableSpecSet: # noqa: D102 - return LinkableSpecSet( - dimension_specs=tuple( - DimensionSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - for path_key in self.path_key_to_linkable_dimensions.keys() - if path_key.element_type is LinkableElementType.DIMENSION - ), - time_dimension_specs=tuple( - TimeDimensionSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - time_granularity=path_key.time_granularity, - date_part=path_key.date_part, - ) - for path_key in self.path_key_to_linkable_dimensions.keys() - if path_key.element_type is LinkableElementType.TIME_DIMENSION and path_key.time_granularity - ), - entity_specs=tuple( - EntitySpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - for path_key in self.path_key_to_linkable_entities - ), - group_by_metric_specs=tuple( - GroupByMetricSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - for path_key in self.path_key_to_linkable_metrics - ), - ) - @property def only_unique_path_keys(self) -> LinkableElementSet: """Returns a set that only includes path keys that map to a single distinct element.""" @@ -290,3 +288,104 @@ def only_unique_path_keys(self) -> LinkableElementSet: if len(set(linkable_metrics)) <= 1 }, ) + + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + semantic_model_references: Set[SemanticModelReference] = set() + for linkable_dimensions in self.path_key_to_linkable_dimensions.values(): + for linkable_dimension in linkable_dimensions: + semantic_model_references.update(linkable_dimension.derived_from_semantic_models) + for linkable_entities in self.path_key_to_linkable_entities.values(): + for linkable_entity in linkable_entities: + semantic_model_references.update(linkable_entity.derived_from_semantic_models) + for linkable_metrics in self.path_key_to_linkable_metrics.values(): + for linkable_metric in linkable_metrics: + semantic_model_references.update(linkable_metric.derived_from_semantic_models) + + return sorted(semantic_model_references, key=lambda reference: reference.semantic_model_name) + + @property + def spec_count(self) -> int: + """If this is mapped to spec objects, the number of specs that would be produced.""" + return ( + len(self.path_key_to_linkable_dimensions.keys()) + + len(self.path_key_to_linkable_entities.keys()) + + len(self.path_key_to_linkable_metrics.keys()) + ) + + @property + def specs(self) -> Sequence[LinkableInstanceSpec]: + """Converts the items in a `LinkableElementSet` to their corresponding spec objects.""" + specs: List[LinkableInstanceSpec] = [] + + for path_key in ( + tuple(self.path_key_to_linkable_dimensions.keys()) + + tuple(self.path_key_to_linkable_entities.keys()) + + tuple(self.path_key_to_linkable_metrics.keys()) + ): + specs.append(self._path_key_to_spec(path_key)) + + return specs + + def _path_key_to_spec(self, path_key: ElementPathKey) -> LinkableInstanceSpec: + if path_key.element_type is LinkableElementType.DIMENSION: + return DimensionSpec( + element_name=path_key.element_name, + entity_links=path_key.entity_links, + ) + elif path_key.element_type is LinkableElementType.TIME_DIMENSION: + assert path_key.time_granularity is not None + return TimeDimensionSpec( + element_name=path_key.element_name, + entity_links=path_key.entity_links, + time_granularity=path_key.time_granularity, + date_part=path_key.date_part, + ) + elif path_key.element_type is LinkableElementType.ENTITY: + return EntitySpec( + element_name=path_key.element_name, + entity_links=path_key.entity_links, + ) + elif path_key.element_type is LinkableElementType.METRIC: + return GroupByMetricSpec( + element_name=path_key.element_name, + entity_links=path_key.entity_links, + ) + else: + assert_values_exhausted(path_key.element_type) + + def filter_by_spec_patterns(self, spec_patterns: Sequence[SpecPattern]) -> LinkableElementSet: + """Filter the elements in the set by the given spec patters. + + Returns a new set consisting of the elements in the `LinkableElementSet` that have a corresponding spec that + match all the given spec patterns. + """ + # Spec patterns need all specs to match properly e.g. `BaseTimeGrainPattern`. + matching_specs: Sequence[InstanceSpec] = self.specs + + for spec_pattern in spec_patterns: + matching_specs = spec_pattern.match(matching_specs) + specs_to_include = set(matching_specs) + + path_key_to_linkable_dimensions: Dict[ElementPathKey, Tuple[LinkableDimension, ...]] = {} + path_key_to_linkable_entities: Dict[ElementPathKey, Tuple[LinkableEntity, ...]] = {} + path_key_to_linkable_metrics: Dict[ElementPathKey, Tuple[LinkableMetric, ...]] = {} + + for path_key, linkable_dimensions in self.path_key_to_linkable_dimensions.items(): + if self._path_key_to_spec(path_key) in specs_to_include: + path_key_to_linkable_dimensions[path_key] = linkable_dimensions + + for path_key, linkable_entities in self.path_key_to_linkable_entities.items(): + if self._path_key_to_spec(path_key) in specs_to_include: + path_key_to_linkable_entities[path_key] = linkable_entities + + for path_key, linkable_metrics in self.path_key_to_linkable_metrics.items(): + if self._path_key_to_spec(path_key) in specs_to_include: + path_key_to_linkable_metrics[path_key] = linkable_metrics + + return LinkableElementSet( + path_key_to_linkable_dimensions=path_key_to_linkable_dimensions, + path_key_to_linkable_entities=path_key_to_linkable_entities, + path_key_to_linkable_metrics=path_key_to_linkable_metrics, + ) From 9cb1650a7865ffd3877dc295c1617b87e6eb134a Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 14:16:08 -0700 Subject: [PATCH 07/14] Use `LinkableElementSet` instead of specs in group-by-item resolution classes. Using `LinkableElementSet` allows retrieval of the semantic models that are needed for computation. --- .../group_by_item_candidate.py | 63 ++++++--- .../candidate_push_down/push_down_visitor.py | 122 +++++++++--------- .../filter_spec_lookup.py | 20 ++- .../filter_spec_resolver.py | 2 +- .../group_by_item/group_by_item_resolver.py | 40 ++++-- .../query/query_parser.py | 4 +- .../query/query_resolution.py | 14 +- .../query/query_resolver.py | 17 ++- 8 files changed, 179 insertions(+), 103 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py index bedfe43650..edecad3525 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/group_by_item_candidate.py @@ -1,19 +1,25 @@ from __future__ import annotations import itertools +import logging from dataclasses import dataclass -from typing import Iterable, Tuple +from typing import Sequence, Tuple +from dbt_semantic_interfaces.references import SemanticModelReference from typing_extensions import override +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.query.group_by_item.path_prefixable import PathPrefixable from metricflow_semantics.query.group_by_item.resolution_path import MetricFlowQueryResolutionPath from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec, LinkableSpecSet +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec + +logger = logging.getLogger(__name__) @dataclass(frozen=True) -class GroupByItemCandidateSet(PathPrefixable): +class GroupByItemCandidateSet(PathPrefixable, SemanticModelDerivation): """The set of candidate specs that could match a given spec pattern. This candidate set is refined as it is passed from the root node (representing a measure) to the leaf node @@ -29,7 +35,7 @@ class GroupByItemCandidateSet(PathPrefixable): error messages, you start analyzing from the leaf node. """ - specs: Tuple[LinkableInstanceSpec, ...] + linkable_element_set: LinkableElementSet measure_paths: Tuple[MetricFlowQueryResolutionPath, ...] path_from_leaf_node: MetricFlowQueryResolutionPath @@ -39,18 +45,30 @@ def __post_init__(self) -> None: # noqa: D105 len(self.specs) == 0 and len(self.measure_paths) == 0 ) + @property + def specs(self) -> Sequence[LinkableInstanceSpec]: # noqa: D102 + return self.linkable_element_set.specs + @staticmethod def intersection( - path_from_leaf_node: MetricFlowQueryResolutionPath, candidate_sets: Iterable[GroupByItemCandidateSet] + path_from_leaf_node: MetricFlowQueryResolutionPath, candidate_sets: Sequence[GroupByItemCandidateSet] ) -> GroupByItemCandidateSet: """Create a new candidate set that is the intersection of the given candidate sets. The intersection is defined as the specs common to all candidate sets. path_from_leaf_node is used to indicate where the new candidate set was created. """ - specs_as_sets = tuple(set(candidate_set.specs) for candidate_set in candidate_sets) - common_specs = set.intersection(*specs_as_sets) if specs_as_sets else set() - if len(common_specs) == 0: + if len(candidate_sets) == 0: + return GroupByItemCandidateSet.empty_instance() + elif len(candidate_sets) == 1: + return GroupByItemCandidateSet( + linkable_element_set=candidate_sets[0].linkable_element_set, + measure_paths=candidate_sets[0].measure_paths, + path_from_leaf_node=path_from_leaf_node, + ) + linkable_element_set_candidates = tuple(candidate_set.linkable_element_set for candidate_set in candidate_sets) + intersection_result = LinkableElementSet.intersection_by_path_key(linkable_element_set_candidates) + if intersection_result.spec_count == 0: return GroupByItemCandidateSet.empty_instance() measure_paths = tuple( @@ -58,7 +76,9 @@ def intersection( ) return GroupByItemCandidateSet( - specs=tuple(common_specs), measure_paths=measure_paths, path_from_leaf_node=path_from_leaf_node + linkable_element_set=intersection_result, + measure_paths=measure_paths, + path_from_leaf_node=path_from_leaf_node, ) @property @@ -72,31 +92,34 @@ def num_candidates(self) -> int: # noqa: D102 @staticmethod def empty_instance() -> GroupByItemCandidateSet: # noqa: D102 return GroupByItemCandidateSet( - specs=(), measure_paths=(), path_from_leaf_node=MetricFlowQueryResolutionPath.empty_instance() + linkable_element_set=LinkableElementSet(), + measure_paths=(), + path_from_leaf_node=MetricFlowQueryResolutionPath.empty_instance(), ) - @property - def spec_set(self) -> LinkableSpecSet: - """Return the candidates as a spec set.""" - return LinkableSpecSet.from_specs(self.specs) - def filter_candidates_by_pattern( self, spec_pattern: SpecPattern, ) -> GroupByItemCandidateSet: """Return a new candidate set that only contains specs that match the given pattern.""" - matching_specs = tuple(InstanceSpecSet.from_specs(spec_pattern.match(self.specs)).linkable_specs) - if len(matching_specs) == 0: + filtered_element_set = self.linkable_element_set.filter_by_spec_patterns((spec_pattern,)) + if filtered_element_set.spec_count == 0: return GroupByItemCandidateSet.empty_instance() - return GroupByItemCandidateSet( - specs=matching_specs, measure_paths=self.measure_paths, path_from_leaf_node=self.path_from_leaf_node + linkable_element_set=filtered_element_set, + measure_paths=self.measure_paths, + path_from_leaf_node=self.path_from_leaf_node, ) @override def with_path_prefix(self, path_prefix: MetricFlowQueryResolutionPath) -> GroupByItemCandidateSet: return GroupByItemCandidateSet( - specs=self.specs, + linkable_element_set=self.linkable_element_set, measure_paths=tuple(path.with_path_prefix(path_prefix) for path in self.measure_paths), path_from_leaf_node=self.path_from_leaf_node.with_path_prefix(path_prefix), ) + + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + return self.linkable_element_set.derived_from_semantic_models diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py index bdc7e5933d..d44180783e 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py @@ -49,7 +49,6 @@ from metricflow_semantics.specs.patterns.base_time_grain import BaseTimeGrainPattern from metricflow_semantics.specs.patterns.none_date_part import NoneDatePartPattern from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec logger = logging.getLogger(__name__) @@ -156,13 +155,11 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu """Push the group-by-item specs that are available to the measure and match the source patterns to the child.""" with self._path_from_start_node_tracker.track_node_visit(node) as current_traversal_path: logger.info(f"Handling {node.ui_description}") - specs_available_for_measure: Sequence[ - LinkableInstanceSpec - ] = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_measure( + items_available_for_measure = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_measure( measure_reference=node.measure_reference, with_any_of=self._with_any_property, without_any_of=self._without_any_property, - ).as_spec_set.as_tuple + ) # The following is needed to handle limitation of cumulative metrics. Filtering could be done at the measure # node, but doing it here makes it a little easier to generate the error message. @@ -192,32 +189,28 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu else: assert_values_exhausted(metric.type) - specs_available_for_measure_given_child_metric = specs_available_for_measure - - for pattern_to_apply in patterns_to_apply: - specs_available_for_measure_given_child_metric = InstanceSpecSet.from_specs( - pattern_to_apply.match(specs_available_for_measure_given_child_metric) - ).linkable_specs - - matching_specs = specs_available_for_measure_given_child_metric - - for source_spec_pattern in self._source_spec_patterns: - matching_specs = InstanceSpecSet.from_specs(source_spec_pattern.match(matching_specs)).linkable_specs + matching_items = items_available_for_measure.filter_by_spec_patterns( + patterns_to_apply + self._source_spec_patterns + ) - logger.debug( - f"For {node.ui_description}:\n" - + indent( - "After applying patterns:\n" - + indent(mf_pformat(patterns_to_apply)) - + "\n" - + "to inputs, matches are:\n" - + indent(mf_pformat(matching_specs)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + f"For {node.ui_description}:\n" + + indent( + "After applying patterns:\n" + + indent(mf_pformat(patterns_to_apply)) + + "\n" + + "to inputs, matches are:\n" + + indent(mf_pformat(matching_items.specs)) + ) ) - ) # The specified patterns don't match to any of the available group-by-items that can be queried for the # measure. - if len(matching_specs) == 0: + if matching_items.spec_count == 0: + items_available_for_measure_given_child_metric = items_available_for_measure.filter_by_spec_patterns( + patterns_to_apply + ) return PushDownResult( candidate_set=GroupByItemCandidateSet.empty_instance(), issue_set=MetricFlowQueryResolutionIssueSet.from_issue( @@ -227,7 +220,7 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu input_suggestions=( tuple( self._suggestion_generator.input_suggestions( - specs_available_for_measure_given_child_metric + items_available_for_measure_given_child_metric.specs ) ) if self._suggestion_generator is not None @@ -240,7 +233,7 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu return PushDownResult( candidate_set=GroupByItemCandidateSet( measure_paths=(current_traversal_path,), - specs=tuple(matching_specs), + linkable_element_set=matching_items, path_from_leaf_node=current_traversal_path, ), issue_set=MetricFlowQueryResolutionIssueSet(), @@ -314,9 +307,10 @@ def visit_metric_node(self, node: MetricGroupByItemResolutionNode) -> PushDownRe current_traversal_path=current_traversal_path, ) logger.info(f"Handling {node.ui_description}") - logger.debug( - "candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) - ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) + ) if merged_result_from_parents.candidate_set.is_empty: return merged_result_from_parents @@ -340,43 +334,47 @@ def visit_metric_node(self, node: MetricGroupByItemResolutionNode) -> PushDownRe else: assert_values_exhausted(metric.type) - candidate_specs: Sequence[LinkableInstanceSpec] = merged_result_from_parents.candidate_set.specs + candidate_items = merged_result_from_parents.candidate_set.linkable_element_set issue_sets_to_merge = [merged_result_from_parents.issue_set] - matched_specs = candidate_specs - for pattern_to_apply in patterns_to_apply: - matched_specs = InstanceSpecSet.from_specs(pattern_to_apply.match(matched_specs)).linkable_specs - - logger.debug( - f"For {node.ui_description}:\n" - + indent( - "After applying patterns:\n" - + indent(mf_pformat(patterns_to_apply)) - + "\n" - + "to inputs, outputs are:\n" - + indent(mf_pformat(matched_specs)) + matched_items = candidate_items.filter_by_spec_patterns(patterns_to_apply) + + if logger.isEnabledFor(logging.DEBUG): + matched_specs = matched_items.specs + logger.debug( + f"For {node.ui_description}:\n" + + indent( + "After applying patterns:\n" + + indent(mf_pformat(patterns_to_apply)) + + "\n" + + "to inputs, outputs are:\n" + + indent(mf_pformat(matched_specs)) + ) ) - ) # There were candidates that were common from the ones passed from parents, but after applying the filters, # none of the candidates were valid. - if len(matched_specs) == 0: + if matched_items.spec_count == 0: issue_sets_to_merge.append( MetricFlowQueryResolutionIssueSet.from_issue( MetricExcludesDatePartIssue.from_parameters( query_resolution_path=current_traversal_path, - candidate_specs=candidate_specs, + candidate_specs=candidate_items.specs, parent_issues=(), ) ) ) + if matched_items.spec_count == 0: + return PushDownResult( + candidate_set=GroupByItemCandidateSet.empty_instance(), + issue_set=MetricFlowQueryResolutionIssueSet.merge_iterable(issue_sets_to_merge), + ) + return PushDownResult( candidate_set=GroupByItemCandidateSet( - specs=tuple(matched_specs), - measure_paths=( - merged_result_from_parents.candidate_set.measure_paths if len(matched_specs) > 0 else () - ), + linkable_element_set=matched_items, + measure_paths=merged_result_from_parents.candidate_set.measure_paths, path_from_leaf_node=current_traversal_path, ), issue_set=MetricFlowQueryResolutionIssueSet.merge_iterable(issue_sets_to_merge), @@ -395,11 +393,10 @@ def visit_query_node(self, node: QueryGroupByItemResolutionNode) -> PushDownResu }, current_traversal_path=current_traversal_path, ) - - logger.info(f"Handling {node.ui_description}") - logger.debug( - "candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) - ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Candidates from parents:\n" + indent(mf_pformat(merged_result_from_parents.candidate_set.specs)) + ) return merged_result_from_parents @@ -410,14 +407,11 @@ def visit_no_metrics_query_node(self, node: NoMetricsGroupByItemSourceNode) -> P logger.info(f"Handling {node.ui_description}") # This is a case for distinct dimension values from semantic models. candidate_elements = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_no_metrics_query() + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Candidate elements are:\n{mf_pformat(candidate_elements)}") + candidates_after_filtering = candidate_elements.filter_by_spec_patterns(self._source_spec_patterns) - matching_specs: Sequence[LinkableInstanceSpec] = tuple( - sorted(candidate_elements.as_spec_set.as_tuple, key=lambda x: x.qualified_name) - ) - for pattern_to_apply in self._source_spec_patterns: - matching_specs = InstanceSpecSet.from_specs(pattern_to_apply.match(matching_specs)).linkable_specs - - if len(matching_specs) == 0: + if candidates_after_filtering.spec_count == 0: return PushDownResult( candidate_set=GroupByItemCandidateSet.empty_instance(), issue_set=MetricFlowQueryResolutionIssueSet.from_issue( @@ -430,7 +424,7 @@ def visit_no_metrics_query_node(self, node: NoMetricsGroupByItemSourceNode) -> P return PushDownResult( candidate_set=GroupByItemCandidateSet( - specs=tuple(matching_specs), + linkable_element_set=candidates_after_filtering, measure_paths=(current_traversal_path,), path_from_leaf_node=current_traversal_path, ), diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py index 2c6468a5f9..6624928c9a 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_lookup.py @@ -24,6 +24,7 @@ from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern if TYPE_CHECKING: + from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec logger = logging.getLogger(__name__) @@ -170,13 +171,30 @@ class FilterSpecResolution: lookup_key: ResolvedSpecLookUpKey where_filter_intersection: WhereFilterIntersection - resolved_spec: Optional[LinkableInstanceSpec] + resolved_linkable_element_set: Optional[LinkableElementSet] spec_pattern: SpecPattern issue_set: MetricFlowQueryResolutionIssueSet # Used for error messages. filter_location_path: MetricFlowQueryResolutionPath object_builder_str: str + def __post_init__(self) -> None: # noqa: D105 + if self.resolved_linkable_element_set is not None: + assert len(self.resolved_linkable_element_set.specs) <= 1 + + @property + def resolved_spec(self) -> Optional[LinkableInstanceSpec]: # noqa: D102 + if self.resolved_linkable_element_set is None: + return None + + specs = self.resolved_linkable_element_set.specs + if len(specs) == 0: + return None + elif len(specs) == 1: + return specs[0] + else: + raise RuntimeError(f"Found {len(specs)} in {self.resolved_linkable_element_set}") + CallParameterSet = Union[ DimensionCallParameterSet, TimeDimensionCallParameterSet, EntityCallParameterSet, MetricCallParameterSet diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py index c698ad5c99..0c3efcb5cc 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/filter_spec_resolution/filter_spec_resolver.py @@ -362,7 +362,7 @@ def _resolve_specs_for_where_filters( call_parameter_set=group_by_item_in_where_filter.call_parameter_set, ), filter_location_path=resolution_path, - resolved_spec=group_by_item_resolution.spec, + resolved_linkable_element_set=group_by_item_resolution.linkable_element_set, where_filter_intersection=where_filter_intersection, spec_pattern=group_by_item_in_where_filter.spec_pattern, issue_set=group_by_item_resolution.issue_set.with_path_prefix(path_prefix), diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py index e501b2a499..5bda159f03 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/group_by_item_resolver.py @@ -2,16 +2,19 @@ import logging from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Optional, Sequence, Tuple from dbt_semantic_interfaces.call_parameter_sets import TimeDimensionCallParameterSet from dbt_semantic_interfaces.naming.keywords import METRIC_TIME_ELEMENT_NAME -from dbt_semantic_interfaces.references import TimeDimensionReference +from dbt_semantic_interfaces.references import SemanticModelReference, TimeDimensionReference from dbt_semantic_interfaces.type_enums import TimeGranularity +from typing_extensions import override from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme from metricflow_semantics.query.group_by_item.candidate_push_down.push_down_visitor import ( PushDownResult, @@ -27,13 +30,14 @@ from metricflow_semantics.specs.patterns.base_time_grain import BaseTimeGrainPattern from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.patterns.typed_patterns import TimeDimensionPattern -from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec, LinkableSpecSet +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet, group_specs_by_type logger = logging.getLogger(__name__) @dataclass(frozen=True) -class GroupByItemResolution: +class GroupByItemResolution(SemanticModelDerivation): """Result object that contains matching spec for a potentially ambiguous input to a query. e.g. "TimeDimension('metric_time')" -> TimeDimensionSpec('metric_time', DAY) @@ -43,8 +47,14 @@ class GroupByItemResolution: # If the spec is None, then the pattern couldn't be resolved spec: Optional[LinkableInstanceSpec] + linkable_element_set: LinkableElementSet issue_set: MetricFlowQueryResolutionIssueSet + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + return self.linkable_element_set.derived_from_semantic_models + @dataclass(frozen=True) class AvailableGroupByItemsResolution: @@ -86,6 +96,7 @@ def resolve_matching_item_for_querying( if push_down_result.candidate_set.num_candidates == 0: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set, ) @@ -101,6 +112,7 @@ def resolve_matching_item_for_querying( if push_down_result.candidate_set.num_candidates > 1: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set.add_issue( AmbiguousGroupByItemIssue.from_parameters( candidate_set=push_down_result.candidate_set, @@ -111,7 +123,11 @@ def resolve_matching_item_for_querying( ), ) - return GroupByItemResolution(spec=push_down_result.candidate_set.specs[0], issue_set=push_down_result.issue_set) + return GroupByItemResolution( + spec=push_down_result.candidate_set.specs[0], + linkable_element_set=push_down_result.candidate_set.linkable_element_set, + issue_set=push_down_result.issue_set, + ) def resolve_matching_item_for_filters( self, @@ -147,12 +163,14 @@ def resolve_matching_item_for_filters( if push_down_result.candidate_set.num_candidates == 0: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set, ) if push_down_result.candidate_set.num_candidates > 1: return GroupByItemResolution( spec=None, + linkable_element_set=LinkableElementSet(), issue_set=push_down_result.issue_set.add_issue( AmbiguousGroupByItemIssue.from_parameters( candidate_set=push_down_result.candidate_set, @@ -163,7 +181,11 @@ def resolve_matching_item_for_filters( ), ) - return GroupByItemResolution(spec=push_down_result.candidate_set.specs[0], issue_set=push_down_result.issue_set) + return GroupByItemResolution( + spec=push_down_result.candidate_set.specs[0], + linkable_element_set=push_down_result.candidate_set.linkable_element_set, + issue_set=push_down_result.issue_set, + ) def resolve_available_items( self, @@ -186,7 +208,7 @@ def resolve_available_items( push_down_result: PushDownResult = resolution_node.accept(push_down_visitor) return AvailableGroupByItemsResolution( - specs=push_down_result.candidate_set.specs, + specs=tuple(push_down_result.candidate_set.specs), issue_set=push_down_result.issue_set, ) @@ -202,9 +224,9 @@ def resolve_min_metric_time_grain(self) -> TimeGranularity: suggestion_generator=None, ) metric_time_spec_set = ( - LinkableSpecSet.from_specs((metric_time_grain_resolution.spec,)) + group_specs_by_type((metric_time_grain_resolution.spec,)) if metric_time_grain_resolution.spec is not None - else LinkableSpecSet.empty_instance() + else InstanceSpecSet.empty_instance() ) if len(metric_time_spec_set.time_dimension_specs) != 1: raise RuntimeError( diff --git a/metricflow-semantics/metricflow_semantics/query/query_parser.py b/metricflow-semantics/metricflow_semantics/query/query_parser.py index c88f9929fc..673e438661 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_parser.py +++ b/metricflow-semantics/metricflow_semantics/query/query_parser.py @@ -58,9 +58,9 @@ from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.time.time_granularity import ( adjust_to_end_of_period, adjust_to_start_of_period, @@ -160,7 +160,7 @@ def _metric_time_granularity(time_dimension_specs: Sequence[TimeDimensionSpec]) ): matching_specs = pattern_to_apply.match(matching_specs) # The conversion below is awkward and needs some more thought. - time_dimension_specs = InstanceSpecSet.from_specs(matching_specs).time_dimension_specs + time_dimension_specs = group_specs_by_type(matching_specs).time_dimension_specs if len(time_dimension_specs) == 0: return None diff --git a/metricflow-semantics/metricflow_semantics/query/query_resolution.py b/metricflow-semantics/metricflow_semantics/query/query_resolution.py index 5111495975..e76e8e8e39 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_resolution.py +++ b/metricflow-semantics/metricflow_semantics/query/query_resolution.py @@ -1,12 +1,14 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional, Sized, Tuple +from typing import Optional, Sequence, Sized, Tuple +from dbt_semantic_interfaces.references import SemanticModelReference from typing_extensions import override from metricflow_semantics.collection_helpers.merger import Mergeable from metricflow_semantics.mf_logging.pretty_print import mf_pformat +from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( FilterSpecResolutionLookUp, ) @@ -67,12 +69,11 @@ def __len__(self) -> int: @dataclass(frozen=True) -class MetricFlowQueryResolution: +class MetricFlowQueryResolution(SemanticModelDerivation): """The result of resolving query inputs to specs.""" # Can be None if there were errors. query_spec: Optional[MetricFlowQuerySpec] - # The resolution DAG generated for the query. resolution_dag: Optional[GroupByItemResolutionDag] # The lookup that is used later in the DataflowPlanBuilder to figure out which specs are required by the filters @@ -80,6 +81,13 @@ class MetricFlowQueryResolution: filter_spec_lookup: FilterSpecResolutionLookUp # Mapping of issues with the inputs. input_to_issue_set: InputToIssueSetMapping + # The semantic models that would be queried to resolve the query. + queried_semantic_models: Tuple[SemanticModelReference, ...] + + @property + @override + def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: + return self.queried_semantic_models @property def checked_query_spec(self) -> MetricFlowQuerySpec: diff --git a/metricflow-semantics/metricflow_semantics/query/query_resolver.py b/metricflow-semantics/metricflow_semantics/query/query_resolver.py index ac5b49dbdf..1a41d3418c 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_resolver.py +++ b/metricflow-semantics/metricflow_semantics/query/query_resolver.py @@ -9,6 +9,7 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.mf_logging.runtime import log_runtime from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.metric_scheme import MetricNamingScheme from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_pattern_factory import ( WhereFilterPatternFactory, @@ -56,10 +57,10 @@ from metricflow_semantics.specs.spec_classes import ( InstanceSpec, LinkableInstanceSpec, - LinkableSpecSet, MetricSpec, OrderBySpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type logger = logging.getLogger(__name__) @@ -103,6 +104,7 @@ class ResolveGroupByItemsResult: resolution_dag: GroupByItemResolutionDag group_by_item_specs: Tuple[LinkableInstanceSpec, ...] input_to_issue_set_mapping: InputToIssueSetMapping + linkable_element_set: LinkableElementSet @dataclass(frozen=True) @@ -231,6 +233,7 @@ def _resolve_group_by_items_result( input_to_issue_set_mapping_items: List[InputToIssueSetMappingItem] = [] group_by_item_specs: List[LinkableInstanceSpec] = [] + linkable_element_sets: List[LinkableElementSet] = [] for group_by_item_input in group_by_item_inputs: resolution = MetricFlowQueryResolver._resolve_group_by_item_input( group_by_item_resolver=group_by_item_resolver, @@ -243,11 +246,13 @@ def _resolve_group_by_items_result( ) if resolution.spec is not None: group_by_item_specs.append(resolution.spec) + linkable_element_sets.append(resolution.linkable_element_set) return ResolveGroupByItemsResult( resolution_dag=resolution_dag, group_by_item_specs=tuple(group_by_item_specs), input_to_issue_set_mapping=InputToIssueSetMapping(tuple(input_to_issue_set_mapping_items)), + linkable_element_set=LinkableElementSet.merge_by_path_key(linkable_element_sets), ) @staticmethod @@ -419,6 +424,7 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=None, filter_spec_lookup=FilterSpecResolutionLookUp.empty_instance(), input_to_issue_set=issue_set_mapping_so_far, + queried_semantic_models=(), ) # Resolve group by items. @@ -478,11 +484,12 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=resolution_dag, filter_spec_lookup=filter_spec_lookup, input_to_issue_set=issue_set_mapping_so_far, + queried_semantic_models=(), ) # No errors. - linkable_spec_set = LinkableSpecSet.from_specs(group_by_item_specs) - logger.info(f"Group-by-items were resolved to:\n{mf_pformat(linkable_spec_set.as_tuple)}") + linkable_spec_set = group_specs_by_type(group_by_item_specs) + logger.info(f"Group-by-items were resolved to:\n{mf_pformat(linkable_spec_set.linkable_specs)}") # Run post-resolution validation rules to generate issues that are generated at the query-level. query_level_issue_set = self._post_resolution_query_validator.validate_query( @@ -510,6 +517,7 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=resolution_dag, filter_spec_lookup=filter_spec_lookup, input_to_issue_set=issue_set_mapping, + queried_semantic_models=(), ) return MetricFlowQueryResolution( @@ -527,4 +535,7 @@ def _resolve_query(self, resolver_input_for_query: ResolverInputForQuery) -> Met resolution_dag=resolution_dag, filter_spec_lookup=filter_spec_lookup, input_to_issue_set=issue_set_mapping, + queried_semantic_models=tuple( + resolve_group_by_item_result.linkable_element_set.derived_from_semantic_models + ), ) From 5ed211e2ce31c330ac9e8cd587b8138db338188e Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 14:18:59 -0700 Subject: [PATCH 08/14] Associated import and method changes in the rest of the codebase. --- .../metricflow_semantics/instances.py | 4 +- .../naming/dunder_scheme.py | 5 +- .../naming/metric_scheme.py | 4 +- .../naming/object_builder_str.py | 7 +- .../test_helpers/snapshot_helpers.py | 3 +- .../semantics/test_linkable_spec_resolver.py | 16 +- .../model/test_semantic_model_container.py | 4 +- .../model/test_where_filter_spec.py | 224 +++++++++++++++--- .../naming/conftest.py | 2 +- .../test_object_builder_naming_scheme.py | 2 +- .../test_available_group_by_items.py | 5 +- .../patterns/test_entity_link_pattern.py | 2 +- .../specs/patterns/test_typed_patterns.py | 2 +- .../tests_metricflow_semantics/test_specs.py | 4 +- .../dataflow/builder/dataflow_plan_builder.py | 16 +- metricflow/dataflow/builder/node_evaluator.py | 7 +- metricflow/dataflow/builder/partitions.py | 4 +- metricflow/dataflow/nodes/filter_elements.py | 2 +- .../source_scan/matching_linkable_specs.py | 2 +- metricflow/engine/metricflow_engine.py | 2 +- metricflow/plan_conversion/dataflow_to_sql.py | 6 +- .../plan_conversion/instance_converters.py | 4 +- metricflow/plan_conversion/node_processor.py | 5 +- metricflow/plan_conversion/spec_transforms.py | 5 +- .../data_warehouse_model_validator.py | 3 +- .../source_scan/test_cm_branch_combiner.py | 3 +- tests_metricflow/examples/test_node_sql.py | 3 +- tests_metricflow/fixtures/setup_fixtures.py | 3 - ...select_columns_with_measures_aggregated.py | 3 +- .../test_dataflow_to_sql_plan.py | 17 +- 30 files changed, 269 insertions(+), 100 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/instances.py b/metricflow-semantics/metricflow_semantics/instances.py index 99f24e33fa..1ee3cd90f3 100644 --- a/metricflow-semantics/metricflow_semantics/instances.py +++ b/metricflow-semantics/metricflow_semantics/instances.py @@ -11,17 +11,17 @@ from metricflow_semantics.aggregation_properties import AggregationState from metricflow_semantics.specs.column_assoc import ColumnAssociation -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, + GroupByMetricSpec, InstanceSpec, - InstanceSpecSet, MeasureSpec, MetadataSpec, MetricSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet # Type for the specification used in the instance. SpecT = TypeVar("SpecT", bound=InstanceSpec) diff --git a/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py b/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py index 40c43f7a43..53a56acd14 100644 --- a/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py +++ b/metricflow-semantics/metricflow_semantics/naming/dunder_scheme.py @@ -17,9 +17,8 @@ ) from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, - InstanceSpecSetTransform, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform, group_spec_by_type class DunderNamingScheme(QueryItemNamingScheme): @@ -37,7 +36,7 @@ def date_part_suffix(date_part: DatePart) -> str: @override def input_str(self, instance_spec: InstanceSpec) -> Optional[str]: - spec_set = InstanceSpecSet.from_specs((instance_spec,)) + spec_set = group_spec_by_type(instance_spec) for time_dimension_spec in spec_set.time_dimension_specs: # From existing comment in StructuredLinkableSpecName: diff --git a/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py b/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py index 8fc0cd7c48..92015b4fa7 100644 --- a/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py +++ b/metricflow-semantics/metricflow_semantics/naming/metric_scheme.py @@ -9,8 +9,8 @@ from metricflow_semantics.specs.patterns.metric_pattern import MetricSpecPattern from metricflow_semantics.specs.spec_classes import ( InstanceSpec, - InstanceSpecSet, ) +from metricflow_semantics.specs.spec_set import group_spec_by_type class MetricNamingScheme(QueryItemNamingScheme): @@ -18,7 +18,7 @@ class MetricNamingScheme(QueryItemNamingScheme): @override def input_str(self, instance_spec: InstanceSpec) -> Optional[str]: - spec_set = InstanceSpecSet.from_specs((instance_spec,)) + spec_set = group_spec_by_type(instance_spec) names = tuple(spec.element_name for spec in spec_set.metric_specs) if len(names) != 1: diff --git a/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py b/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py index ce20280268..d1bd3453de 100644 --- a/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py +++ b/metricflow-semantics/metricflow_semantics/naming/object_builder_str.py @@ -14,7 +14,8 @@ from dbt_semantic_interfaces.type_enums.date_part import DatePart from typing_extensions import override -from metricflow_semantics.specs.spec_classes import InstanceSpec, InstanceSpecSet, InstanceSpecSetTransform +from metricflow_semantics.specs.spec_classes import InstanceSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform, group_spec_by_type class ObjectBuilderNameConverter: @@ -139,9 +140,7 @@ def transform(self, spec_set: InstanceSpecSet) -> Sequence[str]: @staticmethod def input_str_from_spec(instance_spec: InstanceSpec) -> str: # noqa: D102 - names = ObjectBuilderNameConverter._ObjectBuilderNameTransform().transform( - InstanceSpecSet.from_specs((instance_spec,)) - ) + names = ObjectBuilderNameConverter._ObjectBuilderNameTransform().transform(group_spec_by_type(instance_spec)) if len(names) != 1: raise RuntimeError(f"Did not get exactly 1 name from {instance_spec}. Got {names}") diff --git a/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py b/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py index 4354c6eed4..32c5fbd178 100644 --- a/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py +++ b/metricflow-semantics/metricflow_semantics/test_helpers/snapshot_helpers.py @@ -16,7 +16,8 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableSpecSet +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet +from metricflow_semantics.specs.spec_set import InstanceSpecSet logger = logging.getLogger(__name__) diff --git a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py index a67a0cfaee..bc7c201a1e 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_spec_resolver.py @@ -20,6 +20,7 @@ ValidLinkableSpecResolver, ) from metricflow_semantics.model.semantics.semantic_model_join_evaluator import MAX_JOIN_HOPS +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.snapshot_helpers import ( assert_linkable_element_set_snapshot_equal, @@ -193,15 +194,16 @@ def test_linkable_element_set_as_spec_set( double up on the .as_spec_set calls here. Yes, this is lazy. No, I don't care to make another helper to do snapshot comparisons on LinkableSpecSets. """ - linkable_spec_set = simple_model_spec_resolver.get_linkable_element_set_for_measure( - MeasureReference(element_name="listings"), - with_any_of=LinkableElementProperty.all_properties(), - without_any_of=frozenset({}), - ).as_spec_set - + linkable_spec_set = InstanceSpecSet.create_from_specs( + simple_model_spec_resolver.get_linkable_element_set_for_measure( + MeasureReference(element_name="listings"), + with_any_of=LinkableElementProperty.all_properties(), + without_any_of=frozenset({}), + ).specs + ) assert_spec_set_snapshot_equal( request=request, mf_test_configuration=mf_test_configuration, set_id="set0", - spec_set=linkable_spec_set.as_spec_set, + spec_set=linkable_spec_set, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py b/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py index 3eb4aee33f..b0649ba89f 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/test_semantic_model_container.py @@ -79,7 +79,7 @@ def test_local_linked_elements_for_metric( # noqa: D103 with_any_property=frozenset({LinkableElementProperty.LOCAL_LINKED}), without_any_property=frozenset({LinkableElementProperty.DERIVED_TIME_GRANULARITY}), ) - sorted_specs = sorted(linkable_elements.as_spec_set.as_tuple, key=lambda x: x.qualified_name) + sorted_specs = sorted(linkable_elements.specs, key=lambda x: x.qualified_name) assert_object_snapshot_equal( request=request, mf_test_configuration=mf_test_configuration, @@ -140,7 +140,7 @@ def test_linkable_elements_for_no_metrics_query( LinkableElementProperty.DERIVED_TIME_GRANULARITY, } ) - sorted_specs = sorted(linkable_elements.as_spec_set.as_tuple, key=lambda x: x.qualified_name) + sorted_specs = sorted(linkable_elements.specs, key=lambda x: x.qualified_name) assert_object_snapshot_equal( request=request, mf_test_configuration=mf_test_configuration, diff --git a/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py b/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py index 6e7d7cc42b..de5bf4b61e 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/test_where_filter_spec.py @@ -19,10 +19,20 @@ DimensionReference, EntityReference, MetricReference, + SemanticModelReference, TimeDimensionReference, ) +from dbt_semantic_interfaces.type_enums import DimensionType from dbt_semantic_interfaces.type_enums.date_part import DatePart from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from metricflow_semantics.model.semantics.linkable_element import ( + ElementPathKey, + LinkableDimension, + LinkableElementType, + LinkableEntity, + LinkableMetric, +) +from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_location import WhereFilterLocation from metricflow_semantics.query.group_by_item.filter_spec_resolution.filter_spec_lookup import ( @@ -34,12 +44,12 @@ from metricflow_semantics.query.group_by_item.resolution_path import MetricFlowQueryResolutionPath from metricflow_semantics.query.issues.issues_base import MetricFlowQueryResolutionIssueSet from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, + GroupByMetricSpec, LinkableInstanceSpec, - LinkableSpecSet, TimeDimensionSpec, WhereFilterSpec, ) @@ -51,7 +61,9 @@ def create_spec_lookup( - call_parameter_set: CallParameterSet, resolved_spec: LinkableInstanceSpec + call_parameter_set: CallParameterSet, + resolved_spec: LinkableInstanceSpec, + resolved_linkable_element_set: LinkableElementSet, ) -> FilterSpecResolutionLookUp: """Create a FilterSpecResolutionLookUp where the call_parameter_set maps to resolved_spec.""" return FilterSpecResolutionLookUp( @@ -63,7 +75,7 @@ def create_spec_lookup( ), filter_location_path=MetricFlowQueryResolutionPath.empty_instance(), where_filter_intersection=create_where_filter_intersection("Dimension('dummy__dimension')"), - resolved_spec=resolved_spec, + resolved_linkable_element_set=resolved_linkable_element_set, issue_set=MetricFlowQueryResolutionIssueSet.empty_instance(), spec_pattern=ObjectBuilderNamingScheme().spec_pattern("Dimension('dummy__dimension')"), object_builder_str="Dimension('dummy__dimension')", @@ -89,6 +101,28 @@ def test_dimension_in_filter( # noqa: D103 dimension_reference=DimensionReference("country_latest"), ), resolved_spec=DimensionSpec(element_name="country_latest", entity_links=(EntityReference("listing"),)), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="country_latest", + element_type=LinkableElementType.DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=None, + date_part=None, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.CATEGORICAL, + element_name="country_latest", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + time_granularity=None, + date_part=None, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, @@ -97,7 +131,7 @@ def test_dimension_in_filter( # noqa: D103 assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] assert where_filter_spec.where_sql == "listing__country_latest = 'US'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=( DimensionSpec(element_name="country_latest", entity_links=(EntityReference(element_name="listing"),)), ), @@ -115,29 +149,51 @@ def test_dimension_in_filter_with_grain( # noqa: D103 spec_resolution_lookup=create_spec_lookup( call_parameter_set=TimeDimensionCallParameterSet( entity_path=(EntityReference("listing"),), - time_dimension_reference=TimeDimensionReference("country_latest"), + time_dimension_reference=TimeDimensionReference("created_at"), time_granularity=TimeGranularity.WEEK, ), resolved_spec=TimeDimensionSpec( - element_name="country_latest", + element_name="created_at", entity_links=(EntityReference("listing"),), time_granularity=TimeGranularity.WEEK, ), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="created_at", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.WEEK, + date_part=None, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("listings_source"), + dimension_type=DimensionType.TIME, + element_name="created_at", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.WEEK, + date_part=None, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, filter_intersection=create_where_filter_intersection( - "{{ Dimension('listing__country_latest').grain('WEEK') }} = 'US'" + "{{ Dimension('listing__created_at').grain('WEEK') }} = 'US'" ), ) assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] - assert where_filter_spec.where_sql == "listing__country_latest__week = 'US'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert where_filter_spec.where_sql == "listing__created_at__week = 'US'" + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( - element_name="country_latest", + element_name="created_at", entity_links=(EntityReference(element_name="listing"),), time_granularity=TimeGranularity.WEEK, ), @@ -163,6 +219,28 @@ def test_time_dimension_in_filter( # noqa: D103 entity_links=(EntityReference("listing"),), time_granularity=TimeGranularity.MONTH, ), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="created_at", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.MONTH, + date_part=None, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("listings_source"), + dimension_type=DimensionType.CATEGORICAL, + element_name="created_at", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.MONTH, + date_part=None, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, @@ -173,7 +251,7 @@ def test_time_dimension_in_filter( # noqa: D103 assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] assert where_filter_spec.where_sql == "listing__created_at__month = '2020-01-01'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -204,6 +282,28 @@ def test_date_part_in_filter( # noqa: D103 time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR, ), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="metric_time", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.TIME, + element_name="metric_time", + entity_links=(), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ), + ) + } + ), ), ).create_from_where_filter_intersection( filter_location=EXAMPLE_FILTER_LOCATION, @@ -214,7 +314,7 @@ def test_date_part_in_filter( # noqa: D103 assert len(where_filter_specs) == 1 where_filter_spec = where_filter_specs[0] assert where_filter_spec.where_sql == "metric_time__extract_year = '2020'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -248,11 +348,27 @@ def resolved_spec_lookup() -> FilterSpecResolutionLookUp: where_filter_intersection=create_where_filter_intersection( "TimeDimension('metric_time', 'week', 'year')" ), - resolved_spec=TimeDimensionSpec( - element_name="metric_time", - entity_links=(), - time_granularity=TimeGranularity.WEEK, - date_part=DatePart.YEAR, + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="metric_time", + element_type=LinkableElementType.TIME_DIMENSION, + entity_links=(), + time_granularity=TimeGranularity.WEEK, + date_part=DatePart.YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.TIME, + element_name="metric_time", + entity_links=(), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.WEEK, + date_part=DatePart.YEAR, + ), + ) + } ), spec_pattern=ObjectBuilderNamingScheme().spec_pattern("Dimension('dummy__dimension')"), issue_set=MetricFlowQueryResolutionIssueSet.empty_instance(), @@ -284,7 +400,7 @@ def test_date_part_and_grain_in_filter( # noqa: D103 ).create_from_where_filter(EXAMPLE_FILTER_LOCATION, where_filter) assert where_filter_spec.where_sql == "metric_time__extract_year = '2020'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -321,7 +437,7 @@ def test_date_part_less_than_grain_in_filter( # noqa: D103 ).create_from_where_filter(EXAMPLE_FILTER_LOCATION, where_filter) assert where_filter_spec.where_sql == "metric_time__extract_day = '2020'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=( TimeDimensionSpec( @@ -352,11 +468,30 @@ def test_entity_in_filter( # noqa: D103 entity_reference=EntityReference("user"), ), resolved_spec=EntitySpec(element_name="user", entity_links=(EntityReference("listing"),)), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_entities={ + ElementPathKey( + element_name="user", + element_type=LinkableElementType.ENTITY, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ): ( + LinkableEntity( + semantic_model_origin=SemanticModelReference("bookings"), + element_name="user", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + ), + ) + } + ), ), ).create_from_where_filter(filter_location=EXAMPLE_FILTER_LOCATION, where_filter=where_filter) assert where_filter_spec.where_sql == "listing__user == 'example_user_id'" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=(), entity_specs=(EntitySpec(element_name="user", entity_links=(EntityReference(element_name="listing"),)),), @@ -379,11 +514,30 @@ def test_metric_in_filter( # noqa: D103 metric_reference=MetricReference("bookings"), ), resolved_spec=group_by_metric_spec, + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_metrics={ + ElementPathKey( + element_name="bookings", + element_type=LinkableElementType.METRIC, + entity_links=(EntityReference("listing"),), + time_granularity=None, + date_part=None, + ): ( + LinkableMetric( + join_by_semantic_model=SemanticModelReference("bookings"), + element_name="bookings", + entity_links=(EntityReference("listing"),), + join_path=(), + properties=frozenset(), + ), + ) + } + ), ), ).create_from_where_filter(filter_location=EXAMPLE_FILTER_LOCATION, where_filter=where_filter) assert where_filter_spec.where_sql == "listing__bookings > 2" - assert where_filter_spec.linkable_spec_set == LinkableSpecSet( + assert LinkableSpecSet.create_from_specs(where_filter_spec.linkable_specs) == LinkableSpecSet( dimension_specs=(), time_dimension_specs=(), entity_specs=(), @@ -411,11 +565,27 @@ def get_spec(dimension: str) -> WhereFilterSpec: ), filter_location_path=MetricFlowQueryResolutionPath(()), where_filter_intersection=PydanticWhereFilterIntersection(where_filters=[where_filter]), - resolved_spec=TimeDimensionSpec( - element_name=METRIC_TIME_ELEMENT_NAME, - entity_links=(), - time_granularity=TimeGranularity.WEEK, - date_part=DatePart.YEAR, + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name=METRIC_TIME_ELEMENT_NAME, + element_type=LinkableElementType.DIMENSION, + entity_links=(EntityReference("listing"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("bookings"), + dimension_type=DimensionType.TIME, + element_name=METRIC_TIME_ELEMENT_NAME, + entity_links=(), + join_path=(), + properties=frozenset(), + time_granularity=TimeGranularity.WEEK, + date_part=DatePart.YEAR, + ), + ) + } ), spec_pattern=ObjectBuilderNamingScheme().spec_pattern("Dimension('dummy__dimension')"), issue_set=MetricFlowQueryResolutionIssueSet.empty_instance(), diff --git a/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py b/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py index 2f89474db0..3dac120add 100644 --- a/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py +++ b/metricflow-semantics/tests_metricflow_semantics/naming/conftest.py @@ -6,10 +6,10 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, + GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py b/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py index 58de22b339..d5abdfcb27 100644 --- a/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py +++ b/metricflow-semantics/tests_metricflow_semantics/naming/test_object_builder_naming_scheme.py @@ -7,10 +7,10 @@ from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart from metricflow_semantics.naming.object_builder_scheme import ObjectBuilderNamingScheme -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, + GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py b/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py index ffb7af27ff..ad35fa2748 100644 --- a/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py +++ b/metricflow-semantics/tests_metricflow_semantics/query/group_by_item/test_available_group_by_items.py @@ -8,7 +8,8 @@ from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.query.group_by_item.group_by_item_resolver import GroupByItemResolver from metricflow_semantics.query.group_by_item.resolution_dag.dag import GroupByItemResolutionDag -from metricflow_semantics.specs.spec_classes import LinkableSpecSet +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.snapshot_helpers import assert_linkable_spec_set_snapshot_equal @@ -36,5 +37,5 @@ def test_available_group_by_items( # noqa: D103 request=request, mf_test_configuration=mf_test_configuration, set_id="set0", - spec_set=LinkableSpecSet.from_specs(result.specs), + spec_set=LinkableSpecSet.create_from_spec_set(group_specs_by_type(result.specs)), ) diff --git a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py index af5d270e11..cfb1c4da16 100644 --- a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py +++ b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_entity_link_pattern.py @@ -9,7 +9,6 @@ from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.patterns.entity_link_pattern import ( EntityLinkPattern, EntityLinkPatternParameterSet, @@ -18,6 +17,7 @@ from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, + GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py index 0bdb611568..4c16d8f016 100644 --- a/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py +++ b/metricflow-semantics/tests_metricflow_semantics/specs/patterns/test_typed_patterns.py @@ -18,7 +18,6 @@ ) from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.type_enums.date_part import DatePart -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.patterns.typed_patterns import ( DimensionPattern, EntityPattern, @@ -28,6 +27,7 @@ from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntitySpec, + GroupByMetricSpec, LinkableInstanceSpec, TimeDimensionSpec, ) diff --git a/metricflow-semantics/tests_metricflow_semantics/test_specs.py b/metricflow-semantics/tests_metricflow_semantics/test_specs.py index eac5e6a951..8413e1c20f 100644 --- a/metricflow-semantics/tests_metricflow_semantics/test_specs.py +++ b/metricflow-semantics/tests_metricflow_semantics/test_specs.py @@ -4,19 +4,19 @@ import pytest from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, EntitySpec, + GroupByMetricSpec, InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, LinklessEntitySpec, MeasureSpec, MetricSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet @pytest.fixture diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py index 405f33cf8f..c48457ae52 100644 --- a/metricflow/dataflow/builder/dataflow_plan_builder.py +++ b/metricflow/dataflow/builder/dataflow_plan_builder.py @@ -34,16 +34,15 @@ FilterSpecResolutionLookUp, ) from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec +from metricflow_semantics.specs.linkable_spec_set import LinkableSpecSet from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( ConstantPropertySpec, CumulativeMeasureDescription, EntitySpec, - InstanceSpecSet, + GroupByMetricSpec, JoinToTimeSpineDescription, LinkableInstanceSpec, - LinkableSpecSet, LinklessEntitySpec, MeasureSpec, MetadataSpec, @@ -54,6 +53,7 @@ TimeDimensionSpec, WhereFilterSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet, group_specs_by_type from metricflow_semantics.specs.where_filter_transform import WhereSpecFactory from metricflow_semantics.sql.sql_join_type import SqlJoinType @@ -309,7 +309,7 @@ def _build_aggregated_conversion_node( ) filtered_unaggregated_base_node = FilterElementsNode( parent_node=unaggregated_base_measure_node, - include_specs=InstanceSpecSet.from_specs(required_local_specs) + include_specs=group_specs_by_type(required_local_specs) .merge(base_required_linkable_specs.as_spec_set) .dedupe(), ) @@ -1217,9 +1217,11 @@ def __get_required_and_extraneous_linkable_specs( """ linkable_spec_sets_to_merge: List[LinkableSpecSet] = [] for filter_spec in filter_specs: - linkable_spec_sets_to_merge.append(filter_spec.linkable_spec_set) + linkable_spec_sets_to_merge.append(LinkableSpecSet.create_from_specs(filter_spec.linkable_specs)) if non_additive_dimension_spec: - linkable_spec_sets_to_merge.append(non_additive_dimension_spec.linkable_specs) + linkable_spec_sets_to_merge.append( + LinkableSpecSet.create_from_specs(non_additive_dimension_spec.linkable_specs) + ) extraneous_linkable_specs = LinkableSpecSet.merge_iterable(linkable_spec_sets_to_merge).dedupe() required_linkable_specs = queried_linkable_specs.merge(extraneous_linkable_specs).dedupe() @@ -1352,7 +1354,7 @@ def _build_aggregated_measure_from_measure_source_node( filtered_measure_source_node = FilterElementsNode( parent_node=join_to_time_spine_node or time_range_node or measure_recipe.source_node, include_specs=InstanceSpecSet(measure_specs=(measure_spec,)).merge( - InstanceSpecSet.from_specs(measure_recipe.required_local_linkable_specs), + group_specs_by_type(measure_recipe.required_local_linkable_specs), ), ) diff --git a/metricflow/dataflow/builder/node_evaluator.py b/metricflow/dataflow/builder/node_evaluator.py index 858f320059..7e6b36e066 100644 --- a/metricflow/dataflow/builder/node_evaluator.py +++ b/metricflow/dataflow/builder/node_evaluator.py @@ -27,11 +27,10 @@ from metricflow_semantics.model.semantics.semantic_model_join_evaluator import SemanticModelJoinEvaluator from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.specs.spec_classes import ( - InstanceSpecSet, LinkableInstanceSpec, - LinkableSpecSet, LinklessEntitySpec, ) +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow.dataflow.builder.node_data_set import DataflowPlanNodeOutputDataSetResolver @@ -120,7 +119,7 @@ def join_description(self) -> JoinDescription: ] ) filtered_node_to_join = FilterElementsNode( - parent_node=self.node_to_join, include_specs=InstanceSpecSet.from_specs(include_specs) + parent_node=self.node_to_join, include_specs=group_specs_by_type(include_specs) ) return JoinDescription( @@ -201,7 +200,7 @@ def _find_joinable_candidate_nodes_that_can_satisfy_linkable_specs( for right_node in self._nodes_available_for_joins: # If right node is time spine source node, use cross join. if right_node == self._time_spine_node: - needed_metric_time_specs = LinkableSpecSet.from_specs(needed_linkable_specs).metric_time_specs + needed_metric_time_specs = group_specs_by_type(needed_linkable_specs).metric_time_specs candidates_for_join.append( JoinLinkableInstancesRecipe( node_to_join=right_node, diff --git a/metricflow/dataflow/builder/partitions.py b/metricflow/dataflow/builder/partitions.py index 0d65e7f76f..510a9fa4ec 100644 --- a/metricflow/dataflow/builder/partitions.py +++ b/metricflow/dataflow/builder/partitions.py @@ -5,12 +5,12 @@ from typing import List, Sequence, Tuple from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup +from metricflow_semantics.specs.partition_spec_set import PartitionSpecSet from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - InstanceSpecSet, - PartitionSpecSet, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow.dataset.dataset_classes import DataSet diff --git a/metricflow/dataflow/nodes/filter_elements.py b/metricflow/dataflow/nodes/filter_elements.py index 3d110e7e9f..a20ef5c146 100644 --- a/metricflow/dataflow/nodes/filter_elements.py +++ b/metricflow/dataflow/nodes/filter_elements.py @@ -5,7 +5,7 @@ from metricflow_semantics.dag.id_prefix import IdPrefix, StaticIdPrefix from metricflow_semantics.dag.mf_dag import DisplayedProperty from metricflow_semantics.mf_logging.pretty_print import mf_pformat -from metricflow_semantics.specs.spec_classes import InstanceSpecSet +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.visitor import VisitorOutputT from metricflow.dataflow.dataflow_plan import BaseOutput, DataflowPlanNode, DataflowPlanNodeVisitor diff --git a/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py b/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py index e5b3c7da47..3aedbedf71 100644 --- a/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py +++ b/metricflow/dataflow/optimizer/source_scan/matching_linkable_specs.py @@ -1,6 +1,6 @@ from __future__ import annotations -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, InstanceSpecSetTransform +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform class MatchingLinkableSpecsTransform(InstanceSpecSetTransform[bool]): diff --git a/metricflow/engine/metricflow_engine.py b/metricflow/engine/metricflow_engine.py index 915a17ec81..60612b8d8a 100644 --- a/metricflow/engine/metricflow_engine.py +++ b/metricflow/engine/metricflow_engine.py @@ -32,7 +32,7 @@ from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver from metricflow_semantics.specs.query_param_implementations import SavedQueryParameter from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec -from metricflow_semantics.specs.spec_classes import InstanceSpecSet +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.time.time_source import TimeSource from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder diff --git a/metricflow/plan_conversion/dataflow_to_sql.py b/metricflow/plan_conversion/dataflow_to_sql.py index c8278444d0..772dae2ea4 100644 --- a/metricflow/plan_conversion/dataflow_to_sql.py +++ b/metricflow/plan_conversion/dataflow_to_sql.py @@ -30,14 +30,14 @@ ColumnAssociationResolver, SingleColumnCorrelationKey, ) -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( - InstanceSpecSet, + GroupByMetricSpec, MeasureSpec, MetadataSpec, MetricSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow_semantics.time.time_constants import ISO8601_PYTHON_FORMAT @@ -874,7 +874,7 @@ def visit_where_constraint_node(self, node: WhereConstraintNode) -> SqlDataSet: column_associations_in_where_sql: Sequence[ColumnAssociation] = CreateColumnAssociations( column_association_resolver=self._column_association_resolver - ).transform(spec_set=node.where.linkable_spec_set.as_spec_set) + ).transform(spec_set=InstanceSpecSet.create_from_specs(node.where.linkable_specs)) return SqlDataSet( instance_set=output_instance_set, diff --git a/metricflow/plan_conversion/instance_converters.py b/metricflow/plan_conversion/instance_converters.py index 51987abe0f..39fab0d316 100644 --- a/metricflow/plan_conversion/instance_converters.py +++ b/metricflow/plan_conversion/instance_converters.py @@ -29,19 +29,19 @@ from metricflow_semantics.model.semantics.metric_lookup import MetricLookup from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup from metricflow_semantics.specs.column_assoc import ColumnAssociationResolver -from metricflow_semantics.specs.group_by_metric_spec import GroupByMetricSpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, EntityReference, EntitySpec, + GroupByMetricSpec, InstanceSpec, - InstanceSpecSet, LinkableInstanceSpec, LinklessEntitySpec, MeasureSpec, MetricInputMeasureSpec, TimeDimensionSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from more_itertools import bucket from metricflow.dataflow.nodes.join_to_base import ValidityWindowJoinDescription diff --git a/metricflow/plan_conversion/node_processor.py b/metricflow/plan_conversion/node_processor.py index f58756542d..8499ca47b7 100644 --- a/metricflow/plan_conversion/node_processor.py +++ b/metricflow/plan_conversion/node_processor.py @@ -9,7 +9,8 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantics.semantic_model_join_evaluator import MAX_JOIN_HOPS from metricflow_semantics.model.semantics.semantic_model_lookup import SemanticModelLookup -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec, LinklessEntitySpec +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec, LinklessEntitySpec +from metricflow_semantics.specs.spec_set import group_specs_by_type from metricflow_semantics.specs.spec_set_transforms import ToElementNameSet from metricflow_semantics.sql.sql_join_type import SqlJoinType @@ -223,7 +224,7 @@ def _get_candidates_nodes_for_multi_hop( specs = data_set_of_second_node_that_can_be_joined.instance_set.spec_set filtered_joinable_node = FilterElementsNode( parent_node=second_node_that_could_be_joined, - include_specs=InstanceSpecSet.from_specs( + include_specs=group_specs_by_type( specs.dimension_specs + specs.entity_specs + specs.time_dimension_specs ), ) diff --git a/metricflow/plan_conversion/spec_transforms.py b/metricflow/plan_conversion/spec_transforms.py index 9cdab81984..b2e86ac0c7 100644 --- a/metricflow/plan_conversion/spec_transforms.py +++ b/metricflow/plan_conversion/spec_transforms.py @@ -3,10 +3,7 @@ from typing import List, Sequence from metricflow_semantics.specs.column_assoc import ColumnAssociation, ColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import ( - InstanceSpecSet, - InstanceSpecSetTransform, -) +from metricflow_semantics.specs.spec_set import InstanceSpecSet, InstanceSpecSetTransform from metricflow.plan_conversion.select_column_gen import SelectColumnSet from metricflow.plan_conversion.sql_expression_builders import make_coalesced_expr diff --git a/metricflow/validation/data_warehouse_model_validator.py b/metricflow/validation/data_warehouse_model_validator.py index 7d43e34e11..99a35829c2 100644 --- a/metricflow/validation/data_warehouse_model_validator.py +++ b/metricflow/validation/data_warehouse_model_validator.py @@ -30,7 +30,8 @@ ) from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, LinkableInstanceSpec, MeasureSpec +from metricflow_semantics.specs.spec_classes import LinkableInstanceSpec, MeasureSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow.dataflow.builder.node_data_set import DataflowPlanNodeOutputDataSetResolver diff --git a/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py b/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py index 56aca18e6c..88f959e50c 100644 --- a/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py +++ b/tests_metricflow/dataflow/optimizer/source_scan/test_cm_branch_combiner.py @@ -6,7 +6,8 @@ from _pytest.fixtures import FixtureRequest from metricflow_semantics.dag.id_prefix import StaticIdPrefix from metricflow_semantics.dag.mf_dag import DagId -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, MeasureSpec +from metricflow_semantics.specs.spec_classes import MeasureSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration from metricflow_semantics.test_helpers.snapshot_helpers import assert_plan_snapshot_text_equal diff --git a/tests_metricflow/examples/test_node_sql.py b/tests_metricflow/examples/test_node_sql.py index cd6e517563..d3c96fffc6 100644 --- a/tests_metricflow/examples/test_node_sql.py +++ b/tests_metricflow/examples/test_node_sql.py @@ -8,7 +8,8 @@ from metricflow_semantics.mf_logging.pretty_print import mf_pformat from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, TimeDimensionReference, TimeDimensionSpec +from metricflow_semantics.specs.spec_classes import TimeDimensionReference, TimeDimensionSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow.dataflow.builder.node_data_set import DataflowPlanNodeOutputDataSetResolver from metricflow.dataflow.nodes.filter_elements import FilterElementsNode diff --git a/tests_metricflow/fixtures/setup_fixtures.py b/tests_metricflow/fixtures/setup_fixtures.py index 3d441d9c83..c346d2c821 100644 --- a/tests_metricflow/fixtures/setup_fixtures.py +++ b/tests_metricflow/fixtures/setup_fixtures.py @@ -24,9 +24,6 @@ logger = logging.getLogger(__name__) -# from metricflow.test.time.configurable_time_source import ConfigurableTimeSource - - DISPLAY_GRAPHS_CLI_FLAG = "--display-graphs" USE_PERSISTENT_SOURCE_SCHEMA_CLI_FLAG = "--use-persistent-source-schema" diff --git a/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py b/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py index fbd9f7eb10..c07ade0bb6 100644 --- a/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py +++ b/tests_metricflow/plan_conversion/instance_converters/test_create_select_columns_with_measures_aggregated.py @@ -5,7 +5,8 @@ from metricflow_semantics.instances import InstanceSet from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.specs.dunder_column_association_resolver import DunderColumnAssociationResolver -from metricflow_semantics.specs.spec_classes import InstanceSpecSet, MeasureSpec, MetricInputMeasureSpec +from metricflow_semantics.specs.spec_classes import MeasureSpec, MetricInputMeasureSpec +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow.plan_conversion.instance_converters import ( CreateSelectColumnsWithMeasuresAggregated, diff --git a/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py b/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py index 73cee1d92c..5c1e4c51bf 100644 --- a/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py +++ b/tests_metricflow/plan_conversion/test_dataflow_to_sql_plan.py @@ -16,8 +16,6 @@ from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec from metricflow_semantics.specs.spec_classes import ( DimensionSpec, - InstanceSpecSet, - LinkableSpecSet, LinklessEntitySpec, MeasureSpec, MetricInputMeasureSpec, @@ -27,6 +25,7 @@ TimeDimensionSpec, WhereFilterSpec, ) +from metricflow_semantics.specs.spec_set import InstanceSpecSet from metricflow_semantics.sql.sql_bind_parameters import SqlBindParameters from metricflow_semantics.sql.sql_join_type import SqlJoinType from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration @@ -192,14 +191,12 @@ def test_filter_with_where_constraint_node( where_constraint=WhereFilterSpec( where_sql="booking__ds__day = '2020-01-01'", bind_parameters=SqlBindParameters(), - linkable_spec_set=LinkableSpecSet( - time_dimension_specs=( - TimeDimensionSpec( - element_name="ds", - entity_links=(EntityReference(element_name="booking"),), - time_granularity=TimeGranularity.DAY, - ), - ) + linkable_specs=( + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="booking"),), + time_granularity=TimeGranularity.DAY, + ), ), ), ) From 847a4d2da03add2b23b937c1efc04f3ab7114d88 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Wed, 24 Apr 2024 15:10:37 -0700 Subject: [PATCH 09/14] Add interface for `SavedQueryDependencyResolver`. --- .../metricflow_semantics/api/__init__.py | 0 .../metricflow_semantics/api/v0_1/__init__.py | 0 .../v0_1/saved_query_dependency_resolver.py | 75 +++++++++++++++++++ .../api/__init__.py | 0 .../api/v0_1/__init__.py | 0 .../test_saved_query_dependency_resolver.py | 25 +++++++ 6 files changed, 100 insertions(+) create mode 100644 metricflow-semantics/metricflow_semantics/api/__init__.py create mode 100644 metricflow-semantics/metricflow_semantics/api/v0_1/__init__.py create mode 100644 metricflow-semantics/metricflow_semantics/api/v0_1/saved_query_dependency_resolver.py create mode 100644 metricflow-semantics/tests_metricflow_semantics/api/__init__.py create mode 100644 metricflow-semantics/tests_metricflow_semantics/api/v0_1/__init__.py create mode 100644 metricflow-semantics/tests_metricflow_semantics/api/v0_1/test_saved_query_dependency_resolver.py diff --git a/metricflow-semantics/metricflow_semantics/api/__init__.py b/metricflow-semantics/metricflow_semantics/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/metricflow_semantics/api/v0_1/__init__.py b/metricflow-semantics/metricflow_semantics/api/v0_1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/metricflow_semantics/api/v0_1/saved_query_dependency_resolver.py b/metricflow-semantics/metricflow_semantics/api/v0_1/saved_query_dependency_resolver.py new file mode 100644 index 0000000000..50893504cf --- /dev/null +++ b/metricflow-semantics/metricflow_semantics/api/v0_1/saved_query_dependency_resolver.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Tuple + +from dbt_semantic_interfaces.protocols import SemanticManifest +from dbt_semantic_interfaces.references import ( + SemanticModelReference, +) + +from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup +from metricflow_semantics.query.query_parser import MetricFlowQueryParser +from metricflow_semantics.specs.query_param_implementations import SavedQueryParameter + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class SavedQueryDependencySet: + """The dependencies of a saved query. + + The primary use case is to handle creation of the cache item associated with the saved query. The dependencies + listed in this class must be up-to-date before the cache associated with the saved query can be created. Otherwise, + running the export / creating the cache may create a cache item that is out-of-date / unusable. + """ + + # The semantic models that the saved query depends on. + semantic_model_references: Tuple[SemanticModelReference, ...] + + +class SavedQueryDependencyResolver: + """Resolves the dependencies of a saved query. Also see `SavedQueryDependencySet`.""" + + def __init__(self, semantic_manifest: SemanticManifest) -> None: # noqa: D107 + self._semantic_manifest = semantic_manifest + self._query_parser = MetricFlowQueryParser(SemanticManifestLookup(semantic_manifest)) + + def _resolve_dependencies(self, saved_query_name: str) -> SavedQueryDependencySet: + parse_result = self._query_parser.parse_and_validate_saved_query( + saved_query_parameter=SavedQueryParameter(saved_query_name), + where_filter=None, + limit=None, + time_constraint_start=None, + time_constraint_end=None, + order_by_names=None, + order_by_parameters=None, + ) + + return SavedQueryDependencySet( + semantic_model_references=tuple( + sorted( + parse_result.queried_semantic_models, + key=lambda reference: reference.semantic_model_name, + ) + ), + ) + + def resolve_dependencies(self, saved_query_name: str) -> SavedQueryDependencySet: + """Return the dependencies of the given saved query in the manifest.""" + try: + return self._resolve_dependencies(saved_query_name) + except Exception: + logger.exception( + f"Got an exception while getting the dependencies of saved-query {repr(saved_query_name)}. " + f"All semantic models will be returned instead for safety." + ) + return SavedQueryDependencySet( + semantic_model_references=tuple( + sorted( + (semantic_model.reference for semantic_model in self._semantic_manifest.semantic_models), + key=lambda reference: reference.semantic_model_name, + ) + ), + ) diff --git a/metricflow-semantics/tests_metricflow_semantics/api/__init__.py b/metricflow-semantics/tests_metricflow_semantics/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/tests_metricflow_semantics/api/v0_1/__init__.py b/metricflow-semantics/tests_metricflow_semantics/api/v0_1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/metricflow-semantics/tests_metricflow_semantics/api/v0_1/test_saved_query_dependency_resolver.py b/metricflow-semantics/tests_metricflow_semantics/api/v0_1/test_saved_query_dependency_resolver.py new file mode 100644 index 0000000000..d8514db27e --- /dev/null +++ b/metricflow-semantics/tests_metricflow_semantics/api/v0_1/test_saved_query_dependency_resolver.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import pytest +from dbt_semantic_interfaces.implementations.semantic_manifest import ( + PydanticSemanticManifest, +) +from dbt_semantic_interfaces.references import ( + SemanticModelReference, +) +from metricflow_semantics.api.v0_1.saved_query_dependency_resolver import SavedQueryDependencyResolver + + +@pytest.fixture(scope="session") +def resolver( # noqa: D103 + simple_semantic_manifest: PydanticSemanticManifest, +) -> SavedQueryDependencyResolver: + return SavedQueryDependencyResolver(simple_semantic_manifest) + + +def test_saved_query_dependency_resolver(resolver: SavedQueryDependencyResolver) -> None: # noqa: D103 + dependency_set = resolver.resolve_dependencies("p0_booking") + assert tuple(dependency_set.semantic_model_references) == ( + SemanticModelReference(semantic_model_name="bookings_source"), + SemanticModelReference(semantic_model_name="listings_latest"), + ) From d31029e5d67dd2416f50613364c88933f50a5a58 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 12:20:15 -0700 Subject: [PATCH 10/14] Update snapshots for where filter changes. --- .../test_distinct_values_plan__dfp_0.xml | 28 ++-- ..._distinct_values_plan_with_join__dfp_0.xml | 28 ++-- ...join_to_time_spine_with_filters__dfp_0.xml | 60 +++----- .../test_measure_constraint_plan__dfp_0.xml | 132 +++++++----------- ...traint_with_reused_measure_plan__dfp_0.xml | 68 ++++----- ...t_metric_in_metric_where_filter__dfp_0.xml | 13 +- ...st_metric_in_query_where_filter__dfp_0.xml | 39 ++++-- ...ry_have_different_granularities__dfp_0.xml | 60 +++----- ...ry_have_different_granularities__dfp_0.xml | 71 ++++------ .../test_where_constrained_plan__dfp_0.xml | 66 ++++----- ...constrained_plan_time_dimension__dfp_0.xml | 50 +++---- ...ained_with_common_linkable_plan__dfp_0.xml | 66 ++++----- ...constrained_metric_not_combined__dfp_0.xml | 28 ++-- ...onstrained_metric_not_combined__dfpo_0.xml | 28 ++-- 14 files changed, 312 insertions(+), 425 deletions(-) diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml index bf661d834b..eb2573ef1e 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan__dfp_0.xml @@ -26,23 +26,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml index da62fd9e41..d74f407e48 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_distinct_values_plan_with_join__dfp_0.xml @@ -31,23 +31,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml index 55f773dbc8..68a39bcd80 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_with_filters__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -31,19 +26,12 @@ - - - - - - - - - - - - - + + + + + + @@ -68,12 +56,10 @@ - - - - - - + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml index ebd82f88a6..e4e3677e4f 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_plan__dfp_0.xml @@ -12,28 +12,22 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -46,23 +40,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + @@ -135,28 +123,22 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -169,23 +151,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml index 01e049e74c..7b35eae487 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_measure_constraint_with_reused_measure_plan__dfp_0.xml @@ -12,29 +12,23 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + @@ -47,23 +41,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml index 1f539c3a57..75a683ec01 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_metric_where_filter__dfp_0.xml @@ -17,8 +17,17 @@ - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml index f0a5328a55..41bf9e5c26 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_metric_in_query_where_filter__dfp_0.xml @@ -5,16 +5,22 @@ - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -26,8 +32,17 @@ - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml index 489ec19a70..1a30add9bb 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_to_grain_metric_filter_and_query_have_different_granularities__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -33,12 +28,10 @@ - - - - - - + + + + @@ -58,19 +51,12 @@ - - - - - - - - - - - - - + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml index 4e94e82589..944c59cc46 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_offset_window_metric_filter_and_query_have_different_granularities__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -36,12 +31,10 @@ - - - - - - + + + + @@ -64,12 +57,10 @@ - - - - - - + + + + @@ -120,12 +111,10 @@ - - - - - - + + + + @@ -147,12 +136,10 @@ - - - - - - + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml index b134b228aa..0955aebc17 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan__dfp_0.xml @@ -5,28 +5,22 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -43,23 +37,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml index b3ba6c0a1e..2eaabb15d5 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_plan_time_dimension__dfp_0.xml @@ -5,24 +5,19 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -39,19 +34,12 @@ - - - - - - - - - - - - - + + + + + + diff --git a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml index 7a9e744ad3..07b2aa8b7a 100644 --- a/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml +++ b/tests_metricflow/snapshots/test_dataflow_plan_builder.py/DataflowPlan/test_where_constrained_with_common_linkable_plan__dfp_0.xml @@ -5,51 +5,39 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml index 2171124c84..6b6d49f158 100644 --- a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml +++ b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfp_0.xml @@ -47,23 +47,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + diff --git a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml index 0ea28d8b15..f625d9c134 100644 --- a/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml +++ b/tests_metricflow/snapshots/test_source_scan_optimizer.py/DataflowPlan/test_constrained_metric_not_combined__dfpo_0.xml @@ -47,23 +47,17 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + From 76a11f91c24d2c68b77eef936d2e41a44a8d4b07 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 03:27:50 -0700 Subject: [PATCH 11/14] Update snapshots to include spec -> element set updates. --- ...ccumulate_last_2_months_metric__result.txt | 23 ++++++++- ...h_different_parent_time_grains__result.txt | 47 +++++++++++++++---- ...c_with_same_parent_time_grains__result.txt | 23 ++++++++- ...ics_with_different_time_grains__result.txt | 47 +++++++++++++++---- ..._metrics_with_same_time_grains__result.txt | 23 ++++++++- ...me_in_query_filter__no_metrics__result.txt | 20 +++++++- ...in_query_filter__simple_metric__result.txt | 23 ++++++++- .../test_invalid_group_by_item__result.txt | 1 + ..._item_in_derived_metric_parent__result.txt | 1 + ...cs_with_common_filtered_metric__result.txt | 40 +++++++++++++++- ...tion_for_invalid_metric_filter__result.txt | 47 +++++++++++++++---- ...or_invalid_metric_input_filter__result.txt | 47 +++++++++++++++---- ...lution_for_valid_metric_filter__result.txt | 20 +++++++- ..._for_valid_metric_input_filter__result.txt | 20 +++++++- ...ccumulate_last_2_months_metric__result.txt | 20 +++++++- ...h_different_parent_time_grains__result.txt | 47 +++++++++++++++---- ...c_with_same_parent_time_grains__result.txt | 20 +++++++- ...ics_with_different_time_grains__result.txt | 47 +++++++++++++++---- ..._metrics_with_same_time_grains__result.txt | 20 +++++++- ...spec_resolution__simple_metric__result.txt | 20 +++++++- 20 files changed, 483 insertions(+), 73 deletions(-) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__accumulate_last_2_months_metric__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt index 4f335a25f2..a151d9b75c 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_different_parent_time_grains__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoCommonItemsInParents( @@ -11,11 +12,24 @@ GroupByItemResolution( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -36,11 +50,24 @@ GroupByItemResolution( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__derived_metric_with_same_parent_time_grains__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt index ca5af91d22..9778252906 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_different_time_grains__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoCommonItemsInParents( @@ -8,11 +9,24 @@ GroupByItemResolution( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -31,11 +45,24 @@ GroupByItemResolution( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__metrics_with_same_time_grains__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt index d3552b1753..057b0dfe18 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__no_metrics__result.txt @@ -1 +1,19 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=DAY)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=DAY), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=DAY, + ): ( + LinkableDimension( + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=DAY, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt index a687fce8a6..310e9c9004 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_filters.py/GroupByItemResolution/test_ambiguous_metric_time_in_query_filter__simple_metric__result.txt @@ -1 +1,22 @@ -GroupByItemResolution(spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH)) +GroupByItemResolution( + spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), +) diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt index a0fcdd41f3..21234f22f4 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_invalid_group_by_item__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoMatchingItemsForMeasure( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt index 4eac17bbba..a3774cbe89 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_matching_item_for_querying.py/GroupByItemResolution/test_unavailable_group_by_item_in_derived_metric_parent__result.txt @@ -1,4 +1,5 @@ GroupByItemResolution( + linkable_element_set=LinkableElementSet(), issue_set=MetricFlowQueryResolutionIssueSet( issues=( NoMatchingItemsForMeasure( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt index e9dded7b67..e4e1f4ce43 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_derived_metrics_with_common_filtered_metric__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -60,7 +78,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt index 1c8462ad59..063f27b837 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_filter__result.txt @@ -22,6 +22,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -40,11 +41,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -65,11 +79,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt index d64f856af1..8e6b7a0ca8 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_invalid_metric_input_filter__result.txt @@ -22,6 +22,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -41,11 +42,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -68,11 +82,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt index ed4de4e885..60b59c8d79 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_filter__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt index 65513b2d5f..cc155e70a5 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_resolution_for_valid_metric_input_filter__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt index db27d0f522..7d77ed1031 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__accumulate_last_2_months_metric__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt index 7d961e2d45..58383f8a08 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_different_parent_time_grains__result.txt @@ -22,6 +22,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -40,11 +41,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -65,11 +79,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt index c9fd95b464..df8b92b453 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__derived_metric_with_same_parent_time_grains__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt index d8f991770d..2cc7ddb651 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_different_time_grains__result.txt @@ -25,6 +25,7 @@ FilterSpecResolutionLookUp( ), ], ), + resolved_linkable_element_set=LinkableElementSet(), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), @@ -42,11 +43,24 @@ FilterSpecResolutionLookUp( ), parent_candidate_sets=( GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=MONTH, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( @@ -65,11 +79,24 @@ FilterSpecResolutionLookUp( ), ), GroupByItemCandidateSet( - specs=( - TimeDimensionSpec( - element_name='metric_time', - time_granularity=YEAR, - ), + linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=YEAR, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='yearly_measure_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=YEAR, + ), + ), + }, ), measure_paths=( MetricFlowQueryResolutionPath( diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt index d6cbeec091..08165c1164 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__metrics_with_same_time_grains__result.txt @@ -25,7 +25,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), diff --git a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt index b8c3f57c16..30917c4241 100644 --- a/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt +++ b/metricflow-semantics/tests_metricflow_semantics/snapshots/test_spec_lookup.py/str/test_filter_spec_resolution__simple_metric__result.txt @@ -22,7 +22,25 @@ FilterSpecResolutionLookUp( ), ], ), - resolved_spec=TimeDimensionSpec(element_name='metric_time', time_granularity=MONTH), + resolved_linkable_element_set=LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name='metric_time', + element_type=TIME_DIMENSION, + time_granularity=MONTH, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference( + semantic_model_name='monthly_measures_source', + ), + element_name='metric_time', + dimension_type=TIME, + properties=frozenset({}), + time_granularity=MONTH, + ), + ), + }, + ), spec_pattern=TimeDimensionPattern( parameter_set=EntityLinkPatternParameterSet( fields_to_compare=(DATE_PART, ELEMENT_NAME, ENTITY_LINKS), From 4a19e0f0893b35c0e26889d48de3445a264ccd19 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 14:41:31 -0700 Subject: [PATCH 12/14] Add change log for #1155. --- .changes/unreleased/Features-20240426-144119.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20240426-144119.yaml diff --git a/.changes/unreleased/Features-20240426-144119.yaml b/.changes/unreleased/Features-20240426-144119.yaml new file mode 100644 index 0000000000..c3b6afa637 --- /dev/null +++ b/.changes/unreleased/Features-20240426-144119.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add a Dependency Resolver for Saved Queries +time: 2024-04-26T14:41:19.27946-07:00 +custom: + Author: plypaul + Issue: "1155" From bcbf1bf6b8368506e23b2f2a4242e236eb8a9823 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 16:41:32 -0700 Subject: [PATCH 13/14] Move `_path_key_to_spec` to `ElementPathKey`. --- .../model/semantics/linkable_element.py | 36 +++++++++++++ .../model/semantics/linkable_element_set.py | 54 ++++--------------- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py index 86dfb3669c..76b7a72c95 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py @@ -20,6 +20,13 @@ from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantic_model_derivation import SemanticModelDerivation +from metricflow_semantics.specs.spec_classes import ( + DimensionSpec, + EntitySpec, + GroupByMetricSpec, + LinkableInstanceSpec, + TimeDimensionSpec, +) logger = logging.getLogger(__name__) @@ -73,6 +80,35 @@ def __post_init__(self) -> None: else: assert_values_exhausted(element_type) + @property + def spec(self) -> LinkableInstanceSpec: + """The corresponding spec object for this path key.""" + if self.element_type is LinkableElementType.DIMENSION: + return DimensionSpec( + element_name=self.element_name, + entity_links=self.entity_links, + ) + elif self.element_type is LinkableElementType.TIME_DIMENSION: + assert self.time_granularity is not None + return TimeDimensionSpec( + element_name=self.element_name, + entity_links=self.entity_links, + time_granularity=self.time_granularity, + date_part=self.date_part, + ) + elif self.element_type is LinkableElementType.ENTITY: + return EntitySpec( + element_name=self.element_name, + entity_links=self.entity_links, + ) + elif self.element_type is LinkableElementType.METRIC: + return GroupByMetricSpec( + element_name=self.element_name, + entity_links=self.entity_links, + ) + else: + assert_values_exhausted(self.element_type) + @dataclass(frozen=True) class SemanticModelJoinPathElement: diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py index 4408d6f7a4..54b76fd147 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element_set.py @@ -4,7 +4,6 @@ from dataclasses import dataclass, field from typing import Dict, FrozenSet, List, Sequence, Set, Tuple -from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from dbt_semantic_interfaces.references import SemanticModelReference from typing_extensions import override @@ -19,12 +18,8 @@ ) from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_classes import ( - DimensionSpec, - EntitySpec, - GroupByMetricSpec, InstanceSpec, LinkableInstanceSpec, - TimeDimensionSpec, ) @@ -317,43 +312,14 @@ def spec_count(self) -> int: @property def specs(self) -> Sequence[LinkableInstanceSpec]: """Converts the items in a `LinkableElementSet` to their corresponding spec objects.""" - specs: List[LinkableInstanceSpec] = [] - - for path_key in ( - tuple(self.path_key_to_linkable_dimensions.keys()) - + tuple(self.path_key_to_linkable_entities.keys()) - + tuple(self.path_key_to_linkable_metrics.keys()) - ): - specs.append(self._path_key_to_spec(path_key)) - - return specs - - def _path_key_to_spec(self, path_key: ElementPathKey) -> LinkableInstanceSpec: - if path_key.element_type is LinkableElementType.DIMENSION: - return DimensionSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, + return tuple( + path_key.spec + for path_key in ( + tuple(self.path_key_to_linkable_dimensions.keys()) + + tuple(self.path_key_to_linkable_entities.keys()) + + tuple(self.path_key_to_linkable_metrics.keys()) ) - elif path_key.element_type is LinkableElementType.TIME_DIMENSION: - assert path_key.time_granularity is not None - return TimeDimensionSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - time_granularity=path_key.time_granularity, - date_part=path_key.date_part, - ) - elif path_key.element_type is LinkableElementType.ENTITY: - return EntitySpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - elif path_key.element_type is LinkableElementType.METRIC: - return GroupByMetricSpec( - element_name=path_key.element_name, - entity_links=path_key.entity_links, - ) - else: - assert_values_exhausted(path_key.element_type) + ) def filter_by_spec_patterns(self, spec_patterns: Sequence[SpecPattern]) -> LinkableElementSet: """Filter the elements in the set by the given spec patters. @@ -373,15 +339,15 @@ def filter_by_spec_patterns(self, spec_patterns: Sequence[SpecPattern]) -> Linka path_key_to_linkable_metrics: Dict[ElementPathKey, Tuple[LinkableMetric, ...]] = {} for path_key, linkable_dimensions in self.path_key_to_linkable_dimensions.items(): - if self._path_key_to_spec(path_key) in specs_to_include: + if path_key.spec in specs_to_include: path_key_to_linkable_dimensions[path_key] = linkable_dimensions for path_key, linkable_entities in self.path_key_to_linkable_entities.items(): - if self._path_key_to_spec(path_key) in specs_to_include: + if path_key.spec in specs_to_include: path_key_to_linkable_entities[path_key] = linkable_entities for path_key, linkable_metrics in self.path_key_to_linkable_metrics.items(): - if self._path_key_to_spec(path_key) in specs_to_include: + if path_key.spec in specs_to_include: path_key_to_linkable_metrics[path_key] = linkable_metrics return LinkableElementSet( From 07761a3035ace465a337585885923351d3734390 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Fri, 26 Apr 2024 17:36:00 -0700 Subject: [PATCH 14/14] Address comments. --- .../model/semantics/linkable_element.py | 8 +- .../query/query_parser.py | 3 +- .../semantics/test_linkable_element_set.py | 143 ++++++++++++++++++ 3 files changed, 150 insertions(+), 4 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py index 76b7a72c95..58a65f35a0 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_element.py @@ -89,7 +89,9 @@ def spec(self) -> LinkableInstanceSpec: entity_links=self.entity_links, ) elif self.element_type is LinkableElementType.TIME_DIMENSION: - assert self.time_granularity is not None + assert ( + self.time_granularity is not None + ), f"{self.time_granularity=} should not be None as per check in dataclass validation" return TimeDimensionSpec( element_name=self.element_name, entity_links=self.entity_links, @@ -170,7 +172,7 @@ def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: @dataclass(frozen=True) -class LinkableEntity(LinkableElement, SemanticModelDerivation): +class LinkableEntity(LinkableElement): """Describes how an entity can be realized by joining based on entity links.""" # The semantic model where this entity was defined. @@ -201,7 +203,7 @@ def derived_from_semantic_models(self) -> Sequence[SemanticModelReference]: @dataclass(frozen=True) -class LinkableMetric(LinkableElement, SemanticModelDerivation): +class LinkableMetric(LinkableElement): """Describes how a metric can be realized by joining based on entity links.""" element_name: str diff --git a/metricflow-semantics/metricflow_semantics/query/query_parser.py b/metricflow-semantics/metricflow_semantics/query/query_parser.py index 673e438661..873120f59e 100644 --- a/metricflow-semantics/metricflow_semantics/query/query_parser.py +++ b/metricflow-semantics/metricflow_semantics/query/query_parser.py @@ -20,6 +20,7 @@ from metricflow_semantics.filters.time_constraint import TimeRangeConstraint from metricflow_semantics.mf_logging.formatting import indent from metricflow_semantics.mf_logging.pretty_print import mf_pformat +from metricflow_semantics.mf_logging.runtime import log_runtime from metricflow_semantics.model.semantic_manifest_lookup import SemanticManifestLookup from metricflow_semantics.naming.dunder_scheme import DunderNamingScheme from metricflow_semantics.naming.metric_scheme import MetricNamingScheme @@ -340,7 +341,7 @@ def parse_and_validate_query( min_max_only=min_max_only, ).query_spec - # @log_runtime() + @log_runtime() def _parse_and_validate_query( self, metric_names: Optional[Sequence[str]], diff --git a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py index f352052e67..2dd2acafd8 100644 --- a/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py +++ b/metricflow-semantics/tests_metricflow_semantics/model/semantics/test_linkable_element_set.py @@ -11,6 +11,7 @@ import itertools +import pytest from dbt_semantic_interfaces.protocols.dimension import DimensionType from dbt_semantic_interfaces.references import ( DimensionReference, @@ -22,12 +23,20 @@ from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.model.semantics.linkable_element import ( + ElementPathKey, LinkableDimension, + LinkableElementType, LinkableEntity, LinkableMetric, SemanticModelJoinPathElement, ) from metricflow_semantics.model.semantics.linkable_element_set import LinkableElementSet +from metricflow_semantics.specs.patterns.entity_link_pattern import ( + EntityLinkPattern, + EntityLinkPatternParameterSet, + ParameterSetField, +) +from metricflow_semantics.specs.spec_classes import TimeDimensionSpec from more_itertools import bucket AMBIGUOUS_NAME = "ambiguous" @@ -475,3 +484,137 @@ def test_only_unique_path_keys() -> None: _base_entity.path_key: (_base_entity,) }, "Got unexpected value for unique entity sets!" assert unique_path_keys.path_key_to_linkable_metrics == dict(), "Found unexpected unique values for metric sets!" + + +@pytest.fixture(scope="session") +def linkable_set() -> LinkableElementSet: # noqa: D103 + entity_0 = EntityReference("entity_0") + entity_0_source = SemanticModelReference("entity_0_source") + entity_1 = EntityReference("entity_1") + entity_1_source = SemanticModelReference("entity_1_source") + entity_2 = EntityReference("entity_2") + entity_2_source = SemanticModelReference("entity_2_source") + entity_3 = EntityReference("entity_3") + entity_3_source = SemanticModelReference("entity_3_source") + + return LinkableElementSet( + path_key_to_linkable_dimensions={ + ElementPathKey( + element_name="dimension_element", + entity_links=(entity_0,), + element_type=LinkableElementType.DIMENSION, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("dimension_source"), + element_name="dimension_element", + dimension_type=DimensionType.CATEGORICAL, + entity_links=(entity_0,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_0_source, + join_on_entity=entity_0, + ), + ), + properties=frozenset(), + time_granularity=None, + date_part=None, + ), + ), + ElementPathKey( + element_name="time_dimension_element", + entity_links=(entity_1,), + element_type=LinkableElementType.TIME_DIMENSION, + time_granularity=TimeGranularity.DAY, + ): ( + LinkableDimension( + semantic_model_origin=SemanticModelReference("time_dimension_source"), + element_name="time_dimension_element", + dimension_type=DimensionType.TIME, + entity_links=(entity_1,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_1_source, + join_on_entity=entity_1, + ), + ), + properties=frozenset(), + time_granularity=TimeGranularity.DAY, + date_part=None, + ), + ), + }, + path_key_to_linkable_entities={ + ElementPathKey( + element_name="entity_element", + entity_links=(entity_2,), + element_type=LinkableElementType.ENTITY, + ): ( + LinkableEntity( + semantic_model_origin=SemanticModelReference("entity_source"), + element_name="entity_element", + entity_links=(entity_2,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_2_source, + join_on_entity=entity_2, + ), + ), + properties=frozenset(), + ), + ) + }, + path_key_to_linkable_metrics={ + ElementPathKey( + element_name="metric_element", + entity_links=(entity_3,), + element_type=LinkableElementType.METRIC, + ): ( + LinkableMetric( + join_by_semantic_model=SemanticModelReference("metric_source"), + element_name="metric_element", + entity_links=(entity_3,), + join_path=( + SemanticModelJoinPathElement( + semantic_model_reference=entity_3_source, + join_on_entity=entity_3, + ), + ), + properties=frozenset(), + ), + ) + }, + ) + + +def test_derived_semantic_models(linkable_set: LinkableElementSet) -> None: + """Tests that the semantic models in the element set are returned via `derived_from_semantic_models`.""" + assert tuple(linkable_set.derived_from_semantic_models) == ( + SemanticModelReference(semantic_model_name="dimension_source"), + SemanticModelReference(semantic_model_name="entity_0_source"), + SemanticModelReference(semantic_model_name="entity_1_source"), + SemanticModelReference(semantic_model_name="entity_2_source"), + SemanticModelReference(semantic_model_name="entity_3_source"), + SemanticModelReference(semantic_model_name="entity_source"), + SemanticModelReference(semantic_model_name="metric_source"), + SemanticModelReference(semantic_model_name="time_dimension_source"), + ) + + +def test_filter_by_pattern(linkable_set: LinkableElementSet) -> None: + """Tests that the specs produced by the set are properly filtered by spec patterns.""" + spec_pattern = EntityLinkPattern( + EntityLinkPatternParameterSet( + fields_to_compare=(ParameterSetField.ENTITY_LINKS,), + element_name=None, + entity_links=(EntityReference("entity_1"),), + ) + ) + + assert tuple(linkable_set.filter_by_spec_patterns((spec_pattern,)).specs) == ( + TimeDimensionSpec( + element_name="time_dimension_element", + entity_links=(EntityReference("entity_1"),), + time_granularity=TimeGranularity.DAY, + date_part=None, + ), + )