From 850878b4c4400fd9548fce15ca36e670e5ddac56 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Tue, 24 Sep 2024 09:21:41 -0700 Subject: [PATCH] /* PR_START p--short-term-perf 04 */ Allow spec patterns to filter based on linkable element properties. --- .../model/semantics/linkable_spec_resolver.py | 14 ++++++++++---- .../candidate_push_down/push_down_visitor.py | 11 +++++++++-- .../query/suggestion_generator.py | 5 ++++- .../specs/patterns/entity_link_pattern.py | 13 ++++++++++++- .../specs/patterns/no_group_by_metric.py | 8 +++++++- .../specs/patterns/spec_pattern.py | 9 ++++++++- .../specs/patterns/typed_patterns.py | 18 +++++++++++++++++- 7 files changed, 67 insertions(+), 11 deletions(-) diff --git a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py index 40d1d9bdbb..2510546bdb 100644 --- a/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py +++ b/metricflow-semantics/metricflow_semantics/model/semantics/linkable_spec_resolver.py @@ -605,10 +605,16 @@ def _get_linkable_element_set_for_measure( measure_semantic_model = self._get_semantic_model_for_measure(measure_reference) elements_in_semantic_model = self._get_elements_in_semantic_model(measure_semantic_model) - metrics_linked_to_semantic_model = self.get_joinable_metrics_for_semantic_model( - semantic_model=measure_semantic_model, - using_join_path=SemanticModelJoinPath(left_semantic_model_reference=measure_semantic_model.reference), - ) + + # Filter out group-by metrics if not specified by the property as there can be a large number of them. + if LinkableElementProperty.METRIC not in without_any_of: + metrics_linked_to_semantic_model = self.get_joinable_metrics_for_semantic_model( + semantic_model=measure_semantic_model, + using_join_path=SemanticModelJoinPath(left_semantic_model_reference=measure_semantic_model.reference), + ) + else: + metrics_linked_to_semantic_model = LinkableElementSet() + metric_time_elements = self._get_metric_time_elements(measure_reference) joined_elements = self._get_joined_elements(measure_semantic_model) diff --git a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py index 1548f8343a..155383511d 100644 --- a/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py +++ b/metricflow-semantics/metricflow_semantics/query/group_by_item/candidate_push_down/push_down_visitor.py @@ -3,7 +3,7 @@ import logging from contextlib import contextmanager from dataclasses import dataclass -from typing import Dict, FrozenSet, Iterator, List, Optional, Sequence, Tuple +from typing import Dict, FrozenSet, Iterator, List, Optional, Sequence, Set, Tuple from dbt_semantic_interfaces.enum_extension import assert_values_exhausted from dbt_semantic_interfaces.references import MetricReference @@ -170,10 +170,17 @@ def visit_measure_node(self, node: MeasureGroupByItemSourceNode) -> PushDownResu """Push the group-by-item specs that are available to the measure and match the source patterns to the child.""" with self._path_from_start_node_tracker.track_node_visit(node) as current_traversal_path: logger.debug(LazyFormat(lambda: f"Handling {node.ui_description}")) + + without_any_property: Set[LinkableElementProperty] = set() + if self._without_any_property is not None: + without_any_property.update(self._without_any_property) + for spec_pattern in self._source_spec_patterns: + without_any_property.update(spec_pattern.without_linkable_element_properties) + items_available_for_measure = self._semantic_manifest_lookup.metric_lookup.linkable_elements_for_measure( measure_reference=node.measure_reference, with_any_of=self._with_any_property, - without_any_of=self._without_any_property, + without_any_of=frozenset(without_any_property), ) # The following is needed to handle limitation of cumulative metrics. Filtering could be done at the measure diff --git a/metricflow-semantics/metricflow_semantics/query/suggestion_generator.py b/metricflow-semantics/metricflow_semantics/query/suggestion_generator.py index 1ebea20101..02dcb4b54a 100644 --- a/metricflow-semantics/metricflow_semantics/query/suggestion_generator.py +++ b/metricflow-semantics/metricflow_semantics/query/suggestion_generator.py @@ -32,7 +32,10 @@ class QueryItemSuggestionGenerator: ) def __init__( # noqa: D107 - self, input_naming_scheme: QueryItemNamingScheme, input_str: str, candidate_filters: Sequence[SpecPattern] + self, + input_naming_scheme: QueryItemNamingScheme, + input_str: str, + candidate_filters: Sequence[SpecPattern], ) -> None: self._input_naming_scheme = input_naming_scheme self._input_str = input_str diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py index 1f7b793148..8289dc32dd 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/entity_link_pattern.py @@ -3,13 +3,14 @@ import logging from dataclasses import dataclass from enum import Enum -from typing import Any, List, Optional, Sequence, Tuple +from typing import Any, FrozenSet, List, Optional, Sequence, Tuple from dbt_semantic_interfaces.references import EntityReference from dbt_semantic_interfaces.type_enums.date_part import DatePart from more_itertools import is_sorted from typing_extensions import override +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.specs.instance_spec import InstanceSpec, LinkableInstanceSpec from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_set import group_specs_by_type @@ -133,3 +134,13 @@ def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableIns matching_specs.append(spec) return matching_specs + + @property + @override + def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: + if ( + self.parameter_set.metric_subquery_entity_links is None + or len(self.parameter_set.metric_subquery_entity_links) == 0 + ): + return frozenset({LinkableElementProperty.METRIC}) + return frozenset() diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py b/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py index 714aceb3f6..175447cdd7 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/no_group_by_metric.py @@ -1,10 +1,11 @@ from __future__ import annotations from dataclasses import dataclass -from typing import List, Sequence +from typing import FrozenSet, List, Sequence from typing_extensions import override +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.specs.instance_spec import InstanceSpec, LinkableInstanceSpec from metricflow_semantics.specs.patterns.spec_pattern import SpecPattern from metricflow_semantics.specs.spec_set import group_specs_by_type @@ -26,3 +27,8 @@ def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[LinkableIns specs_to_return.extend(spec_set.entity_specs) return specs_to_return + + @property + @override + def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: + return frozenset({LinkableElementProperty.METRIC}) diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py b/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py index 7ad28d532b..caaad9d5d9 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/spec_pattern.py @@ -1,7 +1,9 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Sequence +from typing import TYPE_CHECKING, FrozenSet, Sequence + +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty if TYPE_CHECKING: from metricflow_semantics.specs.instance_spec import InstanceSpec @@ -21,3 +23,8 @@ def match(self, candidate_specs: Sequence[InstanceSpec]) -> Sequence[InstanceSpe def matches_any(self, candidate_specs: Sequence[InstanceSpec]) -> bool: """Returns true if this spec matches any of the given specs.""" return len(self.match(candidate_specs)) > 0 + + @property + def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: + """Returns the set of properties of linkable elements that this won't match.""" + return frozenset() diff --git a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py index 8ddce95d32..5b4f88d9e3 100644 --- a/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py +++ b/metricflow-semantics/metricflow_semantics/specs/patterns/typed_patterns.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import List, Sequence +from typing import FrozenSet, List, Sequence from dbt_semantic_interfaces.call_parameter_sets import ( DimensionCallParameterSet, @@ -12,6 +12,7 @@ from dbt_semantic_interfaces.references import EntityReference from typing_extensions import override +from metricflow_semantics.model.linkable_element_property import LinkableElementProperty from metricflow_semantics.naming.linkable_spec_name import StructuredLinkableSpecName from metricflow_semantics.specs.instance_spec import InstanceSpec, LinkableInstanceSpec from metricflow_semantics.specs.patterns.entity_link_pattern import ( @@ -51,6 +52,11 @@ def from_call_parameter_set( # noqa: D102 ) ) + @property + @override + def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: + return frozenset({LinkableElementProperty.METRIC}) + @dataclass(frozen=True) class TimeDimensionPattern(EntityLinkPattern): @@ -96,6 +102,11 @@ def from_call_parameter_set( ) ) + @property + @override + def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: + return frozenset({LinkableElementProperty.METRIC}) + @dataclass(frozen=True) class EntityPattern(EntityLinkPattern): @@ -122,6 +133,11 @@ def from_call_parameter_set(entity_call_parameter_set: EntityCallParameterSet) - ) ) + @property + @override + def without_linkable_element_properties(self) -> FrozenSet[LinkableElementProperty]: + return frozenset({LinkableElementProperty.METRIC}) + @dataclass(frozen=True) class GroupByMetricPattern(EntityLinkPattern):