Skip to content

Commit

Permalink
Remove metrics that must be queried with metric_time from linkable me…
Browse files Browse the repository at this point in the history 
…tric options
  • Loading branch information
@courtneyholcomb
courtneyholcomb committed Apr 25, 2024
1 parent 9bc8ebb commit a0bee88
Show file tree
Hide file tree
Showing 5 changed files with 612 additions and 740 deletions.
32 changes: 31 additions & 1 deletion metricflow/model/semantics/linkable_spec_resolver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
from dbt_semantic_interfaces.protocols.dimension import Dimension, DimensionType
from dbt_semantic_interfaces.protocols.metric import Metric
from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
from dbt_semantic_interfaces.protocols.semantic_model import SemanticModel
from dbt_semantic_interfaces.references import (
Expand Down Expand Up @@ -565,12 +566,14 @@ def __init__(
self._entity_to_semantic_model[entity.reference.element_name].append(semantic_model)

self._metric_to_linkable_element_sets: Dict[str, List[LinkableElementSet]] = {}
self._metric_references_to_metrics: Dict[MetricReference, Metric] = {}
self._joinable_metrics_for_entities: Dict[EntityReference, Set[MetricSubqueryJoinPathElement]] = defaultdict(
set
)

start_time = time.time()
for metric in self._semantic_manifest.metrics:
self._metric_references_to_metrics[MetricReference(metric.name)] = metric
linkable_sets_for_measure = []
for measure in metric.measure_references:
# Cumulative metrics currently can't be queried by other time granularities.
Expand Down Expand Up @@ -608,8 +611,13 @@ def __init__(

self._metric_to_linkable_element_sets[metric.name] = linkable_sets_for_measure

# This loop must happen after the one above so that _metric_to_linkable_element_sets is populated.
# Populate storage dicts with linkable metrics. This loop must happen after the one above so that
# _metric_to_linkable_element_sets is populated with entities and dimensions.
for metric in self._semantic_manifest.metrics:
# Cumulative metrics and time offset metrics require grouping by metric_time, which is not yet available for
# linkable metrics. So skip those.
if self._metric_requires_metric_time(metric):
continue
metric_reference = MetricReference(metric.name)
linkable_element_set_for_metric = self.get_linkable_elements_for_metrics([metric_reference])
for linkable_entities in linkable_element_set_for_metric.path_key_to_linkable_entities.values():
Expand All @@ -622,6 +630,7 @@ def __init__(
self._joinable_metrics_for_entities[linkable_entity.reference].add(
metric_subquery_join_path_element
)
# TODO: update _metric_to_linkable_element_sets to have linkable metrics

# If no metrics are specified, the query interface supports querying distinct values for dimensions, entities,
# and group by metrics.
Expand All @@ -639,6 +648,27 @@ def __init__(

logger.info(f"Building valid group-by-item indexes took: {time.time() - start_time:.2f}s")

def _metric_requires_metric_time(self, metric: Metric) -> bool:
"""Checks if the metric can only be queried with metric_time. Also checks input metrics.
True if the metric uses cumulative time component or a time offset.
"""
metrics_to_check = [metric]
while metrics_to_check:
metric_to_check = metrics_to_check.pop()
if metric_to_check.type_params.window is not None or metric_to_check.type_params.grain_to_date is not None:
return True
for input_metric in metric_to_check.input_metrics:
if input_metric.offset_window is not None or input_metric.offset_to_grain is not None:
return True
metric_for_input_metric = self._metric_references_to_metrics.get(MetricReference(input_metric.name))
assert (
metric_for_input_metric
), f"Did not find input metric {input_metric.name} in registered metrics. This indicates internal misconfiguration."
metrics_to_check.append(metric_for_input_metric)

return False

def _get_semantic_model_for_measure(self, measure_reference: MeasureReference) -> SemanticModel:
semantic_models_where_measure_was_found = []
for semantic_model in self._semantic_models:
Expand Down
62 changes: 0 additions & 62 deletions ...shots/test_linkable_spec_resolver.py/list/test_linkable_element_set_as_spec_set__set0.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
'listing__average_instant_booking_value',
'listing__bookers',
'listing__booking_fees',
'listing__booking_fees_last_week_per_booker_this_week',
'listing__booking_fees_per_booker',
'listing__booking_fees_since_start_of_month',
'listing__booking_payments',
'listing__booking_value',
'listing__booking_value_for_non_null_listing_id',
Expand All @@ -18,18 +16,9 @@
'listing__booking_value_sub_instant',
'listing__booking_value_sub_instant_add_10',
'listing__bookings',
'listing__bookings_5_day_lag',
'listing__bookings_at_start_of_month',
'listing__bookings_fill_nulls_with_0',
'listing__bookings_fill_nulls_with_0_without_time_spine',
'listing__bookings_growth_2_weeks',
'listing__bookings_growth_2_weeks_fill_nulls_with_0',
'listing__bookings_growth_2_weeks_fill_nulls_with_0_for_non_offset',
'listing__bookings_growth_since_start_of_month',
'listing__bookings_join_to_time_spine',
'listing__bookings_month_start_compared_to_1_month_prior',
'listing__bookings_offset_once',
'listing__bookings_offset_twice',
'listing__bookings_per_booker',
'listing__bookings_per_dollar',
'listing__bookings_per_listing',
Expand Down Expand Up @@ -81,9 +70,6 @@
'listing__ds__quarter',
'listing__ds__week',
'listing__ds__year',
'listing__every_2_days_bookers_2_days_ago',
'listing__every_two_days_bookers',
'listing__every_two_days_bookers_fill_nulls_with_0',
'listing__instant_booking_fraction_of_max_value',
'listing__instant_booking_value',
'listing__instant_booking_value_ratio',
Expand All @@ -103,9 +89,7 @@
'listing__lux_listing__average_instant_booking_value',
'listing__lux_listing__bookers',
'listing__lux_listing__booking_fees',
'listing__lux_listing__booking_fees_last_week_per_booker_this_week',
'listing__lux_listing__booking_fees_per_booker',
'listing__lux_listing__booking_fees_since_start_of_month',
'listing__lux_listing__booking_payments',
'listing__lux_listing__booking_value',
'listing__lux_listing__booking_value_for_non_null_listing_id',
Expand All @@ -114,18 +98,9 @@
'listing__lux_listing__booking_value_sub_instant',
'listing__lux_listing__booking_value_sub_instant_add_10',
'listing__lux_listing__bookings',
'listing__lux_listing__bookings_5_day_lag',
'listing__lux_listing__bookings_at_start_of_month',
'listing__lux_listing__bookings_fill_nulls_with_0',
'listing__lux_listing__bookings_fill_nulls_with_0_without_time_spine',
'listing__lux_listing__bookings_growth_2_weeks',
'listing__lux_listing__bookings_growth_2_weeks_fill_nulls_with_0',
'listing__lux_listing__bookings_growth_2_weeks_fill_nulls_with_0_for_non_offset',
'listing__lux_listing__bookings_growth_since_start_of_month',
'listing__lux_listing__bookings_join_to_time_spine',
'listing__lux_listing__bookings_month_start_compared_to_1_month_prior',
'listing__lux_listing__bookings_offset_once',
'listing__lux_listing__bookings_offset_twice',
'listing__lux_listing__bookings_per_booker',
'listing__lux_listing__bookings_per_dollar',
'listing__lux_listing__bookings_per_listing',
Expand All @@ -135,9 +110,6 @@
'listing__lux_listing__derived_bookings_1',
'listing__lux_listing__discrete_booking_value_p99',
'listing__lux_listing__double_counted_delayed_bookings',
'listing__lux_listing__every_2_days_bookers_2_days_ago',
'listing__lux_listing__every_two_days_bookers',
'listing__lux_listing__every_two_days_bookers_fill_nulls_with_0',
'listing__lux_listing__instant_booking_fraction_of_max_value',
'listing__lux_listing__instant_booking_value',
'listing__lux_listing__instant_booking_value_ratio',
Expand Down Expand Up @@ -199,9 +171,7 @@
'user__average_instant_booking_value',
'user__bookers',
'user__booking_fees',
'user__booking_fees_last_week_per_booker_this_week',
'user__booking_fees_per_booker',
'user__booking_fees_since_start_of_month',
'user__booking_payments',
'user__booking_value',
'user__booking_value_for_non_null_listing_id',
Expand All @@ -210,18 +180,9 @@
'user__booking_value_sub_instant',
'user__booking_value_sub_instant_add_10',
'user__bookings',
'user__bookings_5_day_lag',
'user__bookings_at_start_of_month',
'user__bookings_fill_nulls_with_0',
'user__bookings_fill_nulls_with_0_without_time_spine',
'user__bookings_growth_2_weeks',
'user__bookings_growth_2_weeks_fill_nulls_with_0',
'user__bookings_growth_2_weeks_fill_nulls_with_0_for_non_offset',
'user__bookings_growth_since_start_of_month',
'user__bookings_join_to_time_spine',
'user__bookings_month_start_compared_to_1_month_prior',
'user__bookings_offset_once',
'user__bookings_offset_twice',
'user__bookings_per_booker',
'user__bookings_per_dollar',
'user__bookings_per_listing',
Expand All @@ -235,9 +196,7 @@
'user__company__average_instant_booking_value',
'user__company__bookers',
'user__company__booking_fees',
'user__company__booking_fees_last_week_per_booker_this_week',
'user__company__booking_fees_per_booker',
'user__company__booking_fees_since_start_of_month',
'user__company__booking_payments',
'user__company__booking_value',
'user__company__booking_value_for_non_null_listing_id',
Expand All @@ -246,18 +205,9 @@
'user__company__booking_value_sub_instant',
'user__company__booking_value_sub_instant_add_10',
'user__company__bookings',
'user__company__bookings_5_day_lag',
'user__company__bookings_at_start_of_month',
'user__company__bookings_fill_nulls_with_0',
'user__company__bookings_fill_nulls_with_0_without_time_spine',
'user__company__bookings_growth_2_weeks',
'user__company__bookings_growth_2_weeks_fill_nulls_with_0',
'user__company__bookings_growth_2_weeks_fill_nulls_with_0_for_non_offset',
'user__company__bookings_growth_since_start_of_month',
'user__company__bookings_join_to_time_spine',
'user__company__bookings_month_start_compared_to_1_month_prior',
'user__company__bookings_offset_once',
'user__company__bookings_offset_twice',
'user__company__bookings_per_booker',
'user__company__bookings_per_dollar',
'user__company__bookings_per_view',
Expand All @@ -266,9 +216,6 @@
'user__company__derived_bookings_1',
'user__company__discrete_booking_value_p99',
'user__company__double_counted_delayed_bookings',
'user__company__every_2_days_bookers_2_days_ago',
'user__company__every_two_days_bookers',
'user__company__every_two_days_bookers_fill_nulls_with_0',
'user__company__identity_verifications',
'user__company__instant_booking_fraction_of_max_value',
'user__company__instant_booking_value',
Expand All @@ -290,11 +237,8 @@
'user__company__regional_starting_balance_ratios',
'user__company__revenue',
'user__company__revenue_all_time',
'user__company__revenue_mtd',
'user__company__smallest_listing',
'user__company__total_account_balance_first_day',
'user__company__trailing_2_months_revenue',
'user__company__trailing_2_months_revenue_sub_10',
'user__company__twice_bookings_fill_nulls_with_0_without_time_spine',
'user__company__views',
'user__company__views_times_booking_value',
Expand Down Expand Up @@ -389,9 +333,6 @@
'user__ds_partitioned__quarter',
'user__ds_partitioned__week',
'user__ds_partitioned__year',
'user__every_2_days_bookers_2_days_ago',
'user__every_two_days_bookers',
'user__every_two_days_bookers_fill_nulls_with_0',
'user__home_state',
'user__home_state_latest',
'user__identity_verifications',
Expand All @@ -415,11 +356,8 @@
'user__regional_starting_balance_ratios',
'user__revenue',
'user__revenue_all_time',
'user__revenue_mtd',
'user__smallest_listing',
'user__total_account_balance_first_day',
'user__trailing_2_months_revenue',
'user__trailing_2_months_revenue_sub_10',
'user__twice_bookings_fill_nulls_with_0_without_time_spine',
'user__views',
'user__views_times_booking_value',
Expand Down
Loading

0 comments on commit a0bee88

Please sign in to comment.