Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fill nulls for multi-metric queries #850

Merged
merged 9 commits into from
Nov 8, 2023
22 changes: 20 additions & 2 deletions metricflow/model/semantics/metric_lookup.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
from __future__ import annotations

import logging
from typing import Dict, FrozenSet, List, Sequence
from typing import Dict, FrozenSet, List, Optional, Sequence

from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
from dbt_semantic_interfaces.implementations.filters.where_filter import PydanticWhereFilterIntersection
from dbt_semantic_interfaces.implementations.metric import PydanticMetricTimeWindow
from dbt_semantic_interfaces.protocols import WhereFilter
from dbt_semantic_interfaces.protocols.metric import Metric, MetricType
from dbt_semantic_interfaces.protocols.metric import Metric, MetricInputMeasure, MetricType
from dbt_semantic_interfaces.protocols.semantic_manifest import SemanticManifest
from dbt_semantic_interfaces.references import MetricReference

Expand Down Expand Up @@ -105,6 +106,23 @@ def add_metric(self, metric: Metric) -> None:
)
self._metrics[metric_reference] = metric

def yaml_input_measure_for_metric(self, metric_reference: MetricReference) -> Optional[MetricInputMeasure]:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice! Super nitty but can we rename this away from yaml? YAML makes me grumpy. More importantly, there's no guarantee that the input is defined in YAML (although in practice that is how it works today).

Maybe something like direct_input_measure_for_metric or configured_input_measure_for_metric?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated!

"""Get input measure defined in the metric YAML, if exists.

When SemanticModel is constructed, input measures from input metrics are added to the list of input measures
for a metric. Here, use rules about metric types to determine which input measures were defined in the YAML:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then here we can replace YAML with config or something.

- Simple & cumulative metrics require one input measure, and can't take any input metrics.
- Derived & ratio metrics take no input measures, only input metrics.
"""
metric = self.get_metric(metric_reference=metric_reference)
if metric.type is MetricType.CUMULATIVE or metric.type is MetricType.SIMPLE:
assert len(metric.input_measures) == 1, "Simple and cumulative metrics should have one input measure."
return metric.input_measures[0]
elif metric.type is MetricType.RATIO or metric.type is MetricType.DERIVED:
return None
else:
assert_values_exhausted(metric.type)

def measures_for_metric(
self,
metric_reference: MetricReference,
Expand Down
28 changes: 10 additions & 18 deletions metricflow/plan_conversion/dataflow_to_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -897,24 +897,16 @@ def _make_select_columns_for_multiple_metrics(

# At this point, the MetricSpec might have the alias in place of the element name, so we need to look
# back at where it was defined from to get the metric element name.
metric_name = metric_instance.defined_from.metric_name
input_measures = self._metric_lookup.measures_for_metric(
metric_reference=MetricReference(metric_name),
column_association_resolver=self._column_association_resolver,
)
# If multiple input measures, this is a query with a nested derived/ratio metric. In that case, we can
# skip this step because it has already occurred when rendering the nested metric.
# TODO: this logic might need updating for conversion metrics, which will allow multiple input measures
if len(input_measures) == 1:
input_measure = input_measures[0]
if input_measure.fill_nulls_with is not None:
select_expression = SqlAggregateFunctionExpression(
sql_function=SqlFunction.COALESCE,
sql_function_args=[
select_expression,
SqlStringExpression(str(input_measure.fill_nulls_with)),
],
)
metric_reference = MetricReference(element_name=metric_instance.defined_from.metric_name)
input_measure = self._metric_lookup.yaml_input_measure_for_metric(metric_reference=metric_reference)
if input_measure and input_measure.fill_nulls_with is not None:
select_expression = SqlAggregateFunctionExpression(
sql_function=SqlFunction.COALESCE,
sql_function_args=[
select_expression,
SqlStringExpression(str(input_measure.fill_nulls_with)),
],
)

select_columns.append(
SqlSelectColumn(
Expand Down