Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@courtneyholcomb
courtneyholcomb committed Oct 24, 2024
1 parent dd8df12 commit 57228ab
Show file tree
Hide file tree
Showing 7 changed files with 370 additions and 347 deletions.
14 changes: 7 additions & 7 deletions metricflow/dataflow/builder/dataflow_plan_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ def _build_aggregated_conversion_node(
filtered_unaggregated_base_node = FilterElementsNode.create(
parent_node=unaggregated_base_measure_node,
include_specs=group_specs_by_type(required_local_specs)
.merge(base_measure_recipe.all_linkable_specs_required_for_source_nodes.as_instance_spec_set)
.merge(base_required_linkable_specs.replace_custom_granularity_with_base_granularity().as_instance_spec_set)
.dedupe(),
)

Expand Down Expand Up @@ -1703,12 +1703,12 @@ def _build_aggregated_measure_from_measure_source_node(
non_additive_dimension_grain = self._semantic_model_lookup.get_defined_time_granularity(
TimeDimensionReference(non_additive_dimension_spec.name)
)
queried_time_dimension_spec: Optional[TimeDimensionSpec] = (
self._find_non_additive_dimension_in_linkable_specs(
agg_time_dimension=agg_time_dimension,
linkable_specs=queried_linkable_specs.as_tuple,
non_additive_dimension_spec=non_additive_dimension_spec,
)
queried_time_dimension_spec: Optional[
TimeDimensionSpec
] = self._find_non_additive_dimension_in_linkable_specs(
agg_time_dimension=agg_time_dimension,
linkable_specs=queried_linkable_specs.as_tuple,
non_additive_dimension_spec=non_additive_dimension_spec,
)
time_dimension_spec = TimeDimensionSpec(
# The NonAdditiveDimensionSpec name property is a plain element name
Expand Down
562 changes: 284 additions & 278 deletions ...nularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql
View file

Large diffs are not rendered by default.

95 changes: 48 additions & 47 deletions ...y/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,77 +5,78 @@ SELECT
FROM (
-- Combine Aggregated Outputs
SELECT
COALESCE(subq_24.metric_time__martian_day, subq_36.metric_time__martian_day) AS metric_time__martian_day
, MAX(subq_24.visits) AS visits
, MAX(subq_36.buys) AS buys
COALESCE(subq_25.metric_time__martian_day, subq_38.metric_time__martian_day) AS metric_time__martian_day
, MAX(subq_25.visits) AS visits
, MAX(subq_38.buys) AS buys
FROM (
-- Pass Only Elements: ['visits', 'metric_time__day']
-- Join to Custom Granularity Dataset
-- Pass Only Elements: ['visits', 'metric_time__martian_day']
-- Aggregate Measures
SELECT
subq_21.martian_day AS metric_time__martian_day
, SUM(subq_20.visits) AS visits
subq_22.martian_day AS metric_time__martian_day
, SUM(subq_21.visits) AS visits
FROM (
-- Read Elements From Semantic Model 'visits_source'
-- Metric Time Dimension 'ds'
SELECT
DATE_TRUNC('day', ds) AS metric_time__day
, 1 AS visits
FROM ***************************.fct_visits visits_source_src_28000
) subq_20
) subq_21
LEFT OUTER JOIN
***************************.mf_time_spine subq_21
***************************.mf_time_spine subq_22
ON
subq_20.metric_time__day = subq_21.ds
subq_21.metric_time__day = subq_22.ds
GROUP BY
subq_21.martian_day
) subq_24
subq_22.martian_day
) subq_25
FULL OUTER JOIN (
-- Pass Only Elements: ['buys', 'metric_time__day']
-- Join to Custom Granularity Dataset
-- Pass Only Elements: ['buys', 'metric_time__martian_day']
-- Aggregate Measures
-- Pass Only Elements: ['buys', 'metric_time__martian_day']
SELECT
subq_33.martian_day AS metric_time__martian_day
, SUM(subq_31.buys) AS buys
subq_34.martian_day AS metric_time__martian_day
, SUM(subq_32.buys) AS buys
FROM (
-- Dedupe the fanout with mf_internal_uuid in the conversion data set
SELECT DISTINCT
FIRST_VALUE(subq_27.visits) OVER (
FIRST_VALUE(subq_28.visits) OVER (
PARTITION BY
subq_30.user
, subq_30.ds__day
, subq_30.mf_internal_uuid
ORDER BY subq_27.ds__day DESC
subq_31.user
, subq_31.ds__day
, subq_31.mf_internal_uuid
ORDER BY subq_28.ds__day DESC
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS visits
, FIRST_VALUE(subq_27.ds__day) OVER (
, FIRST_VALUE(subq_28.ds__day) OVER (
PARTITION BY
subq_30.user
, subq_30.ds__day
, subq_30.mf_internal_uuid
ORDER BY subq_27.ds__day DESC
subq_31.user
, subq_31.ds__day
, subq_31.mf_internal_uuid
ORDER BY subq_28.ds__day DESC
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS ds__day
, FIRST_VALUE(subq_27.metric_time__day) OVER (
, FIRST_VALUE(subq_28.metric_time__day) OVER (
PARTITION BY
subq_30.user
, subq_30.ds__day
, subq_30.mf_internal_uuid
ORDER BY subq_27.ds__day DESC
subq_31.user
, subq_31.ds__day
, subq_31.mf_internal_uuid
ORDER BY subq_28.ds__day DESC
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS metric_time__day
, FIRST_VALUE(subq_27.user) OVER (
, FIRST_VALUE(subq_28.user) OVER (
PARTITION BY
subq_30.user
, subq_30.ds__day
, subq_30.mf_internal_uuid
ORDER BY subq_27.ds__day DESC
subq_31.user
, subq_31.ds__day
, subq_31.mf_internal_uuid
ORDER BY subq_28.ds__day DESC
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS user
, subq_30.mf_internal_uuid AS mf_internal_uuid
, subq_30.buys AS buys
, subq_31.mf_internal_uuid AS mf_internal_uuid
, subq_31.buys AS buys
FROM (
-- Read Elements From Semantic Model 'visits_source'
-- Metric Time Dimension 'ds'
Expand All @@ -86,7 +87,7 @@ FROM (
, user_id AS user
, 1 AS visits
FROM ***************************.fct_visits visits_source_src_28000
) subq_27
) subq_28
INNER JOIN (
-- Read Elements From Semantic Model 'buys_source'
-- Metric Time Dimension 'ds'
Expand All @@ -97,27 +98,27 @@ FROM (
, 1 AS buys
, GEN_RANDOM_UUID() AS mf_internal_uuid
FROM ***************************.fct_buys buys_source_src_28000
) subq_30
) subq_31
ON
(
subq_27.user = subq_30.user
subq_28.user = subq_31.user
) AND (
(
subq_27.ds__day <= subq_30.ds__day
subq_28.ds__day <= subq_31.ds__day
) AND (
subq_27.ds__day > subq_30.ds__day - INTERVAL 7 day
subq_28.ds__day > subq_31.ds__day - INTERVAL 7 day
)
)
) subq_31
) subq_32
LEFT OUTER JOIN
***************************.mf_time_spine subq_33
***************************.mf_time_spine subq_34
ON
subq_31.metric_time__day = subq_33.ds
subq_32.metric_time__day = subq_34.ds
GROUP BY
subq_33.martian_day
) subq_36
subq_34.martian_day
) subq_38
ON
subq_24.metric_time__martian_day = subq_36.metric_time__martian_day
subq_25.metric_time__martian_day = subq_38.metric_time__martian_day
GROUP BY
COALESCE(subq_24.metric_time__martian_day, subq_36.metric_time__martian_day)
) subq_37
COALESCE(subq_25.metric_time__martian_day, subq_38.metric_time__martian_day)
) subq_39
10 changes: 8 additions & 2 deletions ...e_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfp_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,15 +216,21 @@
<!-- description = 'Aggregate Measures' -->
<!-- node_id = NodeId(id_str='am_1') -->
<FilterElementsNode>
<!-- description = -->
<!-- "Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day']" -->
<!-- description = -->
<!-- ("Pass Only Elements: ['buys', 'user__home_state_latest', 'visit__referrer_id', " -->
<!-- "'metric_time__day']") -->
<!-- node_id = NodeId(id_str='pfe_9') -->
<!-- include_spec = MeasureSpec(element_name='buys') -->
<!-- include_spec = -->
<!-- DimensionSpec( -->
<!-- element_name='home_state_latest', -->
<!-- entity_links=(EntityReference(element_name='user'),), -->
<!-- ) -->
<!-- include_spec = -->
<!-- DimensionSpec( -->
<!-- element_name='referrer_id', -->
<!-- entity_links=(EntityReference(element_name='visit'),), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
Expand Down
10 changes: 8 additions & 2 deletions ..._pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfpo_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,15 +213,21 @@
<!-- description = 'Aggregate Measures' -->
<!-- node_id = NodeId(id_str='am_3') -->
<FilterElementsNode>
<!-- description = -->
<!-- "Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day']" -->
<!-- description = -->
<!-- ("Pass Only Elements: ['buys', 'user__home_state_latest', 'visit__referrer_id', " -->
<!-- "'metric_time__day']") -->
<!-- node_id = NodeId(id_str='pfe_19') -->
<!-- include_spec = MeasureSpec(element_name='buys') -->
<!-- include_spec = -->
<!-- DimensionSpec( -->
<!-- element_name='home_state_latest', -->
<!-- entity_links=(EntityReference(element_name='user'),), -->
<!-- ) -->
<!-- include_spec = -->
<!-- DimensionSpec( -->
<!-- element_name='referrer_id', -->
<!-- entity_links=(EntityReference(element_name='visit'),), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
Expand Down
5 changes: 4 additions & 1 deletion ...pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,12 +189,14 @@ FROM (
-- Aggregate Measures
SELECT
subq_26.metric_time__day
, subq_26.visit__referrer_id
, subq_26.user__home_state_latest
, SUM(subq_26.buys) AS buys
FROM (
-- Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day']
-- Pass Only Elements: ['buys', 'user__home_state_latest', 'visit__referrer_id', 'metric_time__day']
SELECT
subq_25.metric_time__day
, subq_25.visit__referrer_id
, subq_25.user__home_state_latest
, subq_25.buys
FROM (
Expand Down Expand Up @@ -671,6 +673,7 @@ FROM (
) subq_26
GROUP BY
subq_26.metric_time__day
, subq_26.visit__referrer_id
, subq_26.user__home_state_latest
) subq_27
) subq_28
Expand Down
21 changes: 11 additions & 10 deletions ...endering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ SELECT
FROM (
-- Combine Aggregated Outputs
SELECT
COALESCE(subq_39.metric_time__day, subq_58.metric_time__day) AS metric_time__day
, COALESCE(subq_39.user__home_state_latest, subq_58.user__home_state_latest) AS user__home_state_latest
COALESCE(subq_39.metric_time__day, subq_57.metric_time__day) AS metric_time__day
, COALESCE(subq_39.user__home_state_latest, subq_57.user__home_state_latest) AS user__home_state_latest
, MAX(subq_39.visits) AS visits
, MAX(subq_58.buys) AS buys
, MAX(subq_57.buys) AS buys
FROM (
-- Constrain Output with WHERE
-- Pass Only Elements: ['visits', 'user__home_state_latest', 'metric_time__day']
Expand Down Expand Up @@ -49,11 +49,11 @@ FROM (
) subq_39
FULL OUTER JOIN (
-- Join Standard Outputs
-- Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day']
-- Pass Only Elements: ['buys', 'user__home_state_latest', 'visit__referrer_id', 'metric_time__day']
-- Aggregate Measures
-- Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day']
SELECT
subq_50.metric_time__day AS metric_time__day
, subq_50.visit__referrer_id AS visit__referrer_id
, users_latest_src_28000.home_state_latest AS user__home_state_latest
, SUM(subq_50.buys) AS buys
FROM (
Expand Down Expand Up @@ -174,15 +174,16 @@ FROM (
subq_50.user = users_latest_src_28000.user_id
GROUP BY
subq_50.metric_time__day
, subq_50.visit__referrer_id
, users_latest_src_28000.home_state_latest
) subq_58
) subq_57
ON
(
subq_39.user__home_state_latest = subq_58.user__home_state_latest
subq_39.user__home_state_latest = subq_57.user__home_state_latest
) AND (
subq_39.metric_time__day = subq_58.metric_time__day
subq_39.metric_time__day = subq_57.metric_time__day
)
GROUP BY
COALESCE(subq_39.metric_time__day, subq_58.metric_time__day)
, COALESCE(subq_39.user__home_state_latest, subq_58.user__home_state_latest)
COALESCE(subq_39.metric_time__day, subq_57.metric_time__day)
, COALESCE(subq_39.user__home_state_latest, subq_57.user__home_state_latest)
) subq_59

0 comments on commit 57228ab

Please sign in to comment.