From ba8bdb2a3be09319e0934222ffb84ef99faf23f8 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Wed, 23 Oct 2024 17:10:29 -0700 Subject: [PATCH] WIP --- .../dataflow/builder/dataflow_plan_builder.py | 14 +- ..._metric_with_custom_granularity__plan0.sql | 562 +++++++++--------- ...th_custom_granularity__plan0_optimized.sql | 95 +-- ...rsion_metric_predicate_pushdown__dfp_0.xml | 10 +- ...sion_metric_predicate_pushdown__dfpo_0.xml | 10 +- ...conversion_metric_query_filters__plan0.sql | 5 +- ..._metric_query_filters__plan0_optimized.sql | 21 +- 7 files changed, 370 insertions(+), 347 deletions(-) diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py index 4ce03ac277..7579fc8982 100644 --- a/metricflow/dataflow/builder/dataflow_plan_builder.py +++ b/metricflow/dataflow/builder/dataflow_plan_builder.py @@ -341,7 +341,7 @@ def _build_aggregated_conversion_node( filtered_unaggregated_base_node = FilterElementsNode.create( parent_node=unaggregated_base_measure_node, include_specs=group_specs_by_type(required_local_specs) - .merge(base_measure_recipe.all_linkable_specs_required_for_source_nodes.as_instance_spec_set) + .merge(base_required_linkable_specs.replace_custom_granularity_with_base_granularity().as_instance_spec_set) .dedupe(), ) @@ -1703,12 +1703,12 @@ def _build_aggregated_measure_from_measure_source_node( non_additive_dimension_grain = self._semantic_model_lookup.get_defined_time_granularity( TimeDimensionReference(non_additive_dimension_spec.name) ) - queried_time_dimension_spec: Optional[TimeDimensionSpec] = ( - self._find_non_additive_dimension_in_linkable_specs( - agg_time_dimension=agg_time_dimension, - linkable_specs=queried_linkable_specs.as_tuple, - non_additive_dimension_spec=non_additive_dimension_spec, - ) + queried_time_dimension_spec: Optional[ + TimeDimensionSpec + ] = self._find_non_additive_dimension_in_linkable_specs( + agg_time_dimension=agg_time_dimension, + linkable_specs=queried_linkable_specs.as_tuple, + non_additive_dimension_spec=non_additive_dimension_spec, ) time_dimension_spec = TimeDimensionSpec( # The NonAdditiveDimensionSpec name property is a plain element name diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql index 5489a3f13c..7727ed7106 100644 --- a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql @@ -1,13 +1,13 @@ -- Compute Metrics via Expressions SELECT - subq_18.metric_time__martian_day - , CAST(subq_18.buys AS DOUBLE) / CAST(NULLIF(subq_18.visits, 0) AS DOUBLE) AS visit_buy_conversion_rate_7days + subq_19.metric_time__martian_day + , CAST(subq_19.buys AS DOUBLE) / CAST(NULLIF(subq_19.visits, 0) AS DOUBLE) AS visit_buy_conversion_rate_7days FROM ( -- Combine Aggregated Outputs SELECT - COALESCE(subq_5.metric_time__martian_day, subq_17.metric_time__martian_day) AS metric_time__martian_day + COALESCE(subq_5.metric_time__martian_day, subq_18.metric_time__martian_day) AS metric_time__martian_day , MAX(subq_5.visits) AS visits - , MAX(subq_17.buys) AS buys + , MAX(subq_18.buys) AS buys FROM ( -- Aggregate Measures SELECT @@ -115,298 +115,304 @@ FROM ( subq_4.metric_time__martian_day ) subq_5 FULL OUTER JOIN ( - -- Aggregate Measures + -- Pass Only Elements: ['buys', 'metric_time__martian_day'] SELECT - subq_16.metric_time__martian_day - , SUM(subq_16.buys) AS buys + subq_17.metric_time__martian_day + , subq_17.buys FROM ( - -- Pass Only Elements: ['buys', 'metric_time__martian_day'] + -- Aggregate Measures SELECT - subq_15.metric_time__martian_day - , subq_15.buys + subq_16.metric_time__martian_day + , SUM(subq_16.buys) AS buys FROM ( - -- Pass Only Elements: ['buys', 'metric_time__day'] - -- Join to Custom Granularity Dataset + -- Pass Only Elements: ['buys', 'metric_time__martian_day'] SELECT - subq_13.metric_time__day AS metric_time__day - , subq_13.buys AS buys - , subq_14.martian_day AS metric_time__martian_day + subq_15.metric_time__martian_day + , subq_15.buys FROM ( - -- Find conversions for user within the range of 7 day + -- Pass Only Elements: ['buys', 'metric_time__day'] + -- Join to Custom Granularity Dataset SELECT - subq_12.ds__day - , subq_12.metric_time__day - , subq_12.user - , subq_12.buys - , subq_12.visits + subq_13.metric_time__day AS metric_time__day + , subq_13.buys AS buys + , subq_14.martian_day AS metric_time__martian_day FROM ( - -- Dedupe the fanout with mf_internal_uuid in the conversion data set - SELECT DISTINCT - FIRST_VALUE(subq_8.visits) OVER ( - PARTITION BY - subq_11.user - , subq_11.ds__day - , subq_11.mf_internal_uuid - ORDER BY subq_8.ds__day DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS visits - , FIRST_VALUE(subq_8.ds__day) OVER ( - PARTITION BY - subq_11.user - , subq_11.ds__day - , subq_11.mf_internal_uuid - ORDER BY subq_8.ds__day DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS ds__day - , FIRST_VALUE(subq_8.metric_time__day) OVER ( - PARTITION BY - subq_11.user - , subq_11.ds__day - , subq_11.mf_internal_uuid - ORDER BY subq_8.ds__day DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS metric_time__day - , FIRST_VALUE(subq_8.user) OVER ( - PARTITION BY - subq_11.user - , subq_11.ds__day - , subq_11.mf_internal_uuid - ORDER BY subq_8.ds__day DESC - ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING - ) AS user - , subq_11.mf_internal_uuid AS mf_internal_uuid - , subq_11.buys AS buys + -- Find conversions for user within the range of 7 day + SELECT + subq_12.ds__day + , subq_12.metric_time__day + , subq_12.user + , subq_12.buys + , subq_12.visits FROM ( - -- Pass Only Elements: ['visits', 'ds__day', 'metric_time__day', 'user'] - SELECT - subq_7.ds__day - , subq_7.metric_time__day - , subq_7.user - , subq_7.visits + -- Dedupe the fanout with mf_internal_uuid in the conversion data set + SELECT DISTINCT + FIRST_VALUE(subq_8.visits) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS visits + , FIRST_VALUE(subq_8.ds__day) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS ds__day + , FIRST_VALUE(subq_8.metric_time__day) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS metric_time__day + , FIRST_VALUE(subq_8.user) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS user + , subq_11.mf_internal_uuid AS mf_internal_uuid + , subq_11.buys AS buys FROM ( - -- Metric Time Dimension 'ds' + -- Pass Only Elements: ['visits', 'ds__day', 'metric_time__day', 'user'] SELECT - subq_6.ds__day - , subq_6.ds__week - , subq_6.ds__month - , subq_6.ds__quarter - , subq_6.ds__year - , subq_6.ds__extract_year - , subq_6.ds__extract_quarter - , subq_6.ds__extract_month - , subq_6.ds__extract_day - , subq_6.ds__extract_dow - , subq_6.ds__extract_doy - , subq_6.visit__ds__day - , subq_6.visit__ds__week - , subq_6.visit__ds__month - , subq_6.visit__ds__quarter - , subq_6.visit__ds__year - , subq_6.visit__ds__extract_year - , subq_6.visit__ds__extract_quarter - , subq_6.visit__ds__extract_month - , subq_6.visit__ds__extract_day - , subq_6.visit__ds__extract_dow - , subq_6.visit__ds__extract_doy - , subq_6.ds__day AS metric_time__day - , subq_6.ds__week AS metric_time__week - , subq_6.ds__month AS metric_time__month - , subq_6.ds__quarter AS metric_time__quarter - , subq_6.ds__year AS metric_time__year - , subq_6.ds__extract_year AS metric_time__extract_year - , subq_6.ds__extract_quarter AS metric_time__extract_quarter - , subq_6.ds__extract_month AS metric_time__extract_month - , subq_6.ds__extract_day AS metric_time__extract_day - , subq_6.ds__extract_dow AS metric_time__extract_dow - , subq_6.ds__extract_doy AS metric_time__extract_doy - , subq_6.user - , subq_6.session - , subq_6.visit__user - , subq_6.visit__session - , subq_6.referrer_id - , subq_6.visit__referrer_id - , subq_6.visits - , subq_6.visitors + subq_7.ds__day + , subq_7.metric_time__day + , subq_7.user + , subq_7.visits FROM ( - -- Read Elements From Semantic Model 'visits_source' + -- Metric Time Dimension 'ds' SELECT - 1 AS visits - , visits_source_src_28000.user_id AS visitors - , DATE_TRUNC('day', visits_source_src_28000.ds) AS ds__day - , DATE_TRUNC('week', visits_source_src_28000.ds) AS ds__week - , DATE_TRUNC('month', visits_source_src_28000.ds) AS ds__month - , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS ds__quarter - , DATE_TRUNC('year', visits_source_src_28000.ds) AS ds__year - , EXTRACT(year FROM visits_source_src_28000.ds) AS ds__extract_year - , EXTRACT(quarter FROM visits_source_src_28000.ds) AS ds__extract_quarter - , EXTRACT(month FROM visits_source_src_28000.ds) AS ds__extract_month - , EXTRACT(day FROM visits_source_src_28000.ds) AS ds__extract_day - , EXTRACT(isodow FROM visits_source_src_28000.ds) AS ds__extract_dow - , EXTRACT(doy FROM visits_source_src_28000.ds) AS ds__extract_doy - , visits_source_src_28000.referrer_id - , DATE_TRUNC('day', visits_source_src_28000.ds) AS visit__ds__day - , DATE_TRUNC('week', visits_source_src_28000.ds) AS visit__ds__week - , DATE_TRUNC('month', visits_source_src_28000.ds) AS visit__ds__month - , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS visit__ds__quarter - , DATE_TRUNC('year', visits_source_src_28000.ds) AS visit__ds__year - , EXTRACT(year FROM visits_source_src_28000.ds) AS visit__ds__extract_year - , EXTRACT(quarter FROM visits_source_src_28000.ds) AS visit__ds__extract_quarter - , EXTRACT(month FROM visits_source_src_28000.ds) AS visit__ds__extract_month - , EXTRACT(day FROM visits_source_src_28000.ds) AS visit__ds__extract_day - , EXTRACT(isodow FROM visits_source_src_28000.ds) AS visit__ds__extract_dow - , EXTRACT(doy FROM visits_source_src_28000.ds) AS visit__ds__extract_doy - , visits_source_src_28000.referrer_id AS visit__referrer_id - , visits_source_src_28000.user_id AS user - , visits_source_src_28000.session_id AS session - , visits_source_src_28000.user_id AS visit__user - , visits_source_src_28000.session_id AS visit__session - FROM ***************************.fct_visits visits_source_src_28000 - ) subq_6 - ) subq_7 - ) subq_8 - INNER JOIN ( - -- Add column with generated UUID - SELECT - subq_10.ds__day - , subq_10.ds__week - , subq_10.ds__month - , subq_10.ds__quarter - , subq_10.ds__year - , subq_10.ds__extract_year - , subq_10.ds__extract_quarter - , subq_10.ds__extract_month - , subq_10.ds__extract_day - , subq_10.ds__extract_dow - , subq_10.ds__extract_doy - , subq_10.buy__ds__day - , subq_10.buy__ds__week - , subq_10.buy__ds__month - , subq_10.buy__ds__quarter - , subq_10.buy__ds__year - , subq_10.buy__ds__extract_year - , subq_10.buy__ds__extract_quarter - , subq_10.buy__ds__extract_month - , subq_10.buy__ds__extract_day - , subq_10.buy__ds__extract_dow - , subq_10.buy__ds__extract_doy - , subq_10.metric_time__day - , subq_10.metric_time__week - , subq_10.metric_time__month - , subq_10.metric_time__quarter - , subq_10.metric_time__year - , subq_10.metric_time__extract_year - , subq_10.metric_time__extract_quarter - , subq_10.metric_time__extract_month - , subq_10.metric_time__extract_day - , subq_10.metric_time__extract_dow - , subq_10.metric_time__extract_doy - , subq_10.user - , subq_10.session_id - , subq_10.buy__user - , subq_10.buy__session_id - , subq_10.buys - , subq_10.buyers - , GEN_RANDOM_UUID() AS mf_internal_uuid - FROM ( - -- Metric Time Dimension 'ds' + subq_6.ds__day + , subq_6.ds__week + , subq_6.ds__month + , subq_6.ds__quarter + , subq_6.ds__year + , subq_6.ds__extract_year + , subq_6.ds__extract_quarter + , subq_6.ds__extract_month + , subq_6.ds__extract_day + , subq_6.ds__extract_dow + , subq_6.ds__extract_doy + , subq_6.visit__ds__day + , subq_6.visit__ds__week + , subq_6.visit__ds__month + , subq_6.visit__ds__quarter + , subq_6.visit__ds__year + , subq_6.visit__ds__extract_year + , subq_6.visit__ds__extract_quarter + , subq_6.visit__ds__extract_month + , subq_6.visit__ds__extract_day + , subq_6.visit__ds__extract_dow + , subq_6.visit__ds__extract_doy + , subq_6.ds__day AS metric_time__day + , subq_6.ds__week AS metric_time__week + , subq_6.ds__month AS metric_time__month + , subq_6.ds__quarter AS metric_time__quarter + , subq_6.ds__year AS metric_time__year + , subq_6.ds__extract_year AS metric_time__extract_year + , subq_6.ds__extract_quarter AS metric_time__extract_quarter + , subq_6.ds__extract_month AS metric_time__extract_month + , subq_6.ds__extract_day AS metric_time__extract_day + , subq_6.ds__extract_dow AS metric_time__extract_dow + , subq_6.ds__extract_doy AS metric_time__extract_doy + , subq_6.user + , subq_6.session + , subq_6.visit__user + , subq_6.visit__session + , subq_6.referrer_id + , subq_6.visit__referrer_id + , subq_6.visits + , subq_6.visitors + FROM ( + -- Read Elements From Semantic Model 'visits_source' + SELECT + 1 AS visits + , visits_source_src_28000.user_id AS visitors + , DATE_TRUNC('day', visits_source_src_28000.ds) AS ds__day + , DATE_TRUNC('week', visits_source_src_28000.ds) AS ds__week + , DATE_TRUNC('month', visits_source_src_28000.ds) AS ds__month + , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS ds__quarter + , DATE_TRUNC('year', visits_source_src_28000.ds) AS ds__year + , EXTRACT(year FROM visits_source_src_28000.ds) AS ds__extract_year + , EXTRACT(quarter FROM visits_source_src_28000.ds) AS ds__extract_quarter + , EXTRACT(month FROM visits_source_src_28000.ds) AS ds__extract_month + , EXTRACT(day FROM visits_source_src_28000.ds) AS ds__extract_day + , EXTRACT(isodow FROM visits_source_src_28000.ds) AS ds__extract_dow + , EXTRACT(doy FROM visits_source_src_28000.ds) AS ds__extract_doy + , visits_source_src_28000.referrer_id + , DATE_TRUNC('day', visits_source_src_28000.ds) AS visit__ds__day + , DATE_TRUNC('week', visits_source_src_28000.ds) AS visit__ds__week + , DATE_TRUNC('month', visits_source_src_28000.ds) AS visit__ds__month + , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS visit__ds__quarter + , DATE_TRUNC('year', visits_source_src_28000.ds) AS visit__ds__year + , EXTRACT(year FROM visits_source_src_28000.ds) AS visit__ds__extract_year + , EXTRACT(quarter FROM visits_source_src_28000.ds) AS visit__ds__extract_quarter + , EXTRACT(month FROM visits_source_src_28000.ds) AS visit__ds__extract_month + , EXTRACT(day FROM visits_source_src_28000.ds) AS visit__ds__extract_day + , EXTRACT(isodow FROM visits_source_src_28000.ds) AS visit__ds__extract_dow + , EXTRACT(doy FROM visits_source_src_28000.ds) AS visit__ds__extract_doy + , visits_source_src_28000.referrer_id AS visit__referrer_id + , visits_source_src_28000.user_id AS user + , visits_source_src_28000.session_id AS session + , visits_source_src_28000.user_id AS visit__user + , visits_source_src_28000.session_id AS visit__session + FROM ***************************.fct_visits visits_source_src_28000 + ) subq_6 + ) subq_7 + ) subq_8 + INNER JOIN ( + -- Add column with generated UUID SELECT - subq_9.ds__day - , subq_9.ds__week - , subq_9.ds__month - , subq_9.ds__quarter - , subq_9.ds__year - , subq_9.ds__extract_year - , subq_9.ds__extract_quarter - , subq_9.ds__extract_month - , subq_9.ds__extract_day - , subq_9.ds__extract_dow - , subq_9.ds__extract_doy - , subq_9.buy__ds__day - , subq_9.buy__ds__week - , subq_9.buy__ds__month - , subq_9.buy__ds__quarter - , subq_9.buy__ds__year - , subq_9.buy__ds__extract_year - , subq_9.buy__ds__extract_quarter - , subq_9.buy__ds__extract_month - , subq_9.buy__ds__extract_day - , subq_9.buy__ds__extract_dow - , subq_9.buy__ds__extract_doy - , subq_9.ds__day AS metric_time__day - , subq_9.ds__week AS metric_time__week - , subq_9.ds__month AS metric_time__month - , subq_9.ds__quarter AS metric_time__quarter - , subq_9.ds__year AS metric_time__year - , subq_9.ds__extract_year AS metric_time__extract_year - , subq_9.ds__extract_quarter AS metric_time__extract_quarter - , subq_9.ds__extract_month AS metric_time__extract_month - , subq_9.ds__extract_day AS metric_time__extract_day - , subq_9.ds__extract_dow AS metric_time__extract_dow - , subq_9.ds__extract_doy AS metric_time__extract_doy - , subq_9.user - , subq_9.session_id - , subq_9.buy__user - , subq_9.buy__session_id - , subq_9.buys - , subq_9.buyers + subq_10.ds__day + , subq_10.ds__week + , subq_10.ds__month + , subq_10.ds__quarter + , subq_10.ds__year + , subq_10.ds__extract_year + , subq_10.ds__extract_quarter + , subq_10.ds__extract_month + , subq_10.ds__extract_day + , subq_10.ds__extract_dow + , subq_10.ds__extract_doy + , subq_10.buy__ds__day + , subq_10.buy__ds__week + , subq_10.buy__ds__month + , subq_10.buy__ds__quarter + , subq_10.buy__ds__year + , subq_10.buy__ds__extract_year + , subq_10.buy__ds__extract_quarter + , subq_10.buy__ds__extract_month + , subq_10.buy__ds__extract_day + , subq_10.buy__ds__extract_dow + , subq_10.buy__ds__extract_doy + , subq_10.metric_time__day + , subq_10.metric_time__week + , subq_10.metric_time__month + , subq_10.metric_time__quarter + , subq_10.metric_time__year + , subq_10.metric_time__extract_year + , subq_10.metric_time__extract_quarter + , subq_10.metric_time__extract_month + , subq_10.metric_time__extract_day + , subq_10.metric_time__extract_dow + , subq_10.metric_time__extract_doy + , subq_10.user + , subq_10.session_id + , subq_10.buy__user + , subq_10.buy__session_id + , subq_10.buys + , subq_10.buyers + , GEN_RANDOM_UUID() AS mf_internal_uuid FROM ( - -- Read Elements From Semantic Model 'buys_source' + -- Metric Time Dimension 'ds' SELECT - 1 AS buys - , buys_source_src_28000.user_id AS buyers - , DATE_TRUNC('day', buys_source_src_28000.ds) AS ds__day - , DATE_TRUNC('week', buys_source_src_28000.ds) AS ds__week - , DATE_TRUNC('month', buys_source_src_28000.ds) AS ds__month - , DATE_TRUNC('quarter', buys_source_src_28000.ds) AS ds__quarter - , DATE_TRUNC('year', buys_source_src_28000.ds) AS ds__year - , EXTRACT(year FROM buys_source_src_28000.ds) AS ds__extract_year - , EXTRACT(quarter FROM buys_source_src_28000.ds) AS ds__extract_quarter - , EXTRACT(month FROM buys_source_src_28000.ds) AS ds__extract_month - , EXTRACT(day FROM buys_source_src_28000.ds) AS ds__extract_day - , EXTRACT(isodow FROM buys_source_src_28000.ds) AS ds__extract_dow - , EXTRACT(doy FROM buys_source_src_28000.ds) AS ds__extract_doy - , DATE_TRUNC('day', buys_source_src_28000.ds) AS buy__ds__day - , DATE_TRUNC('week', buys_source_src_28000.ds) AS buy__ds__week - , DATE_TRUNC('month', buys_source_src_28000.ds) AS buy__ds__month - , DATE_TRUNC('quarter', buys_source_src_28000.ds) AS buy__ds__quarter - , DATE_TRUNC('year', buys_source_src_28000.ds) AS buy__ds__year - , EXTRACT(year FROM buys_source_src_28000.ds) AS buy__ds__extract_year - , EXTRACT(quarter FROM buys_source_src_28000.ds) AS buy__ds__extract_quarter - , EXTRACT(month FROM buys_source_src_28000.ds) AS buy__ds__extract_month - , EXTRACT(day FROM buys_source_src_28000.ds) AS buy__ds__extract_day - , EXTRACT(isodow FROM buys_source_src_28000.ds) AS buy__ds__extract_dow - , EXTRACT(doy FROM buys_source_src_28000.ds) AS buy__ds__extract_doy - , buys_source_src_28000.user_id AS user - , buys_source_src_28000.session_id - , buys_source_src_28000.user_id AS buy__user - , buys_source_src_28000.session_id AS buy__session_id - FROM ***************************.fct_buys buys_source_src_28000 - ) subq_9 - ) subq_10 - ) subq_11 - ON - ( - subq_8.user = subq_11.user - ) AND ( + subq_9.ds__day + , subq_9.ds__week + , subq_9.ds__month + , subq_9.ds__quarter + , subq_9.ds__year + , subq_9.ds__extract_year + , subq_9.ds__extract_quarter + , subq_9.ds__extract_month + , subq_9.ds__extract_day + , subq_9.ds__extract_dow + , subq_9.ds__extract_doy + , subq_9.buy__ds__day + , subq_9.buy__ds__week + , subq_9.buy__ds__month + , subq_9.buy__ds__quarter + , subq_9.buy__ds__year + , subq_9.buy__ds__extract_year + , subq_9.buy__ds__extract_quarter + , subq_9.buy__ds__extract_month + , subq_9.buy__ds__extract_day + , subq_9.buy__ds__extract_dow + , subq_9.buy__ds__extract_doy + , subq_9.ds__day AS metric_time__day + , subq_9.ds__week AS metric_time__week + , subq_9.ds__month AS metric_time__month + , subq_9.ds__quarter AS metric_time__quarter + , subq_9.ds__year AS metric_time__year + , subq_9.ds__extract_year AS metric_time__extract_year + , subq_9.ds__extract_quarter AS metric_time__extract_quarter + , subq_9.ds__extract_month AS metric_time__extract_month + , subq_9.ds__extract_day AS metric_time__extract_day + , subq_9.ds__extract_dow AS metric_time__extract_dow + , subq_9.ds__extract_doy AS metric_time__extract_doy + , subq_9.user + , subq_9.session_id + , subq_9.buy__user + , subq_9.buy__session_id + , subq_9.buys + , subq_9.buyers + FROM ( + -- Read Elements From Semantic Model 'buys_source' + SELECT + 1 AS buys + , buys_source_src_28000.user_id AS buyers + , DATE_TRUNC('day', buys_source_src_28000.ds) AS ds__day + , DATE_TRUNC('week', buys_source_src_28000.ds) AS ds__week + , DATE_TRUNC('month', buys_source_src_28000.ds) AS ds__month + , DATE_TRUNC('quarter', buys_source_src_28000.ds) AS ds__quarter + , DATE_TRUNC('year', buys_source_src_28000.ds) AS ds__year + , EXTRACT(year FROM buys_source_src_28000.ds) AS ds__extract_year + , EXTRACT(quarter FROM buys_source_src_28000.ds) AS ds__extract_quarter + , EXTRACT(month FROM buys_source_src_28000.ds) AS ds__extract_month + , EXTRACT(day FROM buys_source_src_28000.ds) AS ds__extract_day + , EXTRACT(isodow FROM buys_source_src_28000.ds) AS ds__extract_dow + , EXTRACT(doy FROM buys_source_src_28000.ds) AS ds__extract_doy + , DATE_TRUNC('day', buys_source_src_28000.ds) AS buy__ds__day + , DATE_TRUNC('week', buys_source_src_28000.ds) AS buy__ds__week + , DATE_TRUNC('month', buys_source_src_28000.ds) AS buy__ds__month + , DATE_TRUNC('quarter', buys_source_src_28000.ds) AS buy__ds__quarter + , DATE_TRUNC('year', buys_source_src_28000.ds) AS buy__ds__year + , EXTRACT(year FROM buys_source_src_28000.ds) AS buy__ds__extract_year + , EXTRACT(quarter FROM buys_source_src_28000.ds) AS buy__ds__extract_quarter + , EXTRACT(month FROM buys_source_src_28000.ds) AS buy__ds__extract_month + , EXTRACT(day FROM buys_source_src_28000.ds) AS buy__ds__extract_day + , EXTRACT(isodow FROM buys_source_src_28000.ds) AS buy__ds__extract_dow + , EXTRACT(doy FROM buys_source_src_28000.ds) AS buy__ds__extract_doy + , buys_source_src_28000.user_id AS user + , buys_source_src_28000.session_id + , buys_source_src_28000.user_id AS buy__user + , buys_source_src_28000.session_id AS buy__session_id + FROM ***************************.fct_buys buys_source_src_28000 + ) subq_9 + ) subq_10 + ) subq_11 + ON ( - subq_8.ds__day <= subq_11.ds__day + subq_8.user = subq_11.user ) AND ( - subq_8.ds__day > subq_11.ds__day - INTERVAL 7 day + ( + subq_8.ds__day <= subq_11.ds__day + ) AND ( + subq_8.ds__day > subq_11.ds__day - INTERVAL 7 day + ) ) - ) - ) subq_12 - ) subq_13 - LEFT OUTER JOIN - ***************************.mf_time_spine subq_14 - ON - subq_13.metric_time__day = subq_14.ds - ) subq_15 - ) subq_16 - GROUP BY - subq_16.metric_time__martian_day - ) subq_17 + ) subq_12 + ) subq_13 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_14 + ON + subq_13.metric_time__day = subq_14.ds + ) subq_15 + ) subq_16 + GROUP BY + subq_16.metric_time__martian_day + ) subq_17 + ) subq_18 ON - subq_5.metric_time__martian_day = subq_17.metric_time__martian_day + subq_5.metric_time__martian_day = subq_18.metric_time__martian_day GROUP BY - COALESCE(subq_5.metric_time__martian_day, subq_17.metric_time__martian_day) -) subq_18 + COALESCE(subq_5.metric_time__martian_day, subq_18.metric_time__martian_day) +) subq_19 diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql index ebdabae734..b9864dd90d 100644 --- a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql @@ -5,17 +5,17 @@ SELECT FROM ( -- Combine Aggregated Outputs SELECT - COALESCE(subq_24.metric_time__martian_day, subq_36.metric_time__martian_day) AS metric_time__martian_day - , MAX(subq_24.visits) AS visits - , MAX(subq_36.buys) AS buys + COALESCE(subq_25.metric_time__martian_day, subq_38.metric_time__martian_day) AS metric_time__martian_day + , MAX(subq_25.visits) AS visits + , MAX(subq_38.buys) AS buys FROM ( -- Pass Only Elements: ['visits', 'metric_time__day'] -- Join to Custom Granularity Dataset -- Pass Only Elements: ['visits', 'metric_time__martian_day'] -- Aggregate Measures SELECT - subq_21.martian_day AS metric_time__martian_day - , SUM(subq_20.visits) AS visits + subq_22.martian_day AS metric_time__martian_day + , SUM(subq_21.visits) AS visits FROM ( -- Read Elements From Semantic Model 'visits_source' -- Metric Time Dimension 'ds' @@ -23,59 +23,60 @@ FROM ( DATE_TRUNC('day', ds) AS metric_time__day , 1 AS visits FROM ***************************.fct_visits visits_source_src_28000 - ) subq_20 + ) subq_21 LEFT OUTER JOIN - ***************************.mf_time_spine subq_21 + ***************************.mf_time_spine subq_22 ON - subq_20.metric_time__day = subq_21.ds + subq_21.metric_time__day = subq_22.ds GROUP BY - subq_21.martian_day - ) subq_24 + subq_22.martian_day + ) subq_25 FULL OUTER JOIN ( -- Pass Only Elements: ['buys', 'metric_time__day'] -- Join to Custom Granularity Dataset -- Pass Only Elements: ['buys', 'metric_time__martian_day'] -- Aggregate Measures + -- Pass Only Elements: ['buys', 'metric_time__martian_day'] SELECT - subq_33.martian_day AS metric_time__martian_day - , SUM(subq_31.buys) AS buys + subq_34.martian_day AS metric_time__martian_day + , SUM(subq_32.buys) AS buys FROM ( -- Dedupe the fanout with mf_internal_uuid in the conversion data set SELECT DISTINCT - FIRST_VALUE(subq_27.visits) OVER ( + FIRST_VALUE(subq_28.visits) OVER ( PARTITION BY - subq_30.user - , subq_30.ds__day - , subq_30.mf_internal_uuid - ORDER BY subq_27.ds__day DESC + subq_31.user + , subq_31.ds__day + , subq_31.mf_internal_uuid + ORDER BY subq_28.ds__day DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS visits - , FIRST_VALUE(subq_27.ds__day) OVER ( + , FIRST_VALUE(subq_28.ds__day) OVER ( PARTITION BY - subq_30.user - , subq_30.ds__day - , subq_30.mf_internal_uuid - ORDER BY subq_27.ds__day DESC + subq_31.user + , subq_31.ds__day + , subq_31.mf_internal_uuid + ORDER BY subq_28.ds__day DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS ds__day - , FIRST_VALUE(subq_27.metric_time__day) OVER ( + , FIRST_VALUE(subq_28.metric_time__day) OVER ( PARTITION BY - subq_30.user - , subq_30.ds__day - , subq_30.mf_internal_uuid - ORDER BY subq_27.ds__day DESC + subq_31.user + , subq_31.ds__day + , subq_31.mf_internal_uuid + ORDER BY subq_28.ds__day DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS metric_time__day - , FIRST_VALUE(subq_27.user) OVER ( + , FIRST_VALUE(subq_28.user) OVER ( PARTITION BY - subq_30.user - , subq_30.ds__day - , subq_30.mf_internal_uuid - ORDER BY subq_27.ds__day DESC + subq_31.user + , subq_31.ds__day + , subq_31.mf_internal_uuid + ORDER BY subq_28.ds__day DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) AS user - , subq_30.mf_internal_uuid AS mf_internal_uuid - , subq_30.buys AS buys + , subq_31.mf_internal_uuid AS mf_internal_uuid + , subq_31.buys AS buys FROM ( -- Read Elements From Semantic Model 'visits_source' -- Metric Time Dimension 'ds' @@ -86,7 +87,7 @@ FROM ( , user_id AS user , 1 AS visits FROM ***************************.fct_visits visits_source_src_28000 - ) subq_27 + ) subq_28 INNER JOIN ( -- Read Elements From Semantic Model 'buys_source' -- Metric Time Dimension 'ds' @@ -97,27 +98,27 @@ FROM ( , 1 AS buys , GEN_RANDOM_UUID() AS mf_internal_uuid FROM ***************************.fct_buys buys_source_src_28000 - ) subq_30 + ) subq_31 ON ( - subq_27.user = subq_30.user + subq_28.user = subq_31.user ) AND ( ( - subq_27.ds__day <= subq_30.ds__day + subq_28.ds__day <= subq_31.ds__day ) AND ( - subq_27.ds__day > subq_30.ds__day - INTERVAL 7 day + subq_28.ds__day > subq_31.ds__day - INTERVAL 7 day ) ) - ) subq_31 + ) subq_32 LEFT OUTER JOIN - ***************************.mf_time_spine subq_33 + ***************************.mf_time_spine subq_34 ON - subq_31.metric_time__day = subq_33.ds + subq_32.metric_time__day = subq_34.ds GROUP BY - subq_33.martian_day - ) subq_36 + subq_34.martian_day + ) subq_38 ON - subq_24.metric_time__martian_day = subq_36.metric_time__martian_day + subq_25.metric_time__martian_day = subq_38.metric_time__martian_day GROUP BY - COALESCE(subq_24.metric_time__martian_day, subq_36.metric_time__martian_day) -) subq_37 + COALESCE(subq_25.metric_time__martian_day, subq_38.metric_time__martian_day) +) subq_39 diff --git a/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfp_0.xml b/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfp_0.xml index bb350581db..d0f1b04804 100644 --- a/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfp_0.xml +++ b/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfp_0.xml @@ -216,8 +216,9 @@ - - + + + @@ -225,6 +226,11 @@ + + + + + diff --git a/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfpo_0.xml b/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfpo_0.xml index 549e3e0d73..443aff9122 100644 --- a/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfpo_0.xml +++ b/tests_metricflow/snapshots/test_predicate_pushdown_optimizer.py/DataflowPlan/test_conversion_metric_predicate_pushdown__dfpo_0.xml @@ -213,8 +213,9 @@ - - + + + @@ -222,6 +223,11 @@ + + + + + diff --git a/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0.sql b/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0.sql index 09634af5ca..ccc5cdaba2 100644 --- a/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0.sql +++ b/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0.sql @@ -189,12 +189,14 @@ FROM ( -- Aggregate Measures SELECT subq_26.metric_time__day + , subq_26.visit__referrer_id , subq_26.user__home_state_latest , SUM(subq_26.buys) AS buys FROM ( - -- Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day'] + -- Pass Only Elements: ['buys', 'user__home_state_latest', 'visit__referrer_id', 'metric_time__day'] SELECT subq_25.metric_time__day + , subq_25.visit__referrer_id , subq_25.user__home_state_latest , subq_25.buys FROM ( @@ -671,6 +673,7 @@ FROM ( ) subq_26 GROUP BY subq_26.metric_time__day + , subq_26.visit__referrer_id , subq_26.user__home_state_latest ) subq_27 ) subq_28 diff --git a/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0_optimized.sql b/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0_optimized.sql index efe71e978d..f8b9ccf159 100644 --- a/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0_optimized.sql +++ b/tests_metricflow/snapshots/test_predicate_pushdown_rendering.py/SqlQueryPlan/DuckDB/test_conversion_metric_query_filters__plan0_optimized.sql @@ -6,10 +6,10 @@ SELECT FROM ( -- Combine Aggregated Outputs SELECT - COALESCE(subq_39.metric_time__day, subq_58.metric_time__day) AS metric_time__day - , COALESCE(subq_39.user__home_state_latest, subq_58.user__home_state_latest) AS user__home_state_latest + COALESCE(subq_39.metric_time__day, subq_57.metric_time__day) AS metric_time__day + , COALESCE(subq_39.user__home_state_latest, subq_57.user__home_state_latest) AS user__home_state_latest , MAX(subq_39.visits) AS visits - , MAX(subq_58.buys) AS buys + , MAX(subq_57.buys) AS buys FROM ( -- Constrain Output with WHERE -- Pass Only Elements: ['visits', 'user__home_state_latest', 'metric_time__day'] @@ -49,11 +49,11 @@ FROM ( ) subq_39 FULL OUTER JOIN ( -- Join Standard Outputs - -- Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day'] + -- Pass Only Elements: ['buys', 'user__home_state_latest', 'visit__referrer_id', 'metric_time__day'] -- Aggregate Measures - -- Pass Only Elements: ['buys', 'user__home_state_latest', 'metric_time__day'] SELECT subq_50.metric_time__day AS metric_time__day + , subq_50.visit__referrer_id AS visit__referrer_id , users_latest_src_28000.home_state_latest AS user__home_state_latest , SUM(subq_50.buys) AS buys FROM ( @@ -174,15 +174,16 @@ FROM ( subq_50.user = users_latest_src_28000.user_id GROUP BY subq_50.metric_time__day + , subq_50.visit__referrer_id , users_latest_src_28000.home_state_latest - ) subq_58 + ) subq_57 ON ( - subq_39.user__home_state_latest = subq_58.user__home_state_latest + subq_39.user__home_state_latest = subq_57.user__home_state_latest ) AND ( - subq_39.metric_time__day = subq_58.metric_time__day + subq_39.metric_time__day = subq_57.metric_time__day ) GROUP BY - COALESCE(subq_39.metric_time__day, subq_58.metric_time__day) - , COALESCE(subq_39.user__home_state_latest, subq_58.user__home_state_latest) + COALESCE(subq_39.metric_time__day, subq_57.metric_time__day) + , COALESCE(subq_39.user__home_state_latest, subq_57.user__home_state_latest) ) subq_59