From 9f08b9f9a9b7f96b0fecbd3319b7a222f07dfc26 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Fri, 18 Oct 2024 15:26:19 -0700 Subject: [PATCH] WIP - custom grain for conversion metrics --- .../dataflow/builder/dataflow_plan_builder.py | 16 +- .../test_custom_granularity.py | 25 +- ..._metric_with_custom_granularity__plan0.sql | 412 ++++++++++++++++++ ...th_custom_granularity__plan0_optimized.sql | 123 ++++++ ..._metric_with_custom_granularity__plan0.sql | 239 ++++++++++ ...th_custom_granularity__plan0_optimized.sql | 31 ++ 6 files changed, 836 insertions(+), 10 deletions(-) create mode 100644 tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql create mode 100644 tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql create mode 100644 tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0.sql create mode 100644 tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0_optimized.sql diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py index 856384028c..29ed58e0a2 100644 --- a/metricflow/dataflow/builder/dataflow_plan_builder.py +++ b/metricflow/dataflow/builder/dataflow_plan_builder.py @@ -333,10 +333,12 @@ def _build_aggregated_conversion_node( unaggregated_base_measure_node = JoinOnEntitiesNode.create( left_node=unaggregated_base_measure_node, join_targets=base_measure_recipe.join_targets ) + # TODO: write explanatory comment; or maybe refactor so this doesn't need to live here + filter_to_specs = base_required_linkable_specs.replace_custom_granularity_with_base_granularity() filtered_unaggregated_base_node = FilterElementsNode.create( parent_node=unaggregated_base_measure_node, include_specs=group_specs_by_type(required_local_specs) - .merge(InstanceSpecSet.create_from_specs(base_required_linkable_specs.as_tuple)) + .merge(InstanceSpecSet.create_from_specs(filter_to_specs.as_tuple)) .dedupe(), ) @@ -1696,12 +1698,12 @@ def _build_aggregated_measure_from_measure_source_node( non_additive_dimension_grain = self._semantic_model_lookup.get_defined_time_granularity( TimeDimensionReference(non_additive_dimension_spec.name) ) - queried_time_dimension_spec: Optional[ - TimeDimensionSpec - ] = self._find_non_additive_dimension_in_linkable_specs( - agg_time_dimension=agg_time_dimension, - linkable_specs=queried_linkable_specs.as_tuple, - non_additive_dimension_spec=non_additive_dimension_spec, + queried_time_dimension_spec: Optional[TimeDimensionSpec] = ( + self._find_non_additive_dimension_in_linkable_specs( + agg_time_dimension=agg_time_dimension, + linkable_specs=queried_linkable_specs.as_tuple, + non_additive_dimension_spec=non_additive_dimension_spec, + ) ) time_dimension_spec = TimeDimensionSpec( # The NonAdditiveDimensionSpec name property is a plain element name diff --git a/tests_metricflow/query_rendering/test_custom_granularity.py b/tests_metricflow/query_rendering/test_custom_granularity.py index 8c8d1a0199..36317b3720 100644 --- a/tests_metricflow/query_rendering/test_custom_granularity.py +++ b/tests_metricflow/query_rendering/test_custom_granularity.py @@ -432,6 +432,25 @@ def test_offset_metric_with_custom_granularity_filter_not_in_group_by( # noqa: ) -# Tests TODO: -# - join_to_timespine metric -# - join_to_timespine metric with filter on custom grain that's not in the group by (dun dun dun) +@pytest.mark.sql_engine_snapshot +def test_conversion_metric_with_custom_granularity( # noqa: D103 + request: FixtureRequest, + mf_test_configuration: MetricFlowTestConfiguration, + dataflow_plan_builder: DataflowPlanBuilder, + dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, + sql_client: SqlClient, + query_parser: MetricFlowQueryParser, +) -> None: + query_spec = query_parser.parse_and_validate_query( + metric_names=("visit_buy_conversion_rate_7days",), + group_by_names=("metric_time__martian_day",), + ).query_spec + + render_and_check( + request=request, + mf_test_configuration=mf_test_configuration, + dataflow_to_sql_converter=dataflow_to_sql_converter, + sql_client=sql_client, + dataflow_plan_builder=dataflow_plan_builder, + query_spec=query_spec, + ) diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql new file mode 100644 index 0000000000..5489a3f13c --- /dev/null +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0.sql @@ -0,0 +1,412 @@ +-- Compute Metrics via Expressions +SELECT + subq_18.metric_time__martian_day + , CAST(subq_18.buys AS DOUBLE) / CAST(NULLIF(subq_18.visits, 0) AS DOUBLE) AS visit_buy_conversion_rate_7days +FROM ( + -- Combine Aggregated Outputs + SELECT + COALESCE(subq_5.metric_time__martian_day, subq_17.metric_time__martian_day) AS metric_time__martian_day + , MAX(subq_5.visits) AS visits + , MAX(subq_17.buys) AS buys + FROM ( + -- Aggregate Measures + SELECT + subq_4.metric_time__martian_day + , SUM(subq_4.visits) AS visits + FROM ( + -- Pass Only Elements: ['visits', 'metric_time__martian_day'] + SELECT + subq_3.metric_time__martian_day + , subq_3.visits + FROM ( + -- Pass Only Elements: ['visits', 'metric_time__day'] + -- Join to Custom Granularity Dataset + SELECT + subq_1.metric_time__day AS metric_time__day + , subq_1.visits AS visits + , subq_2.martian_day AS metric_time__martian_day + FROM ( + -- Metric Time Dimension 'ds' + SELECT + subq_0.ds__day + , subq_0.ds__week + , subq_0.ds__month + , subq_0.ds__quarter + , subq_0.ds__year + , subq_0.ds__extract_year + , subq_0.ds__extract_quarter + , subq_0.ds__extract_month + , subq_0.ds__extract_day + , subq_0.ds__extract_dow + , subq_0.ds__extract_doy + , subq_0.visit__ds__day + , subq_0.visit__ds__week + , subq_0.visit__ds__month + , subq_0.visit__ds__quarter + , subq_0.visit__ds__year + , subq_0.visit__ds__extract_year + , subq_0.visit__ds__extract_quarter + , subq_0.visit__ds__extract_month + , subq_0.visit__ds__extract_day + , subq_0.visit__ds__extract_dow + , subq_0.visit__ds__extract_doy + , subq_0.ds__day AS metric_time__day + , subq_0.ds__week AS metric_time__week + , subq_0.ds__month AS metric_time__month + , subq_0.ds__quarter AS metric_time__quarter + , subq_0.ds__year AS metric_time__year + , subq_0.ds__extract_year AS metric_time__extract_year + , subq_0.ds__extract_quarter AS metric_time__extract_quarter + , subq_0.ds__extract_month AS metric_time__extract_month + , subq_0.ds__extract_day AS metric_time__extract_day + , subq_0.ds__extract_dow AS metric_time__extract_dow + , subq_0.ds__extract_doy AS metric_time__extract_doy + , subq_0.user + , subq_0.session + , subq_0.visit__user + , subq_0.visit__session + , subq_0.referrer_id + , subq_0.visit__referrer_id + , subq_0.visits + , subq_0.visitors + FROM ( + -- Read Elements From Semantic Model 'visits_source' + SELECT + 1 AS visits + , visits_source_src_28000.user_id AS visitors + , DATE_TRUNC('day', visits_source_src_28000.ds) AS ds__day + , DATE_TRUNC('week', visits_source_src_28000.ds) AS ds__week + , DATE_TRUNC('month', visits_source_src_28000.ds) AS ds__month + , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS ds__quarter + , DATE_TRUNC('year', visits_source_src_28000.ds) AS ds__year + , EXTRACT(year FROM visits_source_src_28000.ds) AS ds__extract_year + , EXTRACT(quarter FROM visits_source_src_28000.ds) AS ds__extract_quarter + , EXTRACT(month FROM visits_source_src_28000.ds) AS ds__extract_month + , EXTRACT(day FROM visits_source_src_28000.ds) AS ds__extract_day + , EXTRACT(isodow FROM visits_source_src_28000.ds) AS ds__extract_dow + , EXTRACT(doy FROM visits_source_src_28000.ds) AS ds__extract_doy + , visits_source_src_28000.referrer_id + , DATE_TRUNC('day', visits_source_src_28000.ds) AS visit__ds__day + , DATE_TRUNC('week', visits_source_src_28000.ds) AS visit__ds__week + , DATE_TRUNC('month', visits_source_src_28000.ds) AS visit__ds__month + , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS visit__ds__quarter + , DATE_TRUNC('year', visits_source_src_28000.ds) AS visit__ds__year + , EXTRACT(year FROM visits_source_src_28000.ds) AS visit__ds__extract_year + , EXTRACT(quarter FROM visits_source_src_28000.ds) AS visit__ds__extract_quarter + , EXTRACT(month FROM visits_source_src_28000.ds) AS visit__ds__extract_month + , EXTRACT(day FROM visits_source_src_28000.ds) AS visit__ds__extract_day + , EXTRACT(isodow FROM visits_source_src_28000.ds) AS visit__ds__extract_dow + , EXTRACT(doy FROM visits_source_src_28000.ds) AS visit__ds__extract_doy + , visits_source_src_28000.referrer_id AS visit__referrer_id + , visits_source_src_28000.user_id AS user + , visits_source_src_28000.session_id AS session + , visits_source_src_28000.user_id AS visit__user + , visits_source_src_28000.session_id AS visit__session + FROM ***************************.fct_visits visits_source_src_28000 + ) subq_0 + ) subq_1 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_2 + ON + subq_1.metric_time__day = subq_2.ds + ) subq_3 + ) subq_4 + GROUP BY + subq_4.metric_time__martian_day + ) subq_5 + FULL OUTER JOIN ( + -- Aggregate Measures + SELECT + subq_16.metric_time__martian_day + , SUM(subq_16.buys) AS buys + FROM ( + -- Pass Only Elements: ['buys', 'metric_time__martian_day'] + SELECT + subq_15.metric_time__martian_day + , subq_15.buys + FROM ( + -- Pass Only Elements: ['buys', 'metric_time__day'] + -- Join to Custom Granularity Dataset + SELECT + subq_13.metric_time__day AS metric_time__day + , subq_13.buys AS buys + , subq_14.martian_day AS metric_time__martian_day + FROM ( + -- Find conversions for user within the range of 7 day + SELECT + subq_12.ds__day + , subq_12.metric_time__day + , subq_12.user + , subq_12.buys + , subq_12.visits + FROM ( + -- Dedupe the fanout with mf_internal_uuid in the conversion data set + SELECT DISTINCT + FIRST_VALUE(subq_8.visits) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS visits + , FIRST_VALUE(subq_8.ds__day) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS ds__day + , FIRST_VALUE(subq_8.metric_time__day) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS metric_time__day + , FIRST_VALUE(subq_8.user) OVER ( + PARTITION BY + subq_11.user + , subq_11.ds__day + , subq_11.mf_internal_uuid + ORDER BY subq_8.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS user + , subq_11.mf_internal_uuid AS mf_internal_uuid + , subq_11.buys AS buys + FROM ( + -- Pass Only Elements: ['visits', 'ds__day', 'metric_time__day', 'user'] + SELECT + subq_7.ds__day + , subq_7.metric_time__day + , subq_7.user + , subq_7.visits + FROM ( + -- Metric Time Dimension 'ds' + SELECT + subq_6.ds__day + , subq_6.ds__week + , subq_6.ds__month + , subq_6.ds__quarter + , subq_6.ds__year + , subq_6.ds__extract_year + , subq_6.ds__extract_quarter + , subq_6.ds__extract_month + , subq_6.ds__extract_day + , subq_6.ds__extract_dow + , subq_6.ds__extract_doy + , subq_6.visit__ds__day + , subq_6.visit__ds__week + , subq_6.visit__ds__month + , subq_6.visit__ds__quarter + , subq_6.visit__ds__year + , subq_6.visit__ds__extract_year + , subq_6.visit__ds__extract_quarter + , subq_6.visit__ds__extract_month + , subq_6.visit__ds__extract_day + , subq_6.visit__ds__extract_dow + , subq_6.visit__ds__extract_doy + , subq_6.ds__day AS metric_time__day + , subq_6.ds__week AS metric_time__week + , subq_6.ds__month AS metric_time__month + , subq_6.ds__quarter AS metric_time__quarter + , subq_6.ds__year AS metric_time__year + , subq_6.ds__extract_year AS metric_time__extract_year + , subq_6.ds__extract_quarter AS metric_time__extract_quarter + , subq_6.ds__extract_month AS metric_time__extract_month + , subq_6.ds__extract_day AS metric_time__extract_day + , subq_6.ds__extract_dow AS metric_time__extract_dow + , subq_6.ds__extract_doy AS metric_time__extract_doy + , subq_6.user + , subq_6.session + , subq_6.visit__user + , subq_6.visit__session + , subq_6.referrer_id + , subq_6.visit__referrer_id + , subq_6.visits + , subq_6.visitors + FROM ( + -- Read Elements From Semantic Model 'visits_source' + SELECT + 1 AS visits + , visits_source_src_28000.user_id AS visitors + , DATE_TRUNC('day', visits_source_src_28000.ds) AS ds__day + , DATE_TRUNC('week', visits_source_src_28000.ds) AS ds__week + , DATE_TRUNC('month', visits_source_src_28000.ds) AS ds__month + , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS ds__quarter + , DATE_TRUNC('year', visits_source_src_28000.ds) AS ds__year + , EXTRACT(year FROM visits_source_src_28000.ds) AS ds__extract_year + , EXTRACT(quarter FROM visits_source_src_28000.ds) AS ds__extract_quarter + , EXTRACT(month FROM visits_source_src_28000.ds) AS ds__extract_month + , EXTRACT(day FROM visits_source_src_28000.ds) AS ds__extract_day + , EXTRACT(isodow FROM visits_source_src_28000.ds) AS ds__extract_dow + , EXTRACT(doy FROM visits_source_src_28000.ds) AS ds__extract_doy + , visits_source_src_28000.referrer_id + , DATE_TRUNC('day', visits_source_src_28000.ds) AS visit__ds__day + , DATE_TRUNC('week', visits_source_src_28000.ds) AS visit__ds__week + , DATE_TRUNC('month', visits_source_src_28000.ds) AS visit__ds__month + , DATE_TRUNC('quarter', visits_source_src_28000.ds) AS visit__ds__quarter + , DATE_TRUNC('year', visits_source_src_28000.ds) AS visit__ds__year + , EXTRACT(year FROM visits_source_src_28000.ds) AS visit__ds__extract_year + , EXTRACT(quarter FROM visits_source_src_28000.ds) AS visit__ds__extract_quarter + , EXTRACT(month FROM visits_source_src_28000.ds) AS visit__ds__extract_month + , EXTRACT(day FROM visits_source_src_28000.ds) AS visit__ds__extract_day + , EXTRACT(isodow FROM visits_source_src_28000.ds) AS visit__ds__extract_dow + , EXTRACT(doy FROM visits_source_src_28000.ds) AS visit__ds__extract_doy + , visits_source_src_28000.referrer_id AS visit__referrer_id + , visits_source_src_28000.user_id AS user + , visits_source_src_28000.session_id AS session + , visits_source_src_28000.user_id AS visit__user + , visits_source_src_28000.session_id AS visit__session + FROM ***************************.fct_visits visits_source_src_28000 + ) subq_6 + ) subq_7 + ) subq_8 + INNER JOIN ( + -- Add column with generated UUID + SELECT + subq_10.ds__day + , subq_10.ds__week + , subq_10.ds__month + , subq_10.ds__quarter + , subq_10.ds__year + , subq_10.ds__extract_year + , subq_10.ds__extract_quarter + , subq_10.ds__extract_month + , subq_10.ds__extract_day + , subq_10.ds__extract_dow + , subq_10.ds__extract_doy + , subq_10.buy__ds__day + , subq_10.buy__ds__week + , subq_10.buy__ds__month + , subq_10.buy__ds__quarter + , subq_10.buy__ds__year + , subq_10.buy__ds__extract_year + , subq_10.buy__ds__extract_quarter + , subq_10.buy__ds__extract_month + , subq_10.buy__ds__extract_day + , subq_10.buy__ds__extract_dow + , subq_10.buy__ds__extract_doy + , subq_10.metric_time__day + , subq_10.metric_time__week + , subq_10.metric_time__month + , subq_10.metric_time__quarter + , subq_10.metric_time__year + , subq_10.metric_time__extract_year + , subq_10.metric_time__extract_quarter + , subq_10.metric_time__extract_month + , subq_10.metric_time__extract_day + , subq_10.metric_time__extract_dow + , subq_10.metric_time__extract_doy + , subq_10.user + , subq_10.session_id + , subq_10.buy__user + , subq_10.buy__session_id + , subq_10.buys + , subq_10.buyers + , GEN_RANDOM_UUID() AS mf_internal_uuid + FROM ( + -- Metric Time Dimension 'ds' + SELECT + subq_9.ds__day + , subq_9.ds__week + , subq_9.ds__month + , subq_9.ds__quarter + , subq_9.ds__year + , subq_9.ds__extract_year + , subq_9.ds__extract_quarter + , subq_9.ds__extract_month + , subq_9.ds__extract_day + , subq_9.ds__extract_dow + , subq_9.ds__extract_doy + , subq_9.buy__ds__day + , subq_9.buy__ds__week + , subq_9.buy__ds__month + , subq_9.buy__ds__quarter + , subq_9.buy__ds__year + , subq_9.buy__ds__extract_year + , subq_9.buy__ds__extract_quarter + , subq_9.buy__ds__extract_month + , subq_9.buy__ds__extract_day + , subq_9.buy__ds__extract_dow + , subq_9.buy__ds__extract_doy + , subq_9.ds__day AS metric_time__day + , subq_9.ds__week AS metric_time__week + , subq_9.ds__month AS metric_time__month + , subq_9.ds__quarter AS metric_time__quarter + , subq_9.ds__year AS metric_time__year + , subq_9.ds__extract_year AS metric_time__extract_year + , subq_9.ds__extract_quarter AS metric_time__extract_quarter + , subq_9.ds__extract_month AS metric_time__extract_month + , subq_9.ds__extract_day AS metric_time__extract_day + , subq_9.ds__extract_dow AS metric_time__extract_dow + , subq_9.ds__extract_doy AS metric_time__extract_doy + , subq_9.user + , subq_9.session_id + , subq_9.buy__user + , subq_9.buy__session_id + , subq_9.buys + , subq_9.buyers + FROM ( + -- Read Elements From Semantic Model 'buys_source' + SELECT + 1 AS buys + , buys_source_src_28000.user_id AS buyers + , DATE_TRUNC('day', buys_source_src_28000.ds) AS ds__day + , DATE_TRUNC('week', buys_source_src_28000.ds) AS ds__week + , DATE_TRUNC('month', buys_source_src_28000.ds) AS ds__month + , DATE_TRUNC('quarter', buys_source_src_28000.ds) AS ds__quarter + , DATE_TRUNC('year', buys_source_src_28000.ds) AS ds__year + , EXTRACT(year FROM buys_source_src_28000.ds) AS ds__extract_year + , EXTRACT(quarter FROM buys_source_src_28000.ds) AS ds__extract_quarter + , EXTRACT(month FROM buys_source_src_28000.ds) AS ds__extract_month + , EXTRACT(day FROM buys_source_src_28000.ds) AS ds__extract_day + , EXTRACT(isodow FROM buys_source_src_28000.ds) AS ds__extract_dow + , EXTRACT(doy FROM buys_source_src_28000.ds) AS ds__extract_doy + , DATE_TRUNC('day', buys_source_src_28000.ds) AS buy__ds__day + , DATE_TRUNC('week', buys_source_src_28000.ds) AS buy__ds__week + , DATE_TRUNC('month', buys_source_src_28000.ds) AS buy__ds__month + , DATE_TRUNC('quarter', buys_source_src_28000.ds) AS buy__ds__quarter + , DATE_TRUNC('year', buys_source_src_28000.ds) AS buy__ds__year + , EXTRACT(year FROM buys_source_src_28000.ds) AS buy__ds__extract_year + , EXTRACT(quarter FROM buys_source_src_28000.ds) AS buy__ds__extract_quarter + , EXTRACT(month FROM buys_source_src_28000.ds) AS buy__ds__extract_month + , EXTRACT(day FROM buys_source_src_28000.ds) AS buy__ds__extract_day + , EXTRACT(isodow FROM buys_source_src_28000.ds) AS buy__ds__extract_dow + , EXTRACT(doy FROM buys_source_src_28000.ds) AS buy__ds__extract_doy + , buys_source_src_28000.user_id AS user + , buys_source_src_28000.session_id + , buys_source_src_28000.user_id AS buy__user + , buys_source_src_28000.session_id AS buy__session_id + FROM ***************************.fct_buys buys_source_src_28000 + ) subq_9 + ) subq_10 + ) subq_11 + ON + ( + subq_8.user = subq_11.user + ) AND ( + ( + subq_8.ds__day <= subq_11.ds__day + ) AND ( + subq_8.ds__day > subq_11.ds__day - INTERVAL 7 day + ) + ) + ) subq_12 + ) subq_13 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_14 + ON + subq_13.metric_time__day = subq_14.ds + ) subq_15 + ) subq_16 + GROUP BY + subq_16.metric_time__martian_day + ) subq_17 + ON + subq_5.metric_time__martian_day = subq_17.metric_time__martian_day + GROUP BY + COALESCE(subq_5.metric_time__martian_day, subq_17.metric_time__martian_day) +) subq_18 diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql new file mode 100644 index 0000000000..ebdabae734 --- /dev/null +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_conversion_metric_with_custom_granularity__plan0_optimized.sql @@ -0,0 +1,123 @@ +-- Compute Metrics via Expressions +SELECT + metric_time__martian_day + , CAST(buys AS DOUBLE) / CAST(NULLIF(visits, 0) AS DOUBLE) AS visit_buy_conversion_rate_7days +FROM ( + -- Combine Aggregated Outputs + SELECT + COALESCE(subq_24.metric_time__martian_day, subq_36.metric_time__martian_day) AS metric_time__martian_day + , MAX(subq_24.visits) AS visits + , MAX(subq_36.buys) AS buys + FROM ( + -- Pass Only Elements: ['visits', 'metric_time__day'] + -- Join to Custom Granularity Dataset + -- Pass Only Elements: ['visits', 'metric_time__martian_day'] + -- Aggregate Measures + SELECT + subq_21.martian_day AS metric_time__martian_day + , SUM(subq_20.visits) AS visits + FROM ( + -- Read Elements From Semantic Model 'visits_source' + -- Metric Time Dimension 'ds' + SELECT + DATE_TRUNC('day', ds) AS metric_time__day + , 1 AS visits + FROM ***************************.fct_visits visits_source_src_28000 + ) subq_20 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_21 + ON + subq_20.metric_time__day = subq_21.ds + GROUP BY + subq_21.martian_day + ) subq_24 + FULL OUTER JOIN ( + -- Pass Only Elements: ['buys', 'metric_time__day'] + -- Join to Custom Granularity Dataset + -- Pass Only Elements: ['buys', 'metric_time__martian_day'] + -- Aggregate Measures + SELECT + subq_33.martian_day AS metric_time__martian_day + , SUM(subq_31.buys) AS buys + FROM ( + -- Dedupe the fanout with mf_internal_uuid in the conversion data set + SELECT DISTINCT + FIRST_VALUE(subq_27.visits) OVER ( + PARTITION BY + subq_30.user + , subq_30.ds__day + , subq_30.mf_internal_uuid + ORDER BY subq_27.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS visits + , FIRST_VALUE(subq_27.ds__day) OVER ( + PARTITION BY + subq_30.user + , subq_30.ds__day + , subq_30.mf_internal_uuid + ORDER BY subq_27.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS ds__day + , FIRST_VALUE(subq_27.metric_time__day) OVER ( + PARTITION BY + subq_30.user + , subq_30.ds__day + , subq_30.mf_internal_uuid + ORDER BY subq_27.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS metric_time__day + , FIRST_VALUE(subq_27.user) OVER ( + PARTITION BY + subq_30.user + , subq_30.ds__day + , subq_30.mf_internal_uuid + ORDER BY subq_27.ds__day DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ) AS user + , subq_30.mf_internal_uuid AS mf_internal_uuid + , subq_30.buys AS buys + FROM ( + -- Read Elements From Semantic Model 'visits_source' + -- Metric Time Dimension 'ds' + -- Pass Only Elements: ['visits', 'ds__day', 'metric_time__day', 'user'] + SELECT + DATE_TRUNC('day', ds) AS ds__day + , DATE_TRUNC('day', ds) AS metric_time__day + , user_id AS user + , 1 AS visits + FROM ***************************.fct_visits visits_source_src_28000 + ) subq_27 + INNER JOIN ( + -- Read Elements From Semantic Model 'buys_source' + -- Metric Time Dimension 'ds' + -- Add column with generated UUID + SELECT + DATE_TRUNC('day', ds) AS ds__day + , user_id AS user + , 1 AS buys + , GEN_RANDOM_UUID() AS mf_internal_uuid + FROM ***************************.fct_buys buys_source_src_28000 + ) subq_30 + ON + ( + subq_27.user = subq_30.user + ) AND ( + ( + subq_27.ds__day <= subq_30.ds__day + ) AND ( + subq_27.ds__day > subq_30.ds__day - INTERVAL 7 day + ) + ) + ) subq_31 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_33 + ON + subq_31.metric_time__day = subq_33.ds + GROUP BY + subq_33.martian_day + ) subq_36 + ON + subq_24.metric_time__martian_day = subq_36.metric_time__martian_day + GROUP BY + COALESCE(subq_24.metric_time__martian_day, subq_36.metric_time__martian_day) +) subq_37 diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0.sql new file mode 100644 index 0000000000..30986ea53b --- /dev/null +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0.sql @@ -0,0 +1,239 @@ +-- Compute Metrics via Expressions +SELECT + subq_8.metric_time__martian_day + , subq_8.bookings AS bookings_join_to_time_spine +FROM ( + -- Join to Time Spine Dataset + SELECT + subq_6.metric_time__martian_day AS metric_time__martian_day + , subq_5.bookings AS bookings + FROM ( + -- Time Spine + SELECT + subq_7.ds AS metric_time__martian_day + FROM ***************************.mf_time_spine subq_7 + ) subq_6 + LEFT OUTER JOIN ( + -- Aggregate Measures + SELECT + subq_4.metric_time__martian_day + , SUM(subq_4.bookings) AS bookings + FROM ( + -- Pass Only Elements: ['bookings', 'metric_time__martian_day'] + SELECT + subq_3.metric_time__martian_day + , subq_3.bookings + FROM ( + -- Pass Only Elements: ['bookings', 'metric_time__day'] + -- Join to Custom Granularity Dataset + SELECT + subq_1.metric_time__day AS metric_time__day + , subq_1.bookings AS bookings + , subq_2.martian_day AS metric_time__martian_day + FROM ( + -- Metric Time Dimension 'ds' + SELECT + subq_0.ds__day + , subq_0.ds__week + , subq_0.ds__month + , subq_0.ds__quarter + , subq_0.ds__year + , subq_0.ds__extract_year + , subq_0.ds__extract_quarter + , subq_0.ds__extract_month + , subq_0.ds__extract_day + , subq_0.ds__extract_dow + , subq_0.ds__extract_doy + , subq_0.ds_partitioned__day + , subq_0.ds_partitioned__week + , subq_0.ds_partitioned__month + , subq_0.ds_partitioned__quarter + , subq_0.ds_partitioned__year + , subq_0.ds_partitioned__extract_year + , subq_0.ds_partitioned__extract_quarter + , subq_0.ds_partitioned__extract_month + , subq_0.ds_partitioned__extract_day + , subq_0.ds_partitioned__extract_dow + , subq_0.ds_partitioned__extract_doy + , subq_0.paid_at__day + , subq_0.paid_at__week + , subq_0.paid_at__month + , subq_0.paid_at__quarter + , subq_0.paid_at__year + , subq_0.paid_at__extract_year + , subq_0.paid_at__extract_quarter + , subq_0.paid_at__extract_month + , subq_0.paid_at__extract_day + , subq_0.paid_at__extract_dow + , subq_0.paid_at__extract_doy + , subq_0.booking__ds__day + , subq_0.booking__ds__week + , subq_0.booking__ds__month + , subq_0.booking__ds__quarter + , subq_0.booking__ds__year + , subq_0.booking__ds__extract_year + , subq_0.booking__ds__extract_quarter + , subq_0.booking__ds__extract_month + , subq_0.booking__ds__extract_day + , subq_0.booking__ds__extract_dow + , subq_0.booking__ds__extract_doy + , subq_0.booking__ds_partitioned__day + , subq_0.booking__ds_partitioned__week + , subq_0.booking__ds_partitioned__month + , subq_0.booking__ds_partitioned__quarter + , subq_0.booking__ds_partitioned__year + , subq_0.booking__ds_partitioned__extract_year + , subq_0.booking__ds_partitioned__extract_quarter + , subq_0.booking__ds_partitioned__extract_month + , subq_0.booking__ds_partitioned__extract_day + , subq_0.booking__ds_partitioned__extract_dow + , subq_0.booking__ds_partitioned__extract_doy + , subq_0.booking__paid_at__day + , subq_0.booking__paid_at__week + , subq_0.booking__paid_at__month + , subq_0.booking__paid_at__quarter + , subq_0.booking__paid_at__year + , subq_0.booking__paid_at__extract_year + , subq_0.booking__paid_at__extract_quarter + , subq_0.booking__paid_at__extract_month + , subq_0.booking__paid_at__extract_day + , subq_0.booking__paid_at__extract_dow + , subq_0.booking__paid_at__extract_doy + , subq_0.ds__day AS metric_time__day + , subq_0.ds__week AS metric_time__week + , subq_0.ds__month AS metric_time__month + , subq_0.ds__quarter AS metric_time__quarter + , subq_0.ds__year AS metric_time__year + , subq_0.ds__extract_year AS metric_time__extract_year + , subq_0.ds__extract_quarter AS metric_time__extract_quarter + , subq_0.ds__extract_month AS metric_time__extract_month + , subq_0.ds__extract_day AS metric_time__extract_day + , subq_0.ds__extract_dow AS metric_time__extract_dow + , subq_0.ds__extract_doy AS metric_time__extract_doy + , subq_0.listing + , subq_0.guest + , subq_0.host + , subq_0.booking__listing + , subq_0.booking__guest + , subq_0.booking__host + , subq_0.is_instant + , subq_0.booking__is_instant + , subq_0.bookings + , subq_0.instant_bookings + , subq_0.booking_value + , subq_0.max_booking_value + , subq_0.min_booking_value + , subq_0.bookers + , subq_0.average_booking_value + , subq_0.referred_bookings + , subq_0.median_booking_value + , subq_0.booking_value_p99 + , subq_0.discrete_booking_value_p99 + , subq_0.approximate_continuous_booking_value_p99 + , subq_0.approximate_discrete_booking_value_p99 + FROM ( + -- Read Elements From Semantic Model 'bookings_source' + SELECT + 1 AS bookings + , CASE WHEN is_instant THEN 1 ELSE 0 END AS instant_bookings + , bookings_source_src_28000.booking_value + , bookings_source_src_28000.booking_value AS max_booking_value + , bookings_source_src_28000.booking_value AS min_booking_value + , bookings_source_src_28000.guest_id AS bookers + , bookings_source_src_28000.booking_value AS average_booking_value + , bookings_source_src_28000.booking_value AS booking_payments + , CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings + , bookings_source_src_28000.booking_value AS median_booking_value + , bookings_source_src_28000.booking_value AS booking_value_p99 + , bookings_source_src_28000.booking_value AS discrete_booking_value_p99 + , bookings_source_src_28000.booking_value AS approximate_continuous_booking_value_p99 + , bookings_source_src_28000.booking_value AS approximate_discrete_booking_value_p99 + , bookings_source_src_28000.is_instant + , DATE_TRUNC('day', bookings_source_src_28000.ds) AS ds__day + , DATE_TRUNC('week', bookings_source_src_28000.ds) AS ds__week + , DATE_TRUNC('month', bookings_source_src_28000.ds) AS ds__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds) AS ds__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds) AS ds__year + , EXTRACT(year FROM bookings_source_src_28000.ds) AS ds__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds) AS ds__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds) AS ds__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds) AS ds__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds) AS ds__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds) AS ds__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__day + , DATE_TRUNC('week', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__week + , DATE_TRUNC('month', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__year + , EXTRACT(year FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.paid_at) AS paid_at__day + , DATE_TRUNC('week', bookings_source_src_28000.paid_at) AS paid_at__week + , DATE_TRUNC('month', bookings_source_src_28000.paid_at) AS paid_at__month + , DATE_TRUNC('quarter', bookings_source_src_28000.paid_at) AS paid_at__quarter + , DATE_TRUNC('year', bookings_source_src_28000.paid_at) AS paid_at__year + , EXTRACT(year FROM bookings_source_src_28000.paid_at) AS paid_at__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.paid_at) AS paid_at__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.paid_at) AS paid_at__extract_month + , EXTRACT(day FROM bookings_source_src_28000.paid_at) AS paid_at__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.paid_at) AS paid_at__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.paid_at) AS paid_at__extract_doy + , bookings_source_src_28000.is_instant AS booking__is_instant + , DATE_TRUNC('day', bookings_source_src_28000.ds) AS booking__ds__day + , DATE_TRUNC('week', bookings_source_src_28000.ds) AS booking__ds__week + , DATE_TRUNC('month', bookings_source_src_28000.ds) AS booking__ds__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds) AS booking__ds__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds) AS booking__ds__year + , EXTRACT(year FROM bookings_source_src_28000.ds) AS booking__ds__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds) AS booking__ds__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds) AS booking__ds__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds) AS booking__ds__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds) AS booking__ds__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds) AS booking__ds__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__day + , DATE_TRUNC('week', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__week + , DATE_TRUNC('month', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__year + , EXTRACT(year FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.paid_at) AS booking__paid_at__day + , DATE_TRUNC('week', bookings_source_src_28000.paid_at) AS booking__paid_at__week + , DATE_TRUNC('month', bookings_source_src_28000.paid_at) AS booking__paid_at__month + , DATE_TRUNC('quarter', bookings_source_src_28000.paid_at) AS booking__paid_at__quarter + , DATE_TRUNC('year', bookings_source_src_28000.paid_at) AS booking__paid_at__year + , EXTRACT(year FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_month + , EXTRACT(day FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_doy + , bookings_source_src_28000.listing_id AS listing + , bookings_source_src_28000.guest_id AS guest + , bookings_source_src_28000.host_id AS host + , bookings_source_src_28000.listing_id AS booking__listing + , bookings_source_src_28000.guest_id AS booking__guest + , bookings_source_src_28000.host_id AS booking__host + FROM ***************************.fct_bookings bookings_source_src_28000 + ) subq_0 + ) subq_1 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_2 + ON + subq_1.metric_time__day = subq_2.ds + ) subq_3 + ) subq_4 + GROUP BY + subq_4.metric_time__martian_day + ) subq_5 + ON + subq_6.metric_time__day = subq_5.metric_time__day +) subq_8 diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0_optimized.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0_optimized.sql new file mode 100644 index 0000000000..677493d7f8 --- /dev/null +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity__plan0_optimized.sql @@ -0,0 +1,31 @@ +-- Join to Time Spine Dataset +-- Compute Metrics via Expressions +SELECT + subq_16.ds AS metric_time__martian_day + , subq_14.bookings AS bookings_join_to_time_spine +FROM ***************************.mf_time_spine subq_16 +LEFT OUTER JOIN ( + -- Pass Only Elements: ['bookings', 'metric_time__day'] + -- Join to Custom Granularity Dataset + -- Pass Only Elements: ['bookings', 'metric_time__martian_day'] + -- Aggregate Measures + SELECT + subq_11.martian_day AS metric_time__martian_day + , SUM(subq_10.bookings) AS bookings + FROM ( + -- Read Elements From Semantic Model 'bookings_source' + -- Metric Time Dimension 'ds' + SELECT + DATE_TRUNC('day', ds) AS metric_time__day + , 1 AS bookings + FROM ***************************.fct_bookings bookings_source_src_28000 + ) subq_10 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_11 + ON + subq_10.metric_time__day = subq_11.ds + GROUP BY + subq_11.martian_day +) subq_14 +ON + subq_15.metric_time__day = subq_14.metric_time__day