-
Notifications
You must be signed in to change notification settings - Fork 97
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Write various tests for metric queries with custom granularity
- Loading branch information
1 parent
889ba84
commit 5b86290
Showing
11 changed files
with
1,718 additions
and
0 deletions.
There are no files selected for viewing
170 changes: 170 additions & 0 deletions
170
tests_metricflow/query_rendering/test_custom_granularity.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| """Tests metric query rendering for granularity and date part operations. | ||
| This module runs query requests for various granularity/date part options and compares | ||
| the rendered output against snapshot files. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import datetime as dt | ||
|
|
||
| import pytest | ||
| from _pytest.fixtures import FixtureRequest | ||
| from dbt_semantic_interfaces.references import EntityReference | ||
| from dbt_semantic_interfaces.type_enums.date_part import DatePart | ||
| from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity | ||
| from metricflow_semantics.filters.time_constraint import TimeRangeConstraint | ||
| from metricflow_semantics.specs.metric_spec import MetricSpec | ||
| from metricflow_semantics.specs.query_spec import MetricFlowQuerySpec | ||
| from metricflow_semantics.specs.time_dimension_spec import TimeDimensionSpec | ||
| from metricflow_semantics.test_helpers.config_helpers import MetricFlowTestConfiguration | ||
| from metricflow_semantics.time.granularity import ExpandedTimeGranularity | ||
| from metricflow_semantics.test_helpers.metric_time_dimension import MTD_SPEC_DAY | ||
|
|
||
| from metricflow.dataflow.builder.dataflow_plan_builder import DataflowPlanBuilder | ||
| from metricflow.dataset.dataset_classes import DataSet | ||
| from metricflow.plan_conversion.dataflow_to_sql import DataflowToSqlQueryPlanConverter | ||
| from metricflow.protocols.sql_client import SqlClient | ||
| from tests_metricflow.query_rendering.compare_rendered_query import render_and_check | ||
|
|
||
|
|
||
| metric_time_with_custom_grain = TimeDimensionSpec( | ||
| "metric_time", | ||
| entity_links=(), | ||
| time_granularity=ExpandedTimeGranularity(name="martian_day", base_granularity=TimeGranularity.DAY), | ||
| ) | ||
| normal_time_dim_with_custom_grain1 = TimeDimensionSpec( | ||
| element_name="ds", | ||
| time_granularity=ExpandedTimeGranularity(name="martian_day", base_granularity=TimeGranularity.DAY), | ||
| entity_links=(EntityReference("booking"),), | ||
| ) | ||
| normal_time_dim_with_custom_grain2 = TimeDimensionSpec( | ||
| element_name="bio_added_ts", | ||
| time_granularity=ExpandedTimeGranularity(name="martian_day", base_granularity=TimeGranularity.DAY), | ||
| entity_links=(EntityReference("user"),), | ||
| ) | ||
|
|
||
|
|
||
| # TODO: subqueries in this test should be collapsed. Update optimizer | ||
| @pytest.mark.sql_engine_snapshot | ||
| def test_simple_metric_with_custom_granularity( # noqa: D103 | ||
| request: FixtureRequest, | ||
| mf_test_configuration: MetricFlowTestConfiguration, | ||
| dataflow_plan_builder: DataflowPlanBuilder, | ||
| dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, | ||
| sql_client: SqlClient, | ||
| ) -> None: | ||
| query_spec = MetricFlowQuerySpec( | ||
| metric_specs=(MetricSpec("bookings"),), | ||
| time_dimension_specs=(normal_time_dim_with_custom_grain1,), | ||
| ) | ||
|
|
||
| render_and_check( | ||
| request=request, | ||
| mf_test_configuration=mf_test_configuration, | ||
| dataflow_to_sql_converter=dataflow_to_sql_converter, | ||
| sql_client=sql_client, | ||
| dataflow_plan_builder=dataflow_plan_builder, | ||
| query_spec=query_spec, | ||
| ) | ||
|
|
||
|
|
||
| @pytest.mark.sql_engine_snapshot | ||
| def test_cumulative_metric_with_custom_granularity( # noqa: D103 | ||
| request: FixtureRequest, | ||
| mf_test_configuration: MetricFlowTestConfiguration, | ||
| dataflow_plan_builder: DataflowPlanBuilder, | ||
| dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, | ||
| sql_client: SqlClient, | ||
| ) -> None: | ||
| query_spec = MetricFlowQuerySpec( | ||
| metric_specs=(MetricSpec("trailing_2_months_revenue"),), | ||
| time_dimension_specs=(metric_time_with_custom_grain,), | ||
| ) | ||
|
|
||
| render_and_check( | ||
| request=request, | ||
| mf_test_configuration=mf_test_configuration, | ||
| dataflow_to_sql_converter=dataflow_to_sql_converter, | ||
| sql_client=sql_client, | ||
| dataflow_plan_builder=dataflow_plan_builder, | ||
| query_spec=query_spec, | ||
| ) | ||
|
|
||
|
|
||
| @pytest.mark.sql_engine_snapshot | ||
| def test_derived_metric_with_custom_granularity( # noqa: D103 | ||
| request: FixtureRequest, | ||
| mf_test_configuration: MetricFlowTestConfiguration, | ||
| dataflow_plan_builder: DataflowPlanBuilder, | ||
| dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, | ||
| sql_client: SqlClient, | ||
| ) -> None: | ||
| query_spec = MetricFlowQuerySpec( | ||
| metric_specs=(MetricSpec("booking_fees_per_booker"),), | ||
| time_dimension_specs=(normal_time_dim_with_custom_grain1,), | ||
| ) | ||
|
|
||
| render_and_check( | ||
| request=request, | ||
| mf_test_configuration=mf_test_configuration, | ||
| dataflow_to_sql_converter=dataflow_to_sql_converter, | ||
| sql_client=sql_client, | ||
| dataflow_plan_builder=dataflow_plan_builder, | ||
| query_spec=query_spec, | ||
| ) | ||
|
|
||
|
|
||
| # TODO: subqueries in this test should be collapsed. Update optimizer | ||
| @pytest.mark.sql_engine_snapshot | ||
| def test_multiple_metrics_with_custom_granularity( # noqa: D103 | ||
| request: FixtureRequest, | ||
| mf_test_configuration: MetricFlowTestConfiguration, | ||
| dataflow_plan_builder: DataflowPlanBuilder, | ||
| dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, | ||
| sql_client: SqlClient, | ||
| ) -> None: | ||
| query_spec = MetricFlowQuerySpec( | ||
| metric_specs=(MetricSpec("bookings"), MetricSpec("listings")), | ||
| time_dimension_specs=(metric_time_with_custom_grain,), | ||
| ) | ||
|
|
||
| render_and_check( | ||
| request=request, | ||
| mf_test_configuration=mf_test_configuration, | ||
| dataflow_to_sql_converter=dataflow_to_sql_converter, | ||
| sql_client=sql_client, | ||
| dataflow_plan_builder=dataflow_plan_builder, | ||
| query_spec=query_spec, | ||
| ) | ||
|
|
||
|
|
||
| # TODO: subqueries in this test should be collapsed. Update optimizer | ||
| @pytest.mark.sql_engine_snapshot | ||
| def test_metric_custom_granularity_joined_to_non_default_grain( # noqa: D103 | ||
| request: FixtureRequest, | ||
| mf_test_configuration: MetricFlowTestConfiguration, | ||
| dataflow_plan_builder: DataflowPlanBuilder, | ||
| dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, | ||
| sql_client: SqlClient, | ||
| ) -> None: | ||
| query_spec = MetricFlowQuerySpec( | ||
| metric_specs=(MetricSpec("listings"),), | ||
| time_dimension_specs=( | ||
| metric_time_with_custom_grain, | ||
| TimeDimensionSpec( | ||
| element_name="ds", | ||
| time_granularity=ExpandedTimeGranularity.from_time_granularity(TimeGranularity.MONTH), | ||
| entity_links=(EntityReference("listing"),), | ||
| ), | ||
| ), | ||
| ) | ||
|
|
||
| render_and_check( | ||
| request=request, | ||
| mf_test_configuration=mf_test_configuration, | ||
| dataflow_to_sql_converter=dataflow_to_sql_converter, | ||
| sql_client=sql_client, | ||
| dataflow_plan_builder=dataflow_plan_builder, | ||
| query_spec=query_spec, | ||
| ) |
164 changes: 164 additions & 0 deletions
164
...nularity.py/SqlQueryPlan/DuckDB/test_cumulative_metric_with_custom_granularity__plan0.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,164 @@ | ||
| -- Re-aggregate Metric via Group By | ||
| SELECT | ||
| subq_9.metric_time__martian_day | ||
| , subq_9.trailing_2_months_revenue | ||
| FROM ( | ||
| -- Window Function for Metric Re-aggregation | ||
| SELECT | ||
| subq_8.metric_time__martian_day | ||
| , AVG(subq_8.trailing_2_months_revenue) OVER (PARTITION BY subq_8.metric_time__martian_day) AS trailing_2_months_revenue | ||
| FROM ( | ||
| -- Compute Metrics via Expressions | ||
| SELECT | ||
| subq_7.metric_time__martian_day | ||
| , subq_7.metric_time__day | ||
| , subq_7.txn_revenue AS trailing_2_months_revenue | ||
| FROM ( | ||
| -- Aggregate Measures | ||
| SELECT | ||
| subq_6.metric_time__martian_day | ||
| , subq_6.metric_time__day | ||
| , SUM(subq_6.txn_revenue) AS txn_revenue | ||
| FROM ( | ||
| -- Join to Custom Granularity Dataset | ||
| -- Pass Only Elements: ['txn_revenue', 'metric_time__day', 'metric_time__day'] | ||
| SELECT | ||
| subq_4.metric_time__day AS metric_time__day | ||
| , subq_4.txn_revenue AS txn_revenue | ||
| , subq_5.martian_day AS metric_time__martian_day | ||
| FROM ( | ||
| -- Join Self Over Time Range | ||
| SELECT | ||
| subq_2.metric_time__day AS metric_time__day | ||
| , subq_1.ds__day AS ds__day | ||
| , subq_1.ds__week AS ds__week | ||
| , subq_1.ds__month AS ds__month | ||
| , subq_1.ds__quarter AS ds__quarter | ||
| , subq_1.ds__year AS ds__year | ||
| , subq_1.ds__extract_year AS ds__extract_year | ||
| , subq_1.ds__extract_quarter AS ds__extract_quarter | ||
| , subq_1.ds__extract_month AS ds__extract_month | ||
| , subq_1.ds__extract_day AS ds__extract_day | ||
| , subq_1.ds__extract_dow AS ds__extract_dow | ||
| , subq_1.ds__extract_doy AS ds__extract_doy | ||
| , subq_1.revenue_instance__ds__day AS revenue_instance__ds__day | ||
| , subq_1.revenue_instance__ds__week AS revenue_instance__ds__week | ||
| , subq_1.revenue_instance__ds__month AS revenue_instance__ds__month | ||
| , subq_1.revenue_instance__ds__quarter AS revenue_instance__ds__quarter | ||
| , subq_1.revenue_instance__ds__year AS revenue_instance__ds__year | ||
| , subq_1.revenue_instance__ds__extract_year AS revenue_instance__ds__extract_year | ||
| , subq_1.revenue_instance__ds__extract_quarter AS revenue_instance__ds__extract_quarter | ||
| , subq_1.revenue_instance__ds__extract_month AS revenue_instance__ds__extract_month | ||
| , subq_1.revenue_instance__ds__extract_day AS revenue_instance__ds__extract_day | ||
| , subq_1.revenue_instance__ds__extract_dow AS revenue_instance__ds__extract_dow | ||
| , subq_1.revenue_instance__ds__extract_doy AS revenue_instance__ds__extract_doy | ||
| , subq_1.metric_time__week AS metric_time__week | ||
| , subq_1.metric_time__month AS metric_time__month | ||
| , subq_1.metric_time__quarter AS metric_time__quarter | ||
| , subq_1.metric_time__year AS metric_time__year | ||
| , subq_1.metric_time__extract_year AS metric_time__extract_year | ||
| , subq_1.metric_time__extract_quarter AS metric_time__extract_quarter | ||
| , subq_1.metric_time__extract_month AS metric_time__extract_month | ||
| , subq_1.metric_time__extract_day AS metric_time__extract_day | ||
| , subq_1.metric_time__extract_dow AS metric_time__extract_dow | ||
| , subq_1.metric_time__extract_doy AS metric_time__extract_doy | ||
| , subq_1.user AS user | ||
| , subq_1.revenue_instance__user AS revenue_instance__user | ||
| , subq_1.txn_revenue AS txn_revenue | ||
| FROM ( | ||
| -- Time Spine | ||
| SELECT | ||
| subq_3.ds AS metric_time__day | ||
| FROM ***************************.mf_time_spine subq_3 | ||
| ) subq_2 | ||
| INNER JOIN ( | ||
| -- Metric Time Dimension 'ds' | ||
| SELECT | ||
| subq_0.ds__day | ||
| , subq_0.ds__week | ||
| , subq_0.ds__month | ||
| , subq_0.ds__quarter | ||
| , subq_0.ds__year | ||
| , subq_0.ds__extract_year | ||
| , subq_0.ds__extract_quarter | ||
| , subq_0.ds__extract_month | ||
| , subq_0.ds__extract_day | ||
| , subq_0.ds__extract_dow | ||
| , subq_0.ds__extract_doy | ||
| , subq_0.revenue_instance__ds__day | ||
| , subq_0.revenue_instance__ds__week | ||
| , subq_0.revenue_instance__ds__month | ||
| , subq_0.revenue_instance__ds__quarter | ||
| , subq_0.revenue_instance__ds__year | ||
| , subq_0.revenue_instance__ds__extract_year | ||
| , subq_0.revenue_instance__ds__extract_quarter | ||
| , subq_0.revenue_instance__ds__extract_month | ||
| , subq_0.revenue_instance__ds__extract_day | ||
| , subq_0.revenue_instance__ds__extract_dow | ||
| , subq_0.revenue_instance__ds__extract_doy | ||
| , subq_0.ds__day AS metric_time__day | ||
| , subq_0.ds__week AS metric_time__week | ||
| , subq_0.ds__month AS metric_time__month | ||
| , subq_0.ds__quarter AS metric_time__quarter | ||
| , subq_0.ds__year AS metric_time__year | ||
| , subq_0.ds__extract_year AS metric_time__extract_year | ||
| , subq_0.ds__extract_quarter AS metric_time__extract_quarter | ||
| , subq_0.ds__extract_month AS metric_time__extract_month | ||
| , subq_0.ds__extract_day AS metric_time__extract_day | ||
| , subq_0.ds__extract_dow AS metric_time__extract_dow | ||
| , subq_0.ds__extract_doy AS metric_time__extract_doy | ||
| , subq_0.user | ||
| , subq_0.revenue_instance__user | ||
| , subq_0.txn_revenue | ||
| FROM ( | ||
| -- Read Elements From Semantic Model 'revenue' | ||
| SELECT | ||
| revenue_src_28000.revenue AS txn_revenue | ||
| , DATE_TRUNC('day', revenue_src_28000.created_at) AS ds__day | ||
| , DATE_TRUNC('week', revenue_src_28000.created_at) AS ds__week | ||
| , DATE_TRUNC('month', revenue_src_28000.created_at) AS ds__month | ||
| , DATE_TRUNC('quarter', revenue_src_28000.created_at) AS ds__quarter | ||
| , DATE_TRUNC('year', revenue_src_28000.created_at) AS ds__year | ||
| , EXTRACT(year FROM revenue_src_28000.created_at) AS ds__extract_year | ||
| , EXTRACT(quarter FROM revenue_src_28000.created_at) AS ds__extract_quarter | ||
| , EXTRACT(month FROM revenue_src_28000.created_at) AS ds__extract_month | ||
| , EXTRACT(day FROM revenue_src_28000.created_at) AS ds__extract_day | ||
| , EXTRACT(isodow FROM revenue_src_28000.created_at) AS ds__extract_dow | ||
| , EXTRACT(doy FROM revenue_src_28000.created_at) AS ds__extract_doy | ||
| , DATE_TRUNC('day', revenue_src_28000.created_at) AS revenue_instance__ds__day | ||
| , DATE_TRUNC('week', revenue_src_28000.created_at) AS revenue_instance__ds__week | ||
| , DATE_TRUNC('month', revenue_src_28000.created_at) AS revenue_instance__ds__month | ||
| , DATE_TRUNC('quarter', revenue_src_28000.created_at) AS revenue_instance__ds__quarter | ||
| , DATE_TRUNC('year', revenue_src_28000.created_at) AS revenue_instance__ds__year | ||
| , EXTRACT(year FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_year | ||
| , EXTRACT(quarter FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_quarter | ||
| , EXTRACT(month FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_month | ||
| , EXTRACT(day FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_day | ||
| , EXTRACT(isodow FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_dow | ||
| , EXTRACT(doy FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_doy | ||
| , revenue_src_28000.user_id AS user | ||
| , revenue_src_28000.user_id AS revenue_instance__user | ||
| FROM ***************************.fct_revenue revenue_src_28000 | ||
| ) subq_0 | ||
| ) subq_1 | ||
| ON | ||
| ( | ||
| subq_1.metric_time__day <= subq_2.metric_time__day | ||
| ) AND ( | ||
| subq_1.metric_time__day > subq_2.metric_time__day - INTERVAL 2 month | ||
| ) | ||
| ) subq_4 | ||
| LEFT OUTER JOIN | ||
| ***************************.mf_time_spine subq_5 | ||
| ON | ||
| subq_4.metric_time__day = subq_5.ds | ||
| ) subq_6 | ||
| GROUP BY | ||
| subq_6.metric_time__martian_day | ||
| , subq_6.metric_time__day | ||
| ) subq_7 | ||
| ) subq_8 | ||
| ) subq_9 | ||
| GROUP BY | ||
| subq_9.metric_time__martian_day | ||
| , subq_9.trailing_2_months_revenue |
45 changes: 45 additions & 0 deletions
45
...y/SqlQueryPlan/DuckDB/test_cumulative_metric_with_custom_granularity__plan0_optimized.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| -- Re-aggregate Metric via Group By | ||
| SELECT | ||
| metric_time__martian_day | ||
| , trailing_2_months_revenue | ||
| FROM ( | ||
| -- Compute Metrics via Expressions | ||
| -- Window Function for Metric Re-aggregation | ||
| SELECT | ||
| metric_time__martian_day | ||
| , AVG(txn_revenue) OVER (PARTITION BY metric_time__martian_day) AS trailing_2_months_revenue | ||
| FROM ( | ||
| -- Join to Custom Granularity Dataset | ||
| -- Pass Only Elements: ['txn_revenue', 'metric_time__day', 'metric_time__day'] | ||
| -- Aggregate Measures | ||
| SELECT | ||
| subq_15.martian_day AS metric_time__martian_day | ||
| , subq_14.metric_time__day AS metric_time__day | ||
| , SUM(subq_14.txn_revenue) AS txn_revenue | ||
| FROM ( | ||
| -- Join Self Over Time Range | ||
| SELECT | ||
| subq_13.ds AS metric_time__day | ||
| , revenue_src_28000.revenue AS txn_revenue | ||
| FROM ***************************.mf_time_spine subq_13 | ||
| INNER JOIN | ||
| ***************************.fct_revenue revenue_src_28000 | ||
| ON | ||
| ( | ||
| DATE_TRUNC('day', revenue_src_28000.created_at) <= subq_13.ds | ||
| ) AND ( | ||
| DATE_TRUNC('day', revenue_src_28000.created_at) > subq_13.ds - INTERVAL 2 month | ||
| ) | ||
| ) subq_14 | ||
| LEFT OUTER JOIN | ||
| ***************************.mf_time_spine subq_15 | ||
| ON | ||
| subq_14.metric_time__day = subq_15.ds | ||
| GROUP BY | ||
| subq_15.martian_day | ||
| , subq_14.metric_time__day | ||
| ) subq_17 | ||
| ) subq_19 | ||
| GROUP BY | ||
| metric_time__martian_day | ||
| , trailing_2_months_revenue |
Oops, something went wrong.