diff --git a/metricflow/dataflow/builder/dataflow_plan_builder.py b/metricflow/dataflow/builder/dataflow_plan_builder.py index 856384028c..ca3b232e97 100644 --- a/metricflow/dataflow/builder/dataflow_plan_builder.py +++ b/metricflow/dataflow/builder/dataflow_plan_builder.py @@ -1696,12 +1696,12 @@ def _build_aggregated_measure_from_measure_source_node( non_additive_dimension_grain = self._semantic_model_lookup.get_defined_time_granularity( TimeDimensionReference(non_additive_dimension_spec.name) ) - queried_time_dimension_spec: Optional[ - TimeDimensionSpec - ] = self._find_non_additive_dimension_in_linkable_specs( - agg_time_dimension=agg_time_dimension, - linkable_specs=queried_linkable_specs.as_tuple, - non_additive_dimension_spec=non_additive_dimension_spec, + queried_time_dimension_spec: Optional[TimeDimensionSpec] = ( + self._find_non_additive_dimension_in_linkable_specs( + agg_time_dimension=agg_time_dimension, + linkable_specs=queried_linkable_specs.as_tuple, + non_additive_dimension_spec=non_additive_dimension_spec, + ) ) time_dimension_spec = TimeDimensionSpec( # The NonAdditiveDimensionSpec name property is a plain element name diff --git a/metricflow/plan_conversion/dataflow_to_sql.py b/metricflow/plan_conversion/dataflow_to_sql.py index a17e4ca2fe..0160359bdf 100644 --- a/metricflow/plan_conversion/dataflow_to_sql.py +++ b/metricflow/plan_conversion/dataflow_to_sql.py @@ -275,11 +275,8 @@ def _make_time_spine_data_set( apply_group_by = True for agg_time_dimension_spec in required_time_spine_specs: column_alias = self.column_association_resolver.resolve_spec(agg_time_dimension_spec).column_name - # If the requested granularity is the same as the granularity of the spine, do a direct select. + # If the requested base granularity is the same as the granularity of the spine, do a direct select. agg_time_grain = agg_time_dimension_spec.time_granularity - assert ( - not agg_time_grain.is_custom_granularity - ), "Custom time granularities are not yet supported for all queries." if agg_time_grain.base_granularity == time_spine_source.base_granularity: select_columns += (SqlSelectColumn(expr=column_expr, column_alias=column_alias),) apply_group_by = False @@ -1161,9 +1158,9 @@ def visit_metric_time_dimension_transform_node(self, node: MetricTimeDimensionTr spec=metric_time_dimension_spec, ) ) - output_column_to_input_column[ - metric_time_dimension_column_association.column_name - ] = matching_time_dimension_instance.associated_column.column_name + output_column_to_input_column[metric_time_dimension_column_association.column_name] = ( + matching_time_dimension_instance.associated_column.column_name + ) output_instance_set = InstanceSet( measure_instances=tuple(output_measure_instances), @@ -1400,11 +1397,11 @@ def visit_join_to_time_spine_node(self, node: JoinToTimeSpineNode) -> SqlDataSet f"indicates it may have been configured incorrectly. Expected: {agg_time_dim_for_join_with_base_grain};" f" Got: {[instance.spec for instance in time_spine_dataset.instance_set.time_dimension_instances]}" ) - time_spine_column_select_expr: Union[ - SqlColumnReferenceExpression, SqlDateTruncExpression - ] = SqlColumnReferenceExpression.create( - SqlColumnReference( - table_alias=time_spine_alias, column_name=original_time_spine_dim_instance.spec.qualified_name + time_spine_column_select_expr: Union[SqlColumnReferenceExpression, SqlDateTruncExpression] = ( + SqlColumnReferenceExpression.create( + SqlColumnReference( + table_alias=time_spine_alias, column_name=original_time_spine_dim_instance.spec.qualified_name + ) ) ) diff --git a/tests_metricflow/query_rendering/test_custom_granularity.py b/tests_metricflow/query_rendering/test_custom_granularity.py index beed18d12c..a945b1e73c 100644 --- a/tests_metricflow/query_rendering/test_custom_granularity.py +++ b/tests_metricflow/query_rendering/test_custom_granularity.py @@ -455,3 +455,82 @@ def test_offset_metric_with_custom_granularity_filter_not_in_group_by( # noqa: dataflow_plan_builder=dataflow_plan_builder, query_spec=query_spec, ) + + +@pytest.mark.sql_engine_snapshot +def test_join_to_timespine_metric_with_custom_granularity( # noqa: D103 + request: FixtureRequest, + mf_test_configuration: MetricFlowTestConfiguration, + dataflow_plan_builder: DataflowPlanBuilder, + dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, + sql_client: SqlClient, + query_parser: MetricFlowQueryParser, +) -> None: + query_spec = query_parser.parse_and_validate_query( + metric_names=("bookings_join_to_time_spine",), + group_by_names=("metric_time__martian_day",), + ).query_spec + + render_and_check( + request=request, + mf_test_configuration=mf_test_configuration, + dataflow_to_sql_converter=dataflow_to_sql_converter, + sql_client=sql_client, + dataflow_plan_builder=dataflow_plan_builder, + query_spec=query_spec, + ) + + +@pytest.mark.sql_engine_snapshot +def test_join_to_timespine_metric_with_custom_granularity_filter( # noqa: D103 + request: FixtureRequest, + mf_test_configuration: MetricFlowTestConfiguration, + dataflow_plan_builder: DataflowPlanBuilder, + dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, + sql_client: SqlClient, + query_parser: MetricFlowQueryParser, +) -> None: + query_spec = query_parser.parse_and_validate_query( + metric_names=("bookings_join_to_time_spine",), + group_by_names=("metric_time__martian_day",), + where_constraints=[ + PydanticWhereFilter(where_sql_template=("{{ TimeDimension('metric_time', 'martian_day') }} = '2020-01-01'")) + ], + ).query_spec + + render_and_check( + request=request, + mf_test_configuration=mf_test_configuration, + dataflow_to_sql_converter=dataflow_to_sql_converter, + sql_client=sql_client, + dataflow_plan_builder=dataflow_plan_builder, + query_spec=query_spec, + ) + + +# Results are wrong, martian_day column is not correct. How did that even get rendered? +@pytest.mark.sql_engine_snapshot +def test_join_to_timespine_metric_with_custom_granularity_filter_not_in_group_by( # noqa: D103 + request: FixtureRequest, + mf_test_configuration: MetricFlowTestConfiguration, + dataflow_plan_builder: DataflowPlanBuilder, + dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, + sql_client: SqlClient, + query_parser: MetricFlowQueryParser, +) -> None: + query_spec = query_parser.parse_and_validate_query( + metric_names=("bookings_join_to_time_spine",), + group_by_names=("metric_time__day",), + where_constraints=[ + PydanticWhereFilter(where_sql_template=("{{ TimeDimension('metric_time', 'martian_day') }} = '2020-01-01'")) + ], + ).query_spec + + render_and_check( + request=request, + mf_test_configuration=mf_test_configuration, + dataflow_to_sql_converter=dataflow_to_sql_converter, + sql_client=sql_client, + dataflow_plan_builder=dataflow_plan_builder, + query_spec=query_spec, + ) diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql new file mode 100644 index 0000000000..2d4b4b33d0 --- /dev/null +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql @@ -0,0 +1,254 @@ +-- Compute Metrics via Expressions +SELECT + subq_10.metric_time__day + , subq_10.bookings AS bookings_join_to_time_spine +FROM ( + -- Join to Time Spine Dataset + SELECT + subq_7.metric_time__day AS metric_time__day + , subq_6.bookings AS bookings + FROM ( + -- Filter Time Spine + SELECT + subq_9.metric_time__day + FROM ( + -- Time Spine + SELECT + subq_8.ds AS metric_time__day + , subq_8.ds AS metric_time__martian_day + FROM ***************************.mf_time_spine subq_8 + ) subq_9 + WHERE metric_time__martian_day = '2020-01-01' + ) subq_7 + LEFT OUTER JOIN ( + -- Aggregate Measures + SELECT + subq_5.metric_time__day + , SUM(subq_5.bookings) AS bookings + FROM ( + -- Pass Only Elements: ['bookings', 'metric_time__day'] + SELECT + subq_4.metric_time__day + , subq_4.bookings + FROM ( + -- Constrain Output with WHERE + SELECT + subq_3.metric_time__martian_day + , subq_3.metric_time__day + , subq_3.bookings + FROM ( + -- Pass Only Elements: ['bookings', 'metric_time__day', 'metric_time__day'] + -- Join to Custom Granularity Dataset + SELECT + subq_1.metric_time__day AS metric_time__day + , subq_1.bookings AS bookings + , subq_2.martian_day AS metric_time__martian_day + FROM ( + -- Metric Time Dimension 'ds' + SELECT + subq_0.ds__day + , subq_0.ds__week + , subq_0.ds__month + , subq_0.ds__quarter + , subq_0.ds__year + , subq_0.ds__extract_year + , subq_0.ds__extract_quarter + , subq_0.ds__extract_month + , subq_0.ds__extract_day + , subq_0.ds__extract_dow + , subq_0.ds__extract_doy + , subq_0.ds_partitioned__day + , subq_0.ds_partitioned__week + , subq_0.ds_partitioned__month + , subq_0.ds_partitioned__quarter + , subq_0.ds_partitioned__year + , subq_0.ds_partitioned__extract_year + , subq_0.ds_partitioned__extract_quarter + , subq_0.ds_partitioned__extract_month + , subq_0.ds_partitioned__extract_day + , subq_0.ds_partitioned__extract_dow + , subq_0.ds_partitioned__extract_doy + , subq_0.paid_at__day + , subq_0.paid_at__week + , subq_0.paid_at__month + , subq_0.paid_at__quarter + , subq_0.paid_at__year + , subq_0.paid_at__extract_year + , subq_0.paid_at__extract_quarter + , subq_0.paid_at__extract_month + , subq_0.paid_at__extract_day + , subq_0.paid_at__extract_dow + , subq_0.paid_at__extract_doy + , subq_0.booking__ds__day + , subq_0.booking__ds__week + , subq_0.booking__ds__month + , subq_0.booking__ds__quarter + , subq_0.booking__ds__year + , subq_0.booking__ds__extract_year + , subq_0.booking__ds__extract_quarter + , subq_0.booking__ds__extract_month + , subq_0.booking__ds__extract_day + , subq_0.booking__ds__extract_dow + , subq_0.booking__ds__extract_doy + , subq_0.booking__ds_partitioned__day + , subq_0.booking__ds_partitioned__week + , subq_0.booking__ds_partitioned__month + , subq_0.booking__ds_partitioned__quarter + , subq_0.booking__ds_partitioned__year + , subq_0.booking__ds_partitioned__extract_year + , subq_0.booking__ds_partitioned__extract_quarter + , subq_0.booking__ds_partitioned__extract_month + , subq_0.booking__ds_partitioned__extract_day + , subq_0.booking__ds_partitioned__extract_dow + , subq_0.booking__ds_partitioned__extract_doy + , subq_0.booking__paid_at__day + , subq_0.booking__paid_at__week + , subq_0.booking__paid_at__month + , subq_0.booking__paid_at__quarter + , subq_0.booking__paid_at__year + , subq_0.booking__paid_at__extract_year + , subq_0.booking__paid_at__extract_quarter + , subq_0.booking__paid_at__extract_month + , subq_0.booking__paid_at__extract_day + , subq_0.booking__paid_at__extract_dow + , subq_0.booking__paid_at__extract_doy + , subq_0.ds__day AS metric_time__day + , subq_0.ds__week AS metric_time__week + , subq_0.ds__month AS metric_time__month + , subq_0.ds__quarter AS metric_time__quarter + , subq_0.ds__year AS metric_time__year + , subq_0.ds__extract_year AS metric_time__extract_year + , subq_0.ds__extract_quarter AS metric_time__extract_quarter + , subq_0.ds__extract_month AS metric_time__extract_month + , subq_0.ds__extract_day AS metric_time__extract_day + , subq_0.ds__extract_dow AS metric_time__extract_dow + , subq_0.ds__extract_doy AS metric_time__extract_doy + , subq_0.listing + , subq_0.guest + , subq_0.host + , subq_0.booking__listing + , subq_0.booking__guest + , subq_0.booking__host + , subq_0.is_instant + , subq_0.booking__is_instant + , subq_0.bookings + , subq_0.instant_bookings + , subq_0.booking_value + , subq_0.max_booking_value + , subq_0.min_booking_value + , subq_0.bookers + , subq_0.average_booking_value + , subq_0.referred_bookings + , subq_0.median_booking_value + , subq_0.booking_value_p99 + , subq_0.discrete_booking_value_p99 + , subq_0.approximate_continuous_booking_value_p99 + , subq_0.approximate_discrete_booking_value_p99 + FROM ( + -- Read Elements From Semantic Model 'bookings_source' + SELECT + 1 AS bookings + , CASE WHEN is_instant THEN 1 ELSE 0 END AS instant_bookings + , bookings_source_src_28000.booking_value + , bookings_source_src_28000.booking_value AS max_booking_value + , bookings_source_src_28000.booking_value AS min_booking_value + , bookings_source_src_28000.guest_id AS bookers + , bookings_source_src_28000.booking_value AS average_booking_value + , bookings_source_src_28000.booking_value AS booking_payments + , CASE WHEN referrer_id IS NOT NULL THEN 1 ELSE 0 END AS referred_bookings + , bookings_source_src_28000.booking_value AS median_booking_value + , bookings_source_src_28000.booking_value AS booking_value_p99 + , bookings_source_src_28000.booking_value AS discrete_booking_value_p99 + , bookings_source_src_28000.booking_value AS approximate_continuous_booking_value_p99 + , bookings_source_src_28000.booking_value AS approximate_discrete_booking_value_p99 + , bookings_source_src_28000.is_instant + , DATE_TRUNC('day', bookings_source_src_28000.ds) AS ds__day + , DATE_TRUNC('week', bookings_source_src_28000.ds) AS ds__week + , DATE_TRUNC('month', bookings_source_src_28000.ds) AS ds__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds) AS ds__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds) AS ds__year + , EXTRACT(year FROM bookings_source_src_28000.ds) AS ds__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds) AS ds__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds) AS ds__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds) AS ds__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds) AS ds__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds) AS ds__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__day + , DATE_TRUNC('week', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__week + , DATE_TRUNC('month', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds_partitioned) AS ds_partitioned__year + , EXTRACT(year FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds_partitioned) AS ds_partitioned__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.paid_at) AS paid_at__day + , DATE_TRUNC('week', bookings_source_src_28000.paid_at) AS paid_at__week + , DATE_TRUNC('month', bookings_source_src_28000.paid_at) AS paid_at__month + , DATE_TRUNC('quarter', bookings_source_src_28000.paid_at) AS paid_at__quarter + , DATE_TRUNC('year', bookings_source_src_28000.paid_at) AS paid_at__year + , EXTRACT(year FROM bookings_source_src_28000.paid_at) AS paid_at__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.paid_at) AS paid_at__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.paid_at) AS paid_at__extract_month + , EXTRACT(day FROM bookings_source_src_28000.paid_at) AS paid_at__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.paid_at) AS paid_at__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.paid_at) AS paid_at__extract_doy + , bookings_source_src_28000.is_instant AS booking__is_instant + , DATE_TRUNC('day', bookings_source_src_28000.ds) AS booking__ds__day + , DATE_TRUNC('week', bookings_source_src_28000.ds) AS booking__ds__week + , DATE_TRUNC('month', bookings_source_src_28000.ds) AS booking__ds__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds) AS booking__ds__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds) AS booking__ds__year + , EXTRACT(year FROM bookings_source_src_28000.ds) AS booking__ds__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds) AS booking__ds__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds) AS booking__ds__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds) AS booking__ds__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds) AS booking__ds__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds) AS booking__ds__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__day + , DATE_TRUNC('week', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__week + , DATE_TRUNC('month', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__month + , DATE_TRUNC('quarter', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__quarter + , DATE_TRUNC('year', bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__year + , EXTRACT(year FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_month + , EXTRACT(day FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.ds_partitioned) AS booking__ds_partitioned__extract_doy + , DATE_TRUNC('day', bookings_source_src_28000.paid_at) AS booking__paid_at__day + , DATE_TRUNC('week', bookings_source_src_28000.paid_at) AS booking__paid_at__week + , DATE_TRUNC('month', bookings_source_src_28000.paid_at) AS booking__paid_at__month + , DATE_TRUNC('quarter', bookings_source_src_28000.paid_at) AS booking__paid_at__quarter + , DATE_TRUNC('year', bookings_source_src_28000.paid_at) AS booking__paid_at__year + , EXTRACT(year FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_year + , EXTRACT(quarter FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_quarter + , EXTRACT(month FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_month + , EXTRACT(day FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_day + , EXTRACT(isodow FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_dow + , EXTRACT(doy FROM bookings_source_src_28000.paid_at) AS booking__paid_at__extract_doy + , bookings_source_src_28000.listing_id AS listing + , bookings_source_src_28000.guest_id AS guest + , bookings_source_src_28000.host_id AS host + , bookings_source_src_28000.listing_id AS booking__listing + , bookings_source_src_28000.guest_id AS booking__guest + , bookings_source_src_28000.host_id AS booking__host + FROM ***************************.fct_bookings bookings_source_src_28000 + ) subq_0 + ) subq_1 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_2 + ON + subq_1.metric_time__day = subq_2.ds + ) subq_3 + WHERE metric_time__martian_day = '2020-01-01' + ) subq_4 + ) subq_5 + GROUP BY + subq_5.metric_time__day + ) subq_6 + ON + subq_7.metric_time__day = subq_6.metric_time__day +) subq_10 diff --git a/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity_filter_not_in_group_by__plan0_optimized.sql b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity_filter_not_in_group_by__plan0_optimized.sql new file mode 100644 index 0000000000..ebd2c4e9af --- /dev/null +++ b/tests_metricflow/snapshots/test_custom_granularity.py/SqlQueryPlan/DuckDB/test_join_to_timespine_metric_with_custom_granularity_filter_not_in_group_by__plan0_optimized.sql @@ -0,0 +1,51 @@ +-- Join to Time Spine Dataset +-- Compute Metrics via Expressions +SELECT + subq_18.metric_time__day AS metric_time__day + , subq_17.bookings AS bookings_join_to_time_spine +FROM ( + -- Filter Time Spine + SELECT + metric_time__day + FROM ( + -- Time Spine + SELECT + ds AS metric_time__day + , ds AS metric_time__martian_day + FROM ***************************.mf_time_spine subq_19 + ) subq_20 + WHERE metric_time__martian_day = '2020-01-01' +) subq_18 +LEFT OUTER JOIN ( + -- Constrain Output with WHERE + -- Pass Only Elements: ['bookings', 'metric_time__day'] + -- Aggregate Measures + SELECT + metric_time__day + , SUM(bookings) AS bookings + FROM ( + -- Pass Only Elements: ['bookings', 'metric_time__day', 'metric_time__day'] + -- Join to Custom Granularity Dataset + SELECT + subq_12.metric_time__day AS metric_time__day + , subq_12.bookings AS bookings + , subq_13.martian_day AS metric_time__martian_day + FROM ( + -- Read Elements From Semantic Model 'bookings_source' + -- Metric Time Dimension 'ds' + SELECT + DATE_TRUNC('day', ds) AS metric_time__day + , 1 AS bookings + FROM ***************************.fct_bookings bookings_source_src_28000 + ) subq_12 + LEFT OUTER JOIN + ***************************.mf_time_spine subq_13 + ON + subq_12.metric_time__day = subq_13.ds + ) subq_14 + WHERE metric_time__martian_day = '2020-01-01' + GROUP BY + metric_time__day +) subq_17 +ON + subq_18.metric_time__day = subq_17.metric_time__day