From 61a463f5de44db381c5ac471ba7fd47117f9e759 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Wed, 13 Sep 2023 10:06:30 -0700 Subject: [PATCH] Write tests --- metricflow/test/conftest.py | 15 ++ .../dataset/test_convert_semantic_model.py | 211 ++++++++++++++++++ .../test/integration/configured_test_case.py | 3 +- .../integration/test_cases/itest_metrics.yaml | 74 ++++++ .../test/integration/test_configured_cases.py | 32 +++ .../test/model/test_data_warehouse_tasks.py | 4 +- .../test_dataflow_to_sql_plan.py | 82 +++++++ metricflow/test/query/test_query_parser.py | 70 +++++- metricflow/test/sql/test_sql_expr_render.py | 9 + 9 files changed, 485 insertions(+), 15 deletions(-) diff --git a/metricflow/test/conftest.py b/metricflow/test/conftest.py index 31cb6d558e..13318c9f16 100644 --- a/metricflow/test/conftest.py +++ b/metricflow/test/conftest.py @@ -1,6 +1,11 @@ # These imports are required to properly set up pytest fixtures. from __future__ import annotations +from dataclasses import dataclass +from typing import Optional + +from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity + from metricflow.test.fixtures.cli_fixtures import * # noqa: F401, F403 from metricflow.test.fixtures.dataflow_fixtures import * # noqa: F401, F403 from metricflow.test.fixtures.id_fixtures import * # noqa: F401, F403 @@ -9,3 +14,13 @@ from metricflow.test.fixtures.sql_client_fixtures import * # noqa: F401, F403 from metricflow.test.fixtures.sql_fixtures import * # noqa: F401, F403 from metricflow.test.fixtures.table_fixtures import * # noqa: F401, F403 +from metricflow.time.date_part import DatePart + + +@dataclass +class MockQueryParameter: + """This is a mock that is just used to test the query parser.""" + + name: str + grain: Optional[TimeGranularity] = None + date_part: Optional[DatePart] = None diff --git a/metricflow/test/dataset/test_convert_semantic_model.py b/metricflow/test/dataset/test_convert_semantic_model.py index 3335742f2a..747510ff6e 100644 --- a/metricflow/test/dataset/test_convert_semantic_model.py +++ b/metricflow/test/dataset/test_convert_semantic_model.py @@ -19,6 +19,7 @@ from metricflow.test.fixtures.model_fixtures import ConsistentIdObjectRepository from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState from metricflow.test.sql.compare_sql_plan import assert_rendered_sql_equal +from metricflow.time.date_part import DatePart logger = logging.getLogger(__name__) @@ -49,6 +50,27 @@ def test_convert_table_semantic_model_without_measures( # noqa: D TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.MONTH), TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.QUARTER), TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.YEAR), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.QUARTER + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.MONTH + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.WEEK + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DAY + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOY + ), TimeDimensionSpec( element_name="ds", entity_links=(EntityReference(element_name="user"),), @@ -74,6 +96,48 @@ def test_convert_table_semantic_model_without_measures( # noqa: D entity_links=(EntityReference(element_name="user"),), time_granularity=TimeGranularity.YEAR, ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="user"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="user"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.QUARTER, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="user"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.MONTH, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="user"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.WEEK, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="user"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DAY, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="user"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOW, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="user"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOY, + ), ), ) @@ -125,11 +189,74 @@ def test_convert_table_semantic_model_with_measures( # noqa: D TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.MONTH), TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.QUARTER), TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.YEAR), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.QUARTER + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.MONTH + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.WEEK + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DAY + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW + ), + TimeDimensionSpec( + element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOY + ), TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.DAY), TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.WEEK), TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.MONTH), TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.QUARTER), TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.YEAR), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.QUARTER, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.MONTH, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.WEEK, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DAY, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOW, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOY, + ), TimeDimensionSpec( element_name="ds", entity_links=(EntityReference(element_name="verification"),), @@ -155,6 +282,48 @@ def test_convert_table_semantic_model_with_measures( # noqa: D entity_links=(EntityReference(element_name="verification"),), time_granularity=TimeGranularity.YEAR, ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.QUARTER, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.MONTH, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.WEEK, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DAY, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOW, + ), + TimeDimensionSpec( + element_name="ds", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOY, + ), TimeDimensionSpec( element_name="ds_partitioned", entity_links=(EntityReference(element_name="verification"),), @@ -180,6 +349,48 @@ def test_convert_table_semantic_model_with_measures( # noqa: D entity_links=(EntityReference(element_name="verification"),), time_granularity=TimeGranularity.YEAR, ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.YEAR, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.QUARTER, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.MONTH, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.WEEK, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DAY, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOW, + ), + TimeDimensionSpec( + element_name="ds_partitioned", + entity_links=(EntityReference(element_name="verification"),), + time_granularity=TimeGranularity.DAY, + date_part=DatePart.DOY, + ), ), ) diff --git a/metricflow/test/integration/configured_test_case.py b/metricflow/test/integration/configured_test_case.py index a6c0ac8e20..766873b9b0 100644 --- a/metricflow/test/integration/configured_test_case.py +++ b/metricflow/test/integration/configured_test_case.py @@ -4,7 +4,7 @@ import os from collections import OrderedDict from enum import Enum -from typing import Optional, Sequence, Tuple +from typing import Dict, Optional, Sequence, Tuple import yaml from dbt_semantic_interfaces.implementations.base import FrozenBaseModel @@ -53,6 +53,7 @@ class Config: # noqa: D check_query: str file_path: str group_bys: Tuple[str, ...] = () + group_by_objs: Tuple[Dict, ...] = () order_bys: Tuple[str, ...] = () # The required features in the DW engine for the test to complete. required_features: Tuple[RequiredDwEngineFeatures, ...] = () diff --git a/metricflow/test/integration/test_cases/itest_metrics.yaml b/metricflow/test/integration/test_cases/itest_metrics.yaml index 856b591781..bc3b0ec79e 100644 --- a/metricflow/test/integration/test_cases/itest_metrics.yaml +++ b/metricflow/test/integration/test_cases/itest_metrics.yaml @@ -1043,3 +1043,77 @@ integration_test: ) b ON a.metric_time__week = b.metric_time__week AND a.metric_time__month = b.metric_time__month +--- +integration_test: + name: simple_query_with_date_part + description: Test query using date_part + model: SIMPLE_MODEL + metrics: ["bookings"] + group_by_objs: [{"name": "metric_time", "date_part": "year"}] + check_query: | + SELECT + SUM(1) AS bookings + , {{ render_extract("ds", DatePart.YEAR) }} AS metric_time__extract_year + FROM {{ source_schema }}.fct_bookings + GROUP BY {{ render_extract("ds", DatePart.YEAR) }}; +--- +integration_test: + name: simple_query_with_multiple_date_parts + description: Test query using multiple date_parts + model: SIMPLE_MODEL + metrics: ["bookings"] + group_by_objs: [ + {"name": "metric_time", "date_part": "quarter"}, + {"name": "metric_time", "date_part": "dow"}, + {"name": "metric_time", "date_part": "doy"}, + {"name": "metric_time", "date_part": "day"}, + {"name": "metric_time", "date_part": "week"}, + ] + check_query: | + SELECT + SUM(1) AS bookings + , {{ render_extract("ds", DatePart.QUARTER) }} AS metric_time__extract_quarter + , {{ render_extract("ds", DatePart.DOW) }} AS metric_time__extract_dow + , {{ render_extract("ds", DatePart.DOY) }} AS metric_time__extract_doy + , {{ render_extract("ds", DatePart.DAY) }} AS metric_time__extract_day + , {{ render_extract("ds", DatePart.WEEK) }} AS metric_time__extract_week + FROM {{ source_schema }}.fct_bookings + GROUP BY + {{ render_extract("ds", DatePart.QUARTER) }} + , {{ render_extract("ds", DatePart.DOW) }} + , {{ render_extract("ds", DatePart.DOY) }} + , {{ render_extract("ds", DatePart.DAY) }} + , {{ render_extract("ds", DatePart.WEEK) }}; +--- +integration_test: + name: derived_metric_offset_window_and_date_part + description: Tests a derived metric offset query with window and date_part + model: SIMPLE_MODEL + metrics: ["bookings_5_day_lag"] + group_by_objs: [{"name": "metric_time", "date_part": "month"}] + check_query: | + SELECT + {{ render_extract("a.ds", DatePart.MONTH) }} AS metric_time__extract_month + , SUM(b.bookings_5_day_lag) AS bookings_5_day_lag + FROM {{ mf_time_spine_source }} a + INNER JOIN ( + SELECT + ds AS metric_time__day + , 1 AS bookings_5_day_lag + FROM {{ source_schema }}.fct_bookings + ) b + ON {{ render_date_sub("a", "ds", 5, TimeGranularity.DAY) }} = b.metric_time__day + GROUP BY metric_time__extract_month +--- +integration_test: + name: date_part_overrides_granularity + description: Test query using date_part with incompatible granularity; should override granularity + model: SIMPLE_MODEL + metrics: ["bookings"] + group_by_objs: [{"name": "metric_time", "grain": "month", "date_part": "dow"}] + check_query: | + SELECT + SUM(1) AS bookings + , {{ render_extract("ds", DatePart.DOW) }} AS metric_time__extract_dow + FROM {{ source_schema }}.fct_bookings + GROUP BY {{ render_extract("ds", DatePart.DOW) }}; diff --git a/metricflow/test/integration/test_configured_cases.py b/metricflow/test/integration/test_configured_cases.py index 12b6aac0f3..28bfc0112c 100644 --- a/metricflow/test/integration/test_configured_cases.py +++ b/metricflow/test/integration/test_configured_cases.py @@ -2,6 +2,7 @@ import datetime import logging +from copy import copy from typing import List, Optional, Sequence, Tuple import jinja2 @@ -23,6 +24,7 @@ SqlColumnReference, SqlColumnReferenceExpression, SqlDateTruncExpression, + SqlExtractExpression, SqlPercentileExpression, SqlPercentileExpressionArgument, SqlPercentileFunctionType, @@ -30,6 +32,7 @@ SqlTimeDeltaExpression, ) from metricflow.test.compare_df import assert_dataframes_equal +from metricflow.test.conftest import MockQueryParameter from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState from metricflow.test.integration.configured_test_case import ( CONFIGURED_INTEGRATION_TESTS_REPOSITORY, @@ -39,6 +42,7 @@ from metricflow.test.time.configurable_time_source import ( ConfigurableTimeSource, ) +from metricflow.time.date_part import DatePart logger = logging.getLogger(__name__) @@ -100,6 +104,19 @@ def render_date_trunc(self, expr: str, granularity: TimeGranularity) -> str: ) return self._sql_client.sql_query_plan_renderer.expr_renderer.render_sql_expr(renderable_expr).sql + def render_extract(self, expr: str, date_part: DatePart) -> str: + """Return the EXTRACT call that can be used for converting the given expr to the date_part.""" + renderable_expr = SqlExtractExpression( + date_part=date_part, + arg=SqlCastToTimestampExpression( + arg=SqlStringExpression( + sql_expr=expr, + requires_parenthesis=False, + ) + ), + ) + return self._sql_client.sql_query_plan_renderer.expr_renderer.render_sql_expr(renderable_expr).sql + def render_percentile_expr( self, expr: str, percentile: float, use_discrete_percentile: bool, use_approximate_percentile: bool ) -> str: @@ -238,10 +255,21 @@ def test_case( check_query_helpers = CheckQueryHelpers(sql_client) + group_by: List[MockQueryParameter] = [] + for group_by_kwargs in case.group_by_objs: + kwargs = copy(group_by_kwargs) + date_part = kwargs.get("date_part") + grain = kwargs.get("grain") + if date_part: + kwargs["date_part"] = DatePart(date_part) + if grain: + kwargs["grain"] = TimeGranularity(grain) + group_by.append(MockQueryParameter(**kwargs)) query_result = engine.query( MetricFlowQueryRequest.create_with_random_request_id( metric_names=case.metrics, group_by_names=case.group_bys, + group_by=tuple(group_by), limit=case.limit, time_constraint_start=parser.parse(case.time_constraint[0]) if case.time_constraint else None, time_constraint_end=parser.parse(case.time_constraint[1]) if case.time_constraint else None, @@ -252,8 +280,10 @@ def test_case( source_schema=mf_test_session_state.mf_source_schema, render_time_constraint=check_query_helpers.render_time_constraint, TimeGranularity=TimeGranularity, + DatePart=DatePart, render_date_sub=check_query_helpers.render_date_sub, render_date_trunc=check_query_helpers.render_date_trunc, + render_extract=check_query_helpers.render_extract, render_percentile_expr=check_query_helpers.render_percentile_expr, mf_time_spine_source=semantic_manifest_lookup.time_spine_source.spine_table.sql, double_data_type_name=check_query_helpers.double_data_type_name, @@ -277,8 +307,10 @@ def test_case( source_schema=mf_test_session_state.mf_source_schema, render_time_constraint=check_query_helpers.render_time_constraint, TimeGranularity=TimeGranularity, + DatePart=DatePart, render_date_sub=check_query_helpers.render_date_sub, render_date_trunc=check_query_helpers.render_date_trunc, + render_extract=check_query_helpers.render_extract, render_percentile_expr=check_query_helpers.render_percentile_expr, mf_time_spine_source=semantic_manifest_lookup.time_spine_source.spine_table.sql, double_data_type_name=check_query_helpers.double_data_type_name, diff --git a/metricflow/test/model/test_data_warehouse_tasks.py b/metricflow/test/model/test_data_warehouse_tasks.py index 1c66625f22..3675751f40 100644 --- a/metricflow/test/model/test_data_warehouse_tasks.py +++ b/metricflow/test/model/test_data_warehouse_tasks.py @@ -110,8 +110,8 @@ def test_build_dimension_tasks( # noqa: D ) # on semantic model query with all dimensions assert len(tasks) == 1 - # 1 categorical dimension task, 1 time dimension task, 4 granularity based time dimension tasks - assert len(tasks[0].on_fail_subtasks) == 6 + # 1 categorical dimension task, 1 time dimension task, 4 granularity based time dimension tasks, 7 date_part tasks + assert len(tasks[0].on_fail_subtasks) == 13 def test_validate_dimensions( # noqa: D diff --git a/metricflow/test/plan_conversion/test_dataflow_to_sql_plan.py b/metricflow/test/plan_conversion/test_dataflow_to_sql_plan.py index a387521655..782fa1caf3 100644 --- a/metricflow/test/plan_conversion/test_dataflow_to_sql_plan.py +++ b/metricflow/test/plan_conversion/test_dataflow_to_sql_plan.py @@ -56,6 +56,7 @@ from metricflow.test.snapshot_utils import assert_plan_snapshot_text_equal from metricflow.test.sql.compare_sql_plan import assert_rendered_sql_from_plan_equal, assert_sql_plan_text_equal from metricflow.test.time.metric_time_dimension import MTD_SPEC_DAY, MTD_SPEC_QUARTER, MTD_SPEC_WEEK, MTD_SPEC_YEAR +from metricflow.time.date_part import DatePart @pytest.fixture(scope="session") @@ -1843,3 +1844,84 @@ def test_derived_offset_cumulative_metric( # noqa: D sql_client=sql_client, node=dataflow_plan.sink_output_nodes[0].parent_node, ) + + +def test_simple_query_with_date_part( # noqa: D + request: FixtureRequest, + mf_test_session_state: MetricFlowTestSessionState, + dataflow_plan_builder: DataflowPlanBuilder, + dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, + sql_client: SqlClient, +) -> None: + dataflow_plan = dataflow_plan_builder.build_plan( + MetricFlowQuerySpec( + metric_specs=(MetricSpec(element_name="bookings"),), + time_dimension_specs=( + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW), + ), + ) + ) + + convert_and_check( + request=request, + mf_test_session_state=mf_test_session_state, + dataflow_to_sql_converter=dataflow_to_sql_converter, + sql_client=sql_client, + node=dataflow_plan.sink_output_nodes[0].parent_node, + ) + + +def test_simple_query_with_multiple_date_parts( # noqa: D + request: FixtureRequest, + mf_test_session_state: MetricFlowTestSessionState, + dataflow_plan_builder: DataflowPlanBuilder, + dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, + sql_client: SqlClient, +) -> None: + dataflow_plan = dataflow_plan_builder.build_plan( + MetricFlowQuerySpec( + metric_specs=(MetricSpec(element_name="bookings"),), + time_dimension_specs=( + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DAY), + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW), + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOY), + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.WEEK), + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.MONTH), + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.QUARTER), + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR), + ), + ) + ) + + convert_and_check( + request=request, + mf_test_session_state=mf_test_session_state, + dataflow_to_sql_converter=dataflow_to_sql_converter, + sql_client=sql_client, + node=dataflow_plan.sink_output_nodes[0].parent_node, + ) + + +def test_offset_window_with_date_part( # noqa: D + request: FixtureRequest, + mf_test_session_state: MetricFlowTestSessionState, + dataflow_plan_builder: DataflowPlanBuilder, + dataflow_to_sql_converter: DataflowToSqlQueryPlanConverter, + sql_client: SqlClient, +) -> None: + dataflow_plan = dataflow_plan_builder.build_plan( + MetricFlowQuerySpec( + metric_specs=(MetricSpec(element_name="bookings_growth_2_weeks"),), + time_dimension_specs=( + DataSet.metric_time_dimension_spec(time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW), + ), + ) + ) + + convert_and_check( + request=request, + mf_test_session_state=mf_test_session_state, + dataflow_to_sql_converter=dataflow_to_sql_converter, + sql_client=sql_client, + node=dataflow_plan.sink_output_nodes[0].parent_node, + ) diff --git a/metricflow/test/query/test_query_parser.py b/metricflow/test/query/test_query_parser.py index d585ab4465..b1926067b7 100644 --- a/metricflow/test/query/test_query_parser.py +++ b/metricflow/test/query/test_query_parser.py @@ -21,9 +21,11 @@ OrderBySpec, TimeDimensionSpec, ) +from metricflow.test.conftest import MockQueryParameter from metricflow.test.fixtures.model_fixtures import query_parser_from_yaml from metricflow.test.model.example_project_configuration import EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE from metricflow.test.time.metric_time_dimension import MTD +from metricflow.time.date_part import DatePart from metricflow.time.time_granularity_solver import RequestTimeGranularityException logger = logging.getLogger(__name__) @@ -134,6 +136,18 @@ - name: revenue offset_window: 14 days alias: revenue_2_weeks_ago + --- + metric: + name: revenue_since_start_of_year + description: Revenue since start of year + type: derived + type_params: + expr: revenue - revenue_start_of_year + metrics: + - name: revenue + - name: revenue + offset_to_grain: year + alias: revenue_start_of_year """ ) @@ -171,15 +185,6 @@ def test_query_parser(bookings_query_parser: MetricFlowQueryParser) -> None: # ) -class MockQueryParameter: - """This is a mock that is just used to test the query parser.""" - - grain = None - - def __init__(self, name: str): # noqa: D - self.name = name - - def test_query_parser_with_object_params(bookings_query_parser: MetricFlowQueryParser) -> None: # noqa: D Metric = namedtuple("Metric", ["name"]) metric = Metric("bookings") @@ -371,11 +376,10 @@ def test_derived_metric_query_parsing() -> None: def test_derived_metric_with_offset_parsing() -> None: """Test that querying derived metrics with a time offset requires a time dimension.""" - bookings_yaml_file = YamlConfigFile(filepath="inline_for_test_1", contents=BOOKINGS_YAML) - bookings_yaml_file = YamlConfigFile(filepath="inline_for_test_1", contents=REVENUE_YAML) + revenue_yaml_file = YamlConfigFile(filepath="inline_for_test_1", contents=REVENUE_YAML) metrics_yaml_file = YamlConfigFile(filepath="inline_for_test_1", contents=METRICS_YAML) query_parser = query_parser_from_yaml( - [EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE, bookings_yaml_file, metrics_yaml_file] + [EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE, revenue_yaml_file, metrics_yaml_file] ) # Attempt to query with no dimension with pytest.raises(UnableToSatisfyQueryError): @@ -396,3 +400,45 @@ def test_derived_metric_with_offset_parsing() -> None: metric_names=["revenue_growth_2_weeks"], group_by_names=[MTD], ) + + +def test_date_part_parsing() -> None: + """Test that querying with a date_part verifies compatibility with time_granularity.""" + revenue_yaml_file = YamlConfigFile(filepath="inline_for_test_1", contents=REVENUE_YAML) + metrics_yaml_file = YamlConfigFile(filepath="inline_for_test_1", contents=METRICS_YAML) + query_parser = query_parser_from_yaml( + [EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE, revenue_yaml_file, metrics_yaml_file] + ) + + # Date part is incompatible with metric's defined time granularity + with pytest.raises(RequestTimeGranularityException): + query_parser.parse_and_validate_query( + metric_names=["revenue"], + group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.DOW)], + ) + + # Can't query date part for cumulative metrics + with pytest.raises(UnableToSatisfyQueryError): + query_parser.parse_and_validate_query( + metric_names=["revenue_cumulative"], + group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.DOY)], + ) + + # Can't query date part for metrics with offset to grain + with pytest.raises(UnableToSatisfyQueryError): + query_parser.parse_and_validate_query( + metric_names=["revenue_since_start_of_year"], + group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.DAY)], + ) + + # Date part is compatible + query_parser.parse_and_validate_query( + metric_names=["revenue"], + group_by=[MockQueryParameter(name="metric_time", date_part=DatePart.MONTH)], + ) + + # Incompatible granularity gets overriden + query_parser.parse_and_validate_query( + metric_names=["revenue"], + group_by=[MockQueryParameter(name="metric_time", grain=TimeGranularity.YEAR, date_part=DatePart.MONTH)], + ) diff --git a/metricflow/test/sql/test_sql_expr_render.py b/metricflow/test/sql/test_sql_expr_render.py index c123843ae2..5d3e309b36 100644 --- a/metricflow/test/sql/test_sql_expr_render.py +++ b/metricflow/test/sql/test_sql_expr_render.py @@ -17,6 +17,7 @@ SqlComparison, SqlComparisonExpression, SqlDateTruncExpression, + SqlExtractExpression, SqlFunction, SqlIsNullExpression, SqlLogicalExpression, @@ -29,6 +30,7 @@ SqlWindowFunctionExpression, SqlWindowOrderByArgument, ) +from metricflow.time.date_part import DatePart logger = logging.getLogger(__name__) @@ -193,6 +195,13 @@ def test_date_trunc_expr(default_expr_renderer: DefaultSqlExpressionRenderer) -> assert actual == "DATE_TRUNC('month', ds)" +def test_extract_expr(default_expr_renderer: DefaultSqlExpressionRenderer) -> None: # noqa: D + actual = default_expr_renderer.render_sql_expr( + SqlExtractExpression(date_part=DatePart.DOY, arg=SqlStringExpression("ds")) + ).sql + assert actual == "EXTRACT(doy FROM ds)" + + def test_ratio_computation_expr(default_expr_renderer: DefaultSqlExpressionRenderer) -> None: # noqa: D actual = default_expr_renderer.render_sql_expr( SqlRatioComputationExpression(