Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Querying with DATE PART #772

Merged
merged 14 commits into from
Sep 19, 2023
Merged

Querying with DATE PART #772

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
9b8257b
Changelog
courtneyholcomb Sep 13, 2023
26591e2
DatePart enum
courtneyholcomb Sep 13, 2023
32533b0
Render EXTRACT columns in semantic model dataset by default
courtneyholcomb Sep 13, 2023
aaec2e5
Support date_part in query params
courtneyholcomb Sep 13, 2023
f164610
Handle date_part in cumulative & offset metric queries
courtneyholcomb Sep 13, 2023
61a463f
Write tests
courtneyholcomb Sep 13, 2023
8a950d4
Update snapshots
courtneyholcomb Sep 13, 2023
1015b5f
PR feedback
courtneyholcomb Sep 18, 2023
28e3c88
Merge main
courtneyholcomb Sep 18, 2023
1c8a21e
More generic error message for granularity issues
courtneyholcomb Sep 18, 2023
8a63473
Add implementation for QueryParameter
courtneyholcomb Sep 18, 2023
8eecab9
TODO
courtneyholcomb Sep 18, 2023
9c5ea8a
Only allow requesting the default granularity with date part
courtneyholcomb Sep 19, 2023
8b76246
PR feedback
courtneyholcomb Sep 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions metricflow/test/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
# These imports are required to properly set up pytest fixtures.
from __future__ import annotations

from dataclasses import dataclass
from typing import Optional

from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity

from metricflow.test.fixtures.cli_fixtures import * # noqa: F401, F403
from metricflow.test.fixtures.dataflow_fixtures import * # noqa: F401, F403
from metricflow.test.fixtures.id_fixtures import * # noqa: F401, F403
Expand All @@ -9,3 +14,13 @@
from metricflow.test.fixtures.sql_client_fixtures import * # noqa: F401, F403
from metricflow.test.fixtures.sql_fixtures import * # noqa: F401, F403
from metricflow.test.fixtures.table_fixtures import * # noqa: F401, F403
from metricflow.time.date_part import DatePart


@dataclass
class MockQueryParameter:
"""This is a mock that is just used to test the query parser."""

name: str
grain: Optional[TimeGranularity] = None
date_part: Optional[DatePart] = None
211 changes: 211 additions & 0 deletions metricflow/test/dataset/test_convert_semantic_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from metricflow.test.fixtures.model_fixtures import ConsistentIdObjectRepository
from metricflow.test.fixtures.setup_fixtures import MetricFlowTestSessionState
from metricflow.test.sql.compare_sql_plan import assert_rendered_sql_equal
from metricflow.time.date_part import DatePart

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -49,6 +50,27 @@ def test_convert_table_semantic_model_without_measures( # noqa: D
TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.MONTH),
TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.QUARTER),
TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.YEAR),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.QUARTER
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.MONTH
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.WEEK
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DAY
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOY
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
Expand All @@ -74,6 +96,48 @@ def test_convert_table_semantic_model_without_measures( # noqa: D
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.YEAR,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.YEAR,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.QUARTER,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.MONTH,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.WEEK,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DAY,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOW,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="user"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOY,
),
),
)

Expand Down Expand Up @@ -125,11 +189,74 @@ def test_convert_table_semantic_model_with_measures( # noqa: D
TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.MONTH),
TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.QUARTER),
TimeDimensionSpec(element_name="ds", entity_links=(), time_granularity=TimeGranularity.YEAR),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.YEAR
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.QUARTER
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.MONTH
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.WEEK
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DAY
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOW
),
TimeDimensionSpec(
element_name="ds", entity_links=(), time_granularity=TimeGranularity.DAY, date_part=DatePart.DOY
),
TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.DAY),
TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.WEEK),
TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.MONTH),
TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.QUARTER),
TimeDimensionSpec(element_name="ds_partitioned", entity_links=(), time_granularity=TimeGranularity.YEAR),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.YEAR,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.QUARTER,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.MONTH,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.WEEK,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DAY,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOW,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOY,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
Expand All @@ -155,6 +282,48 @@ def test_convert_table_semantic_model_with_measures( # noqa: D
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.YEAR,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.YEAR,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.QUARTER,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.MONTH,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.WEEK,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DAY,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOW,
),
TimeDimensionSpec(
element_name="ds",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOY,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
Expand All @@ -180,6 +349,48 @@ def test_convert_table_semantic_model_with_measures( # noqa: D
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.YEAR,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.YEAR,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.QUARTER,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.MONTH,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.WEEK,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DAY,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOW,
),
TimeDimensionSpec(
element_name="ds_partitioned",
entity_links=(EntityReference(element_name="verification"),),
time_granularity=TimeGranularity.DAY,
date_part=DatePart.DOY,
),
),
)

Expand Down
3 changes: 2 additions & 1 deletion metricflow/test/integration/configured_test_case.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import os
from collections import OrderedDict
from enum import Enum
from typing import Optional, Sequence, Tuple
from typing import Dict, Optional, Sequence, Tuple

import yaml
from dbt_semantic_interfaces.implementations.base import FrozenBaseModel
Expand Down Expand Up @@ -53,6 +53,7 @@ class Config: # noqa: D
check_query: str
file_path: str
group_bys: Tuple[str, ...] = ()
group_by_objs: Tuple[Dict, ...] = ()
order_bys: Tuple[str, ...] = ()
# The required features in the DW engine for the test to complete.
required_features: Tuple[RequiredDwEngineFeatures, ...] = ()
Expand Down
74 changes: 74 additions & 0 deletions metricflow/test/integration/test_cases/itest_metrics.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1043,3 +1043,77 @@ integration_test:
) b
ON a.metric_time__week = b.metric_time__week
AND a.metric_time__month = b.metric_time__month
---
integration_test:
name: simple_query_with_date_part
description: Test query using date_part
model: SIMPLE_MODEL
metrics: ["bookings"]
group_by_objs: [{"name": "metric_time", "date_part": "year"}]
check_query: |
SELECT
SUM(1) AS bookings
courtneyholcomb marked this conversation as resolved.
Show resolved Hide resolved
, {{ render_extract("ds", DatePart.YEAR) }} AS metric_time__extract_year
FROM {{ source_schema }}.fct_bookings
GROUP BY {{ render_extract("ds", DatePart.YEAR) }};
---
integration_test:
name: simple_query_with_multiple_date_parts
description: Test query using multiple date_parts
model: SIMPLE_MODEL
metrics: ["bookings"]
group_by_objs: [
{"name": "metric_time", "date_part": "quarter"},
{"name": "metric_time", "date_part": "dow"},
{"name": "metric_time", "date_part": "doy"},
{"name": "metric_time", "date_part": "day"},
{"name": "metric_time", "date_part": "week"},
]
check_query: |
SELECT
SUM(1) AS bookings
, {{ render_extract("ds", DatePart.QUARTER) }} AS metric_time__extract_quarter
, {{ render_extract("ds", DatePart.DOW) }} AS metric_time__extract_dow
, {{ render_extract("ds", DatePart.DOY) }} AS metric_time__extract_doy
, {{ render_extract("ds", DatePart.DAY) }} AS metric_time__extract_day
, {{ render_extract("ds", DatePart.WEEK) }} AS metric_time__extract_week
FROM {{ source_schema }}.fct_bookings
GROUP BY
{{ render_extract("ds", DatePart.QUARTER) }}
, {{ render_extract("ds", DatePart.DOW) }}
, {{ render_extract("ds", DatePart.DOY) }}
, {{ render_extract("ds", DatePart.DAY) }}
, {{ render_extract("ds", DatePart.WEEK) }};
---
integration_test:
name: derived_metric_offset_window_and_date_part
description: Tests a derived metric offset query with window and date_part
model: SIMPLE_MODEL
metrics: ["bookings_5_day_lag"]
group_by_objs: [{"name": "metric_time", "date_part": "month"}]
check_query: |
SELECT
{{ render_extract("a.ds", DatePart.MONTH) }} AS metric_time__extract_month
, SUM(b.bookings_5_day_lag) AS bookings_5_day_lag
FROM {{ mf_time_spine_source }} a
INNER JOIN (
SELECT
ds AS metric_time__day
, 1 AS bookings_5_day_lag
FROM {{ source_schema }}.fct_bookings
) b
ON {{ render_date_sub("a", "ds", 5, TimeGranularity.DAY) }} = b.metric_time__day
GROUP BY metric_time__extract_month
---
integration_test:
name: date_part_overrides_granularity
description: Test query using date_part with incompatible granularity; should override granularity
courtneyholcomb marked this conversation as resolved.
Show resolved Hide resolved
model: SIMPLE_MODEL
metrics: ["bookings"]
group_by_objs: [{"name": "metric_time", "grain": "month", "date_part": "dow"}]
check_query: |
SELECT
SUM(1) AS bookings
, {{ render_extract("ds", DatePart.DOW) }} AS metric_time__extract_dow
FROM {{ source_schema }}.fct_bookings
GROUP BY {{ render_extract("ds", DatePart.DOW) }};
Loading