From 47458140814d0df68a604fb935698ad3511c6181 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 20 Jun 2024 16:49:21 -0700 Subject: [PATCH 01/11] Upgrade to new version of DSI that contains CumulativeTypeParams --- core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/setup.py b/core/setup.py index 56572111054..42b60836931 100644 --- a/core/setup.py +++ b/core/setup.py @@ -69,7 +69,7 @@ # Accept patches but avoid automatically updating past a set minor version range. "dbt-extractor>=0.5.0,<=0.6", "minimal-snowplow-tracker>=0.0.2,<0.1", - "dbt-semantic-interfaces>=0.5.1,<0.7", + "dbt-semantic-interfaces>=0.6.1,<0.7", # Minor versions for these are expected to be backwards-compatible "dbt-common>=1.3.0,<2.0", "dbt-adapters>=1.1.1,<2.0", From ee9016eda0b5da226b33043ae980cec511f42030 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 20 Jun 2024 16:52:41 -0700 Subject: [PATCH 02/11] Add CumulativeTypeParams to core parser --- core/dbt/artifacts/resources/v1/metric.py | 9 +++++ core/dbt/contracts/graph/unparsed.py | 13 ++++++- core/dbt/parser/schema_yaml_readers.py | 41 +++++++++++++++++++---- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/core/dbt/artifacts/resources/v1/metric.py b/core/dbt/artifacts/resources/v1/metric.py index 4f4dcfb98c9..87a56f4b2ba 100644 --- a/core/dbt/artifacts/resources/v1/metric.py +++ b/core/dbt/artifacts/resources/v1/metric.py @@ -6,6 +6,7 @@ from dbt_semantic_interfaces.type_enums import ( ConversionCalculationType, MetricType, + PeriodAggregation, TimeGranularity, ) @@ -80,6 +81,13 @@ class ConversionTypeParams(dbtClassMixin): constant_properties: Optional[List[ConstantPropertyInput]] = None +@dataclass +class CumulativeTypeParams(dbtClassMixin): + window: Optional[MetricTimeWindow] = None + grain_to_date: Optional[TimeGranularity] = None + period_agg: PeriodAggregation = PeriodAggregation.FIRST + + @dataclass class MetricTypeParams(dbtClassMixin): measure: Optional[MetricInputMeasure] = None @@ -91,6 +99,7 @@ class MetricTypeParams(dbtClassMixin): grain_to_date: Optional[TimeGranularity] = None metrics: Optional[List[MetricInput]] = None conversion_type_params: Optional[ConversionTypeParams] = None + cumulative_type_params: Optional[CumulativeTypeParams] = None @dataclass diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index f2fb390c69e..d463f4be709 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -4,7 +4,10 @@ from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Sequence, Union -from dbt_semantic_interfaces.type_enums import ConversionCalculationType +from dbt_semantic_interfaces.type_enums import ( + ConversionCalculationType, + PeriodAggregation, +) # trigger the PathEncoder import dbt_common.helper_types # noqa:F401 @@ -532,6 +535,13 @@ class UnparsedConversionTypeParams(dbtClassMixin): constant_properties: Optional[List[ConstantPropertyInput]] = None +@dataclass +class UnparsedCumulativeTypeParams(dbtClassMixin): + window: Optional[str] = None + grain_to_date: Optional[str] = None + period_agg: str = PeriodAggregation.FIRST.value + + @dataclass class UnparsedMetricTypeParams(dbtClassMixin): measure: Optional[Union[UnparsedMetricInputMeasure, str]] = None @@ -542,6 +552,7 @@ class UnparsedMetricTypeParams(dbtClassMixin): grain_to_date: Optional[str] = None # str is really a TimeGranularity Enum metrics: Optional[List[Union[UnparsedMetricInput, str]]] = None conversion_type_params: Optional[UnparsedConversionTypeParams] = None + cumulative_type_params: Optional[UnparsedCumulativeTypeParams] = None @dataclass diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 646de376763..5abb15eac76 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -6,11 +6,13 @@ DimensionType, EntityType, MetricType, + PeriodAggregation, TimeGranularity, ) from dbt.artifacts.resources import ( ConversionTypeParams, + CumulativeTypeParams, Dimension, DimensionTypeParams, Entity, @@ -42,6 +44,7 @@ from dbt.contracts.graph.nodes import Exposure, Group, Metric, SavedQuery, SemanticModel from dbt.contracts.graph.unparsed import ( UnparsedConversionTypeParams, + UnparsedCumulativeTypeParams, UnparsedDimension, UnparsedDimensionTypeParams, UnparsedEntity, @@ -221,9 +224,19 @@ def _get_input_measures( return input_measures - def _get_time_window( - self, - unparsed_window: Optional[str], + def _get_period_agg(self, unparsed_period_agg: str) -> Optional[PeriodAggregation]: + return PeriodAggregation(unparsed_period_agg) + + def _get_optional_grain_to_date( + self, unparsed_grain_to_date: Optional[str] + ) -> Optional[TimeGranularity]: + if not unparsed_grain_to_date: + return None + + return TimeGranularity(unparsed_grain_to_date) + + def _get_optional_time_window( + self, unparsed_window: Optional[str] ) -> Optional[MetricTimeWindow]: if unparsed_window is not None: parts = unparsed_window.split(" ") @@ -277,7 +290,7 @@ def _get_metric_input(self, unparsed: Union[UnparsedMetricInput, str]) -> Metric name=unparsed.name, filter=parse_where_filter(unparsed.filter), alias=unparsed.alias, - offset_window=self._get_time_window(unparsed.offset_window), + offset_window=self._get_optional_time_window(unparsed.offset_window), offset_to_grain=offset_to_grain, ) @@ -311,10 +324,21 @@ def _get_optional_conversion_type_params( conversion_measure=self._get_input_measure(unparsed.conversion_measure), entity=unparsed.entity, calculation=ConversionCalculationType(unparsed.calculation), - window=self._get_time_window(unparsed.window), + window=self._get_optional_time_window(unparsed.window), constant_properties=unparsed.constant_properties, ) + def _get_optional_cumulative_type_params( + self, unparsed: Optional[UnparsedCumulativeTypeParams] + ) -> Optional[CumulativeTypeParams]: + if unparsed is None: + return None + return CumulativeTypeParams( + window=self._get_optional_time_window(unparsed.window), + grain_to_date=self._get_optional_grain_to_date(unparsed.grain_to_date), + period_agg=self._get_period_agg(unparsed.period_agg), + ) + def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> MetricTypeParams: grain_to_date: Optional[TimeGranularity] = None if type_params.grain_to_date is not None: @@ -325,12 +349,15 @@ def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> Metr numerator=self._get_optional_metric_input(type_params.numerator), denominator=self._get_optional_metric_input(type_params.denominator), expr=str(type_params.expr) if type_params.expr is not None else None, - window=self._get_time_window(type_params.window), + window=self._get_optional_time_window(type_params.window), grain_to_date=grain_to_date, metrics=self._get_metric_inputs(type_params.metrics), conversion_type_params=self._get_optional_conversion_type_params( type_params.conversion_type_params - ) + ), + cumulative_type_params=self._get_optional_cumulative_type_params( + type_params.cumulative_type_params + ), # input measures are calculated via metric processing post parsing # input_measures=?, ) From 2cd734cc2eba4fe8eaa3bf5cdf0905a9c44de342 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 20 Jun 2024 16:58:56 -0700 Subject: [PATCH 03/11] Add CumulativeTypeParams to core parser --- core/dbt/artifacts/resources/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbt/artifacts/resources/__init__.py b/core/dbt/artifacts/resources/__init__.py index 5d456dd6ef9..e3ac2f77d7c 100644 --- a/core/dbt/artifacts/resources/__init__.py +++ b/core/dbt/artifacts/resources/__init__.py @@ -38,6 +38,7 @@ from dbt.artifacts.resources.v1.metric import ( ConstantPropertyInput, ConversionTypeParams, + CumulativeTypeParams, Metric, MetricConfig, MetricInput, From e32cadb5a7eb0195bde09fc40a8fcd3ab5ca0bc1 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 20 Jun 2024 17:06:32 -0700 Subject: [PATCH 04/11] Regenerate JSON schema --- schemas/dbt/catalog/v1.json | 2 +- schemas/dbt/manifest/v12.json | 362 ++++++++++++++++++++++++++------ schemas/dbt/run-results/v6.json | 2 +- schemas/dbt/sources/v3.json | 2 +- 4 files changed, 300 insertions(+), 68 deletions(-) diff --git a/schemas/dbt/catalog/v1.json b/schemas/dbt/catalog/v1.json index 3aabdf34ecf..25c2b25b2b3 100644 --- a/schemas/dbt/catalog/v1.json +++ b/schemas/dbt/catalog/v1.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.8.0a1" + "default": "1.9.0a1" }, "generated_at": { "type": "string" diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 6da1edee266..6e125cc0c98 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -13,7 +13,7 @@ }, "dbt_version": { "type": "string", - "default": "1.8.0b3" + "default": "1.9.0a1" }, "generated_at": { "type": "string" @@ -4388,18 +4388,10 @@ "default": null }, "primary_key": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ], - "default": null + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -8257,6 +8249,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8281,6 +8279,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8371,6 +8375,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8395,6 +8405,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8442,6 +8458,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8466,6 +8488,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8546,6 +8574,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8570,6 +8604,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8769,6 +8809,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -8830,6 +8876,89 @@ } ], "default": null + }, + "cumulative_type_params": { + "anyOf": [ + { + "type": "object", + "title": "CumulativeTypeParams", + "properties": { + "window": { + "anyOf": [ + { + "type": "object", + "title": "MetricTimeWindow", + "properties": { + "count": { + "type": "integer" + }, + "granularity": { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + } + }, + "additionalProperties": false, + "required": [ + "count", + "granularity" + ] + }, + { + "type": "null" + } + ], + "default": null + }, + "grain_to_date": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null + }, + "period_agg": { + "enum": [ + "first", + "last", + "average" + ], + "default": "first" + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false @@ -13453,18 +13582,10 @@ "default": null }, "primary_key": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ], - "default": null + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -17104,6 +17225,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17128,6 +17255,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17218,6 +17351,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17242,6 +17381,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17289,6 +17434,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17313,6 +17464,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17393,6 +17550,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17417,6 +17580,12 @@ "anyOf": [ { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17616,6 +17785,12 @@ }, "granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -17677,6 +17852,89 @@ } ], "default": null + }, + "cumulative_type_params": { + "anyOf": [ + { + "type": "object", + "title": "CumulativeTypeParams", + "properties": { + "window": { + "anyOf": [ + { + "type": "object", + "title": "MetricTimeWindow", + "properties": { + "count": { + "type": "integer" + }, + "granularity": { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + } + }, + "additionalProperties": false, + "required": [ + "count", + "granularity" + ] + }, + { + "type": "null" + } + ], + "default": null + }, + "grain_to_date": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null + }, + "period_agg": { + "enum": [ + "first", + "last", + "average" + ], + "default": "first" + } + }, + "additionalProperties": false + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false @@ -17930,26 +18188,7 @@ "type": "string" }, "resource_type": { - "enum": [ - "model", - "analysis", - "test", - "snapshot", - "operation", - "seed", - "rpc", - "sql_operation", - "doc", - "source", - "macro", - "exposure", - "metric", - "group", - "saved_query", - "semantic_model", - "unit_test", - "fixture" - ] + "const": "saved_query" }, "package_name": { "type": "string" @@ -18741,6 +18980,12 @@ "properties": { "time_granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", @@ -19468,26 +19713,7 @@ "type": "string" }, "resource_type": { - "enum": [ - "model", - "analysis", - "test", - "snapshot", - "operation", - "seed", - "rpc", - "sql_operation", - "doc", - "source", - "macro", - "exposure", - "metric", - "group", - "saved_query", - "semantic_model", - "unit_test", - "fixture" - ] + "const": "saved_query" }, "package_name": { "type": "string" @@ -20286,6 +20512,12 @@ "properties": { "time_granularity": { "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", "day", "week", "month", diff --git a/schemas/dbt/run-results/v6.json b/schemas/dbt/run-results/v6.json index 01843769d86..86b79e1206c 100644 --- a/schemas/dbt/run-results/v6.json +++ b/schemas/dbt/run-results/v6.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.8.0a1" + "default": "1.9.0a1" }, "generated_at": { "type": "string" diff --git a/schemas/dbt/sources/v3.json b/schemas/dbt/sources/v3.json index 5fae8131f63..5ade4a90be0 100644 --- a/schemas/dbt/sources/v3.json +++ b/schemas/dbt/sources/v3.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.8.0a1" + "default": "1.9.0a1" }, "generated_at": { "type": "string" From f660f32987b1ac0ce2c478f5e76c4f4f1da6649d Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 20 Jun 2024 17:14:15 -0700 Subject: [PATCH 05/11] Fix spelling error --- core/dbt/contracts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/contracts/README.md b/core/dbt/contracts/README.md index 367601803f1..40a7bfe8a02 100644 --- a/core/dbt/contracts/README.md +++ b/core/dbt/contracts/README.md @@ -4,7 +4,7 @@ ## Artifacts ### Generating JSON schemas -A helper script, `sripts/collect-artifact-schema.py` is available to generate json schemas corresponding to versioned artifacts (`ArtifactMixin`s). +A helper script, `scripts/collect-artifact-schema.py` is available to generate json schemas corresponding to versioned artifacts (`ArtifactMixin`s). This script is necessary to run when a new artifact schema version is created, or when changes are made to existing artifact versions, and writes json schema to `schema/dbt//v.json`. From 3e6f7757919d254b1427e60b990999dc1d5890c4 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Tue, 25 Jun 2024 09:51:12 -0700 Subject: [PATCH 06/11] Changelog --- .../unreleased/Features-20240625-095107.yaml | 6 ++ index.html | 75 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 .changes/unreleased/Features-20240625-095107.yaml create mode 100644 index.html diff --git a/.changes/unreleased/Features-20240625-095107.yaml b/.changes/unreleased/Features-20240625-095107.yaml new file mode 100644 index 00000000000..1b06d80ae19 --- /dev/null +++ b/.changes/unreleased/Features-20240625-095107.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support cumulative_type_params in semantic manifest. +time: 2024-06-25T09:51:07.983248-07:00 +custom: + Author: courtneyholcomb + Issue: "10360" diff --git a/index.html b/index.html new file mode 100644 index 00000000000..c580ce917dd --- /dev/null +++ b/index.html @@ -0,0 +1,75 @@ +dbt Docs
icons
From c9507c1804250772b6a9b9a18574cf237d7fdaa3 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Tue, 25 Jun 2024 10:08:10 -0700 Subject: [PATCH 07/11] Type fix --- core/dbt/parser/schema_yaml_readers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 5abb15eac76..bd05e397949 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -224,7 +224,7 @@ def _get_input_measures( return input_measures - def _get_period_agg(self, unparsed_period_agg: str) -> Optional[PeriodAggregation]: + def _get_period_agg(self, unparsed_period_agg: str) -> PeriodAggregation: return PeriodAggregation(unparsed_period_agg) def _get_optional_grain_to_date( From 40bb8e7e03aa49d626023068074af184b4333e40 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Tue, 25 Jun 2024 11:26:45 -0700 Subject: [PATCH 08/11] Tests --- tests/functional/metrics/fixtures.py | 28 +++++++++++ tests/functional/metrics/test_metrics.py | 50 +++++++++++++++++++ ..._semantic_layer_nodes_satisfy_protocols.py | 13 ++++- 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index c9d2d0ab190..5f37e61b6f3 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -744,6 +744,34 @@ """ +cumulative_metric_yml = """ +version: 2 +metrics: + - name: weekly_visits + label: Rolling sum of visits over the last 7 days + type: cumulative + type_params: + measure: num_visits + cumulative_type_params: + window: 7 days + period_agg: average + - name: cumulative_orders + label: Rolling total of orders (all time) + type: cumulative + type_params: + measure: num_orders + cumulative_type_params: + period_agg: last + - name: orders_ytd + label: Total orders since the start of the year + type: cumulative + type_params: + measure: num_orders + cumulative_type_params: + grain_to_date: year + period_agg: first +""" + conversion_metric_yml = """ version: 2 metrics: diff --git a/tests/functional/metrics/test_metrics.py b/tests/functional/metrics/test_metrics.py index b640c90c199..d6d3ca2a3dd 100644 --- a/tests/functional/metrics/test_metrics.py +++ b/tests/functional/metrics/test_metrics.py @@ -1,5 +1,8 @@ import pytest +from dbt_semantic_interfaces.type_enums.period_agg import PeriodAggregation +from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity +from dbt.artifacts.resources.v1.metric import CumulativeTypeParams, MetricTimeWindow from dbt.cli.main import dbtRunner from dbt.contracts.graph.manifest import Manifest from dbt.exceptions import ParsingError @@ -8,6 +11,7 @@ basic_metrics_yml, conversion_metric_yml, conversion_semantic_model_purchasing_yml, + cumulative_metric_yml, derived_metric_yml, downstream_model_sql, duplicate_measure_metric_yml, @@ -402,6 +406,52 @@ def test_conversion_metric( ) +class TestCumulativeMetric: + @pytest.fixture(scope="class") + def models(self): + return { + "purchasing.sql": purchasing_model_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": conversion_semantic_model_purchasing_yml, + "conversion_metric.yml": cumulative_metric_yml, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"mock_purchase_data.csv": mock_purchase_data_csv} + + def test_cumulative_metric(self, project): + # initial parse + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert result.success + assert isinstance(result.result, Manifest) + + manifest = get_manifest(project.project_root) + metric_ids = set(manifest.metrics.keys()) + expected_metric_ids_to_cumulative_type_params = { + "metric.test.weekly_visits": CumulativeTypeParams( + window=MetricTimeWindow(count=7, granularity=TimeGranularity.DAY), + period_agg=PeriodAggregation.AVERAGE, + ), + "metric.test.cumulative_orders": CumulativeTypeParams( + period_agg=PeriodAggregation.LAST + ), + "metric.test.orders_ytd": CumulativeTypeParams( + grain_to_date=TimeGranularity.YEAR, period_agg=PeriodAggregation.FIRST + ), + } + assert metric_ids == set(expected_metric_ids_to_cumulative_type_params.keys()) + for ( + metric_id, + expected_cumulative_type_params, + ) in expected_metric_ids_to_cumulative_type_params.items(): + assert ( + manifest.metrics[metric_id].type_params.cumulative_type_params + == expected_cumulative_type_params + ) + + class TestFilterParsing: @pytest.fixture(scope="class") def models(self): diff --git a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py index 379ae46e7a9..b78c254f78d 100644 --- a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py +++ b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py @@ -23,6 +23,7 @@ from dbt.artifacts.resources import ( ConstantPropertyInput, ConversionTypeParams, + CumulativeTypeParams, Defaults, Dimension, DimensionTypeParams, @@ -245,9 +246,18 @@ def conversion_type_params( ) +@pytest.fixture(scope="session") +def cumulative_type_params() -> CumulativeTypeParams: + return CumulativeTypeParams() + + @pytest.fixture(scope="session") def complex_metric_type_params( - metric_time_window, simple_metric_input, simple_metric_input_measure + metric_time_window, + simple_metric_input, + simple_metric_input_measure, + conversion_type_params, + cumulative_type_params, ) -> MetricTypeParams: return MetricTypeParams( measure=simple_metric_input_measure, @@ -258,6 +268,7 @@ def complex_metric_type_params( grain_to_date=TimeGranularity.DAY, metrics=[simple_metric_input], conversion_type_params=conversion_type_params, + cumulative_type_params=cumulative_type_params, ) From d3e33625fc31b0d0246da3cee50c3c4374c6fb21 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 27 Jun 2024 15:23:51 -0700 Subject: [PATCH 09/11] Cleanup --- .../unreleased/Features-20240625-095107.yaml | 2 +- index.html | 75 ------------------- 2 files changed, 1 insertion(+), 76 deletions(-) delete mode 100644 index.html diff --git a/.changes/unreleased/Features-20240625-095107.yaml b/.changes/unreleased/Features-20240625-095107.yaml index 1b06d80ae19..ce7c3d6c803 100644 --- a/.changes/unreleased/Features-20240625-095107.yaml +++ b/.changes/unreleased/Features-20240625-095107.yaml @@ -1,5 +1,5 @@ kind: Features -body: Support cumulative_type_params in semantic manifest. +body: Support cumulative_type_params & sub-daily granularities in semantic manifest. time: 2024-06-25T09:51:07.983248-07:00 custom: Author: courtneyholcomb diff --git a/index.html b/index.html deleted file mode 100644 index c580ce917dd..00000000000 --- a/index.html +++ /dev/null @@ -1,75 +0,0 @@ -dbt Docs
icons
From 9ab091584f6644154a7cd952078db5e25df84146 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 27 Jun 2024 15:53:07 -0700 Subject: [PATCH 10/11] Copy old fields into new fields for deprecation purposes --- core/dbt/parser/schema_yaml_readers.py | 48 ++++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index bd05e397949..45f28090948 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -329,17 +329,43 @@ def _get_optional_conversion_type_params( ) def _get_optional_cumulative_type_params( - self, unparsed: Optional[UnparsedCumulativeTypeParams] + self, unparsed_metric: UnparsedMetric ) -> Optional[CumulativeTypeParams]: - if unparsed is None: - return None - return CumulativeTypeParams( - window=self._get_optional_time_window(unparsed.window), - grain_to_date=self._get_optional_grain_to_date(unparsed.grain_to_date), - period_agg=self._get_period_agg(unparsed.period_agg), - ) + unparsed_type_params = unparsed_metric.type_params + if unparsed_metric.type.lower() == MetricType.CUMULATIVE.value: + if not unparsed_type_params.cumulative_type_params: + unparsed_type_params.cumulative_type_params = UnparsedCumulativeTypeParams() + + if ( + unparsed_type_params.window + and not unparsed_type_params.cumulative_type_params.window + ): + unparsed_type_params.cumulative_type_params.window = unparsed_type_params.window + if ( + unparsed_type_params.grain_to_date + and not unparsed_type_params.cumulative_type_params.grain_to_date + ): + unparsed_type_params.cumulative_type_params.grain_to_date = ( + unparsed_type_params.grain_to_date + ) + + return CumulativeTypeParams( + window=self._get_optional_time_window( + unparsed_type_params.cumulative_type_params.window + ), + grain_to_date=self._get_optional_grain_to_date( + unparsed_type_params.cumulative_type_params.grain_to_date + ), + period_agg=self._get_period_agg( + unparsed_type_params.cumulative_type_params.period_agg + ), + ) + + return None + + def _get_metric_type_params(self, unparsed_metric: UnparsedMetric) -> MetricTypeParams: + type_params = unparsed_metric.type_params - def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> MetricTypeParams: grain_to_date: Optional[TimeGranularity] = None if type_params.grain_to_date is not None: grain_to_date = TimeGranularity(type_params.grain_to_date) @@ -356,7 +382,7 @@ def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> Metr type_params.conversion_type_params ), cumulative_type_params=self._get_optional_cumulative_type_params( - type_params.cumulative_type_params + unparsed_metric=unparsed_metric, ), # input measures are calculated via metric processing post parsing # input_measures=?, @@ -407,7 +433,7 @@ def parse_metric(self, unparsed: UnparsedMetric, generated: bool = False): description=unparsed.description, label=unparsed.label, type=MetricType(unparsed.type), - type_params=self._get_metric_type_params(unparsed.type_params), + type_params=self._get_metric_type_params(unparsed), filter=parse_where_filter(unparsed.filter), meta=unparsed.meta, tags=unparsed.tags, From 96b79a87eeb44ec43e2728ed5b52c88bb5607187 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Thu, 27 Jun 2024 15:53:26 -0700 Subject: [PATCH 11/11] Add tests for manifests with old fields set --- tests/functional/metrics/fixtures.py | 25 ++++++++++++++++++++++++ tests/functional/metrics/test_metrics.py | 16 ++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index 5f37e61b6f3..39d11fb0a95 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -770,6 +770,31 @@ cumulative_type_params: grain_to_date: year period_agg: first + - name: monthly_orders + label: Orders in the past month + type: cumulative + type_params: + measure: num_orders + window: 1 month + cumulative_type_params: + period_agg: average + - name: yearly_orders + label: Orders in the past year + type: cumulative + type_params: + measure: num_orders + window: 1 year + - name: visits_mtd + label: Visits since start of month + type: cumulative + type_params: + measure: num_visits + grain_to_date: month + - name: cumulative_visits + label: Rolling total of visits (all time) + type: cumulative + type_params: + measure: num_visits """ conversion_metric_yml = """ diff --git a/tests/functional/metrics/test_metrics.py b/tests/functional/metrics/test_metrics.py index d6d3ca2a3dd..fa6c8a86e86 100644 --- a/tests/functional/metrics/test_metrics.py +++ b/tests/functional/metrics/test_metrics.py @@ -440,6 +440,20 @@ def test_cumulative_metric(self, project): "metric.test.orders_ytd": CumulativeTypeParams( grain_to_date=TimeGranularity.YEAR, period_agg=PeriodAggregation.FIRST ), + "metric.test.monthly_orders": CumulativeTypeParams( + window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH), + period_agg=PeriodAggregation.AVERAGE, + ), + "metric.test.yearly_orders": CumulativeTypeParams( + window=MetricTimeWindow(count=1, granularity=TimeGranularity.YEAR), + period_agg=PeriodAggregation.FIRST, + ), + "metric.test.visits_mtd": CumulativeTypeParams( + grain_to_date=TimeGranularity.MONTH, period_agg=PeriodAggregation.FIRST + ), + "metric.test.cumulative_visits": CumulativeTypeParams( + period_agg=PeriodAggregation.FIRST + ), } assert metric_ids == set(expected_metric_ids_to_cumulative_type_params.keys()) for ( @@ -449,7 +463,7 @@ def test_cumulative_metric(self, project): assert ( manifest.metrics[metric_id].type_params.cumulative_type_params == expected_cumulative_type_params - ) + ), f"Found unexpected cumulative type params for {metric_id}" class TestFilterParsing: