From 0a78a96bf223237b3f017dcc2d86e847495140a7 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 5 Oct 2023 15:42:01 -0700 Subject: [PATCH] [Backport 1.6.latest] Support null coalescing properties for metric nodes (#8700) (#8773) * Support null coalescing properties for metric nodes (#8700) * Include 'join_to_timespine` and `fill_nulls_with` in metric fixture * Support `join_to_timespine` and `fill_nulls_with` properties on measure inputs to metrics * Assert new `fill_nulls_with` and `join_to_timespine` properties don't break associated DSI protocol * Add doc for metric null coalescing improvements * Fix unit test for unparsed metric objects The `assert_symmetric` function asserts that dictionaries are mostly equivalent. I say mostly equivalent because it drops keys that are `None`. The issue is that that `join_to_timespine` gets defaulted to `False`, so we have to specify it in the `get_ok_dict` so that they match. * Regenerate v10 schema to include null coalescing options --- .../unreleased/Features-20230922-150754.yaml | 6 + core/dbt/contracts/graph/nodes.py | 2 + core/dbt/contracts/graph/unparsed.py | 2 + core/dbt/parser/schema_yaml_readers.py | 2 + schemas/dbt/manifest/v10.json | 215 +++--------------- tests/functional/metrics/fixtures.py | 2 + tests/unit/test_contracts_graph_unparsed.py | 1 + ..._semantic_layer_nodes_satisfy_protocols.py | 6 +- 8 files changed, 51 insertions(+), 185 deletions(-) create mode 100644 .changes/unreleased/Features-20230922-150754.yaml diff --git a/.changes/unreleased/Features-20230922-150754.yaml b/.changes/unreleased/Features-20230922-150754.yaml new file mode 100644 index 00000000000..6492c3d934a --- /dev/null +++ b/.changes/unreleased/Features-20230922-150754.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support `fill_nulls_with` and `join_to_timespine` for metric nodes +time: 2023-09-22T15:07:54.981752-07:00 +custom: + Author: QMalcolm + Issue: "8593" diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index da5d90a2b45..02283a10eb6 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1342,6 +1342,8 @@ class MetricInputMeasure(dbtClassMixin): name: str filter: Optional[WhereFilter] = None alias: Optional[str] = None + join_to_timespine: bool = False + fill_nulls_with: Optional[int] = None def measure_reference(self) -> MeasureReference: return MeasureReference(element_name=self.name) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 2fe3276b8c3..e2d3efe3607 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -597,6 +597,8 @@ class UnparsedMetricInputMeasure(dbtClassMixin): name: str filter: Optional[str] = None alias: Optional[str] = None + join_to_timespine: bool = False + fill_nulls_with: Optional[int] = None @dataclass diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 145e6485ffe..cf8a38bb16a 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -178,6 +178,8 @@ def _get_input_measure( name=unparsed_input_measure.name, filter=filter, alias=unparsed_input_measure.alias, + join_to_timespine=unparsed_input_measure.join_to_timespine, + fill_nulls_with=unparsed_input_measure.fill_nulls_with, ) def _get_optional_input_measure( diff --git a/schemas/dbt/manifest/v10.json b/schemas/dbt/manifest/v10.json index 353b2f23bf2..39a495af9cb 100644 --- a/schemas/dbt/manifest/v10.json +++ b/schemas/dbt/manifest/v10.json @@ -232,7 +232,7 @@ "generated_at": { "type": "string", "format": "date-time", - "default": "2023-10-04T12:51:08.278576Z" + "default": "2023-10-05T00:33:14.410024Z" }, "invocation_id": { "oneOf": [ @@ -243,7 +243,7 @@ "type": "null" } ], - "default": "91c3fe8c-af16-45af-addf-bad2baaac57b" + "default": "603e2fae-9c7d-4d17-8530-7d28c9875263" }, "env": { "type": "object", @@ -474,7 +474,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.282719 + "default": 1696465994.411958 }, "config_call_dict": { "type": "object", @@ -1187,7 +1187,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.286342 + "default": 1696465994.413604 }, "config_call_dict": { "type": "object", @@ -1575,7 +1575,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.2879858 + "default": 1696465994.414359 }, "config_call_dict": { "type": "object", @@ -1851,7 +1851,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.2906518 + "default": 1696465994.4150689 }, "config_call_dict": { "type": "object", @@ -2273,7 +2273,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.293976 + "default": 1696465994.416128 }, "config_call_dict": { "type": "object", @@ -2539,7 +2539,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.29565 + "default": 1696465994.41679 }, "config_call_dict": { "type": "object", @@ -2797,7 +2797,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.2974122 + "default": 1696465994.4175282 }, "config_call_dict": { "type": "object", @@ -3092,7 +3092,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.3007638 + "default": 1696465994.418854 }, "config_call_dict": { "type": "object", @@ -3599,7 +3599,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.3035781 + "default": 1696465994.420199 }, "config_call_dict": { "type": "object", @@ -4020,7 +4020,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.3070972 + "default": 1696465994.421661 } }, "additionalProperties": false, @@ -4120,138 +4120,6 @@ "additionalProperties": false, "description": "FreshnessThreshold(warn_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , error_after: Union[dbt.contracts.graph.unparsed.Time, NoneType] = , filter: Union[str, NoneType] = None)" }, - "FreshnessMetadata": { - "type": "object", - "required": [], - "properties": { - "dbt_schema_version": { - "type": "string", - "default": "https://schemas.getdbt.com/dbt/sources/v3.json" - }, - "dbt_version": { - "type": "string", - "default": "1.6.5" - }, - "generated_at": { - "type": "string", - "format": "date-time", - "default": "2023-10-04T12:51:08.273991Z" - }, - "invocation_id": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "91c3fe8c-af16-45af-addf-bad2baaac57b" - }, - "env": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "default": {} - } - }, - "additionalProperties": false, - "description": "FreshnessMetadata(dbt_schema_version: str = , dbt_version: str = '1.6.5', generated_at: datetime.datetime = , invocation_id: Union[str, NoneType] = , env: Dict[str, str] = )" - }, - "SourceFreshnessRuntimeError": { - "type": "object", - "required": [ - "unique_id", - "status" - ], - "properties": { - "unique_id": { - "type": "string" - }, - "error": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "integer" - }, - { - "type": "null" - } - ] - }, - "status": { - "type": "string", - "enum": [ - "runtime error" - ] - } - }, - "additionalProperties": false, - "description": "SourceFreshnessRuntimeError(unique_id: str, error: Union[str, int, NoneType], status: dbt.contracts.results.FreshnessErrorEnum)" - }, - "SourceFreshnessOutput": { - "type": "object", - "required": [ - "unique_id", - "max_loaded_at", - "snapshotted_at", - "max_loaded_at_time_ago_in_s", - "status", - "criteria", - "adapter_response", - "timing", - "thread_id", - "execution_time" - ], - "properties": { - "unique_id": { - "type": "string" - }, - "max_loaded_at": { - "type": "string", - "format": "date-time" - }, - "snapshotted_at": { - "type": "string", - "format": "date-time" - }, - "max_loaded_at_time_ago_in_s": { - "type": "number" - }, - "status": { - "type": "string", - "enum": [ - "pass", - "warn", - "error", - "runtime error" - ] - }, - "criteria": { - "$ref": "#/definitions/FreshnessThreshold" - }, - "adapter_response": { - "type": "object" - }, - "timing": { - "type": "array", - "items": { - "$ref": "#/definitions/TimingInfo" - } - }, - "thread_id": { - "type": "string" - }, - "execution_time": { - "type": "number" - } - }, - "additionalProperties": false, - "description": "SourceFreshnessOutput(unique_id: str, max_loaded_at: datetime.datetime, snapshotted_at: datetime.datetime, max_loaded_at_time_ago_in_s: float, status: dbt.contracts.results.FreshnessStatus, criteria: dbt.contracts.graph.unparsed.FreshnessThreshold, adapter_response: Dict[str, Any], timing: List[dbt.contracts.results.TimingInfo], thread_id: str, execution_time: float)" - }, "Time": { "type": "object", "required": [], @@ -4285,41 +4153,6 @@ "additionalProperties": false, "description": "Time(count: Union[int, NoneType] = None, period: Union[dbt.contracts.graph.unparsed.TimePeriod, NoneType] = None)" }, - "TimingInfo": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string" - }, - "started_at": { - "oneOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "completed_at": { - "oneOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - } - }, - "additionalProperties": false, - "description": "TimingInfo(name: str, started_at: Union[datetime.datetime, NoneType] = None, completed_at: Union[datetime.datetime, NoneType] = None)" - }, "ExternalTable": { "type": "object", "required": [], @@ -4499,7 +4332,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.3079429 + "default": 1696465994.421958 }, "supported_languages": { "oneOf": [ @@ -4739,7 +4572,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.309657 + "default": 1696465994.422623 } }, "additionalProperties": false, @@ -4924,7 +4757,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.3126311 + "default": 1696465994.4238322 }, "group": { "oneOf": [ @@ -5063,10 +4896,24 @@ "type": "null" } ] + }, + "join_to_timespine": { + "type": "boolean", + "default": false + }, + "fill_nulls_with": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] } }, "additionalProperties": false, - "description": "MetricInputMeasure(name: str, filter: Union[dbt.contracts.graph.nodes.WhereFilter, NoneType] = None, alias: Union[str, NoneType] = None)" + "description": "MetricInputMeasure(name: str, filter: Union[dbt.contracts.graph.nodes.WhereFilter, NoneType] = None, alias: Union[str, NoneType] = None, join_to_timespine: bool = False, fill_nulls_with: Union[int, NoneType] = None)" }, "WhereFilter": { "type": "object", @@ -5413,7 +5260,7 @@ }, "created_at": { "type": "number", - "default": 1696423868.316482 + "default": 1696465994.425479 }, "config": { "$ref": "#/definitions/SemanticModelConfig", diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index 65d61ad74ad..89eeb930043 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -116,6 +116,8 @@ measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" + join_to_timespine: true + fill_nulls_with: 0 - name: collective_window label: "Collective window" diff --git a/tests/unit/test_contracts_graph_unparsed.py b/tests/unit/test_contracts_graph_unparsed.py index 2bff9e5fade..3a53435173b 100644 --- a/tests/unit/test_contracts_graph_unparsed.py +++ b/tests/unit/test_contracts_graph_unparsed.py @@ -870,6 +870,7 @@ def get_ok_dict(self): "measure": { "name": "customers", "filter": "is_new = true", + "join_to_timespine": False, }, }, "config": {}, diff --git a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py index 7325a41da19..85d24797ef1 100644 --- a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py +++ b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py @@ -216,7 +216,11 @@ def simple_metric_input_measure() -> MetricInputMeasure: @pytest.fixture(scope="session") def complex_metric_input_measure(where_filter) -> MetricInputMeasure: return MetricInputMeasure( - name="test_complex_metric_input_measure", filter=where_filter, alias="complex_alias" + name="test_complex_metric_input_measure", + filter=where_filter, + alias="complex_alias", + join_to_timespine=True, + fill_nulls_with=0, )