From 0a160fc27a3245bdcf4e70ff703921cf9126e069 Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Mon, 29 Jul 2024 10:39:39 -0700 Subject: [PATCH] Support time spine configs for sub-daily granularity (#10483) --- .../unreleased/Features-20240722-202238.yaml | 6 + Makefile | 4 + core/dbt/artifacts/resources/__init__.py | 2 +- core/dbt/artifacts/resources/v1/components.py | 2 + core/dbt/artifacts/resources/v1/model.py | 7 + core/dbt/constants.py | 6 +- core/dbt/contracts/graph/nodes.py | 2 + core/dbt/contracts/graph/semantic_manifest.py | 110 ++- core/dbt/contracts/graph/unparsed.py | 27 + core/dbt/parser/common.py | 11 +- core/dbt/parser/schemas.py | 15 +- core/setup.py | 2 +- schemas/dbt/manifest/v12.json | 796 ++++++++++++++++++ .../functional/artifacts/expected_manifest.py | 38 + .../configs/test_contract_configs.py | 2 +- .../functional/docs/test_good_docs_blocks.py | 6 + .../models/metricflow_time_spine_second.sql | 2 + .../happy_path_project/models/schema.yml | 10 + tests/functional/list/test_list.py | 55 +- tests/functional/time_spines/fixtures.py | 86 ++ .../time_spines/test_time_spines.py | 198 +++++ tests/unit/contracts/graph/test_manifest.py | 1 + 22 files changed, 1357 insertions(+), 31 deletions(-) create mode 100644 .changes/unreleased/Features-20240722-202238.yaml create mode 100644 tests/functional/fixtures/happy_path_project/models/metricflow_time_spine_second.sql create mode 100644 tests/functional/time_spines/fixtures.py create mode 100644 tests/functional/time_spines/test_time_spines.py diff --git a/.changes/unreleased/Features-20240722-202238.yaml b/.changes/unreleased/Features-20240722-202238.yaml new file mode 100644 index 00000000000..5fda200b919 --- /dev/null +++ b/.changes/unreleased/Features-20240722-202238.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support new semantic layer time spine configs to enable sub-daily granularity. +time: 2024-07-22T20:22:38.258249-07:00 +custom: + Author: courtneyholcomb + Issue: "10475" diff --git a/Makefile b/Makefile index bd8b3be57f6..eac08719ed6 100644 --- a/Makefile +++ b/Makefile @@ -144,3 +144,7 @@ help: ## Show this help message. @echo @echo 'options:' @echo 'use USE_DOCKER=true to run target in a docker container' + +.PHONY: json_schema +json_schema: ## Update generated JSON schema using code changes. + scripts/collect-artifact-schema.py --path schemas diff --git a/core/dbt/artifacts/resources/__init__.py b/core/dbt/artifacts/resources/__init__.py index e3ac2f77d7c..3435c386daf 100644 --- a/core/dbt/artifacts/resources/__init__.py +++ b/core/dbt/artifacts/resources/__init__.py @@ -46,7 +46,7 @@ MetricTimeWindow, MetricTypeParams, ) -from dbt.artifacts.resources.v1.model import Model, ModelConfig +from dbt.artifacts.resources.v1.model import Model, ModelConfig, TimeSpine from dbt.artifacts.resources.v1.owner import Owner from dbt.artifacts.resources.v1.saved_query import ( Export, diff --git a/core/dbt/artifacts/resources/v1/components.py b/core/dbt/artifacts/resources/v1/components.py index 6e6605c18ab..fc6f44a38f0 100644 --- a/core/dbt/artifacts/resources/v1/components.py +++ b/core/dbt/artifacts/resources/v1/components.py @@ -10,6 +10,7 @@ from dbt_common.contracts.constraints import ColumnLevelConstraint from dbt_common.contracts.util import Mergeable from dbt_common.dataclass_schema import ExtensibleDbtClassMixin, dbtClassMixin +from dbt_semantic_interfaces.type_enums import TimeGranularity NodeVersion = Union[str, float] @@ -66,6 +67,7 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin): quote: Optional[bool] = None tags: List[str] = field(default_factory=list) _extra: Dict[str, Any] = field(default_factory=dict) + granularity: Optional[TimeGranularity] = None @dataclass diff --git a/core/dbt/artifacts/resources/v1/model.py b/core/dbt/artifacts/resources/v1/model.py index 821d04f147f..051efc5e987 100644 --- a/core/dbt/artifacts/resources/v1/model.py +++ b/core/dbt/artifacts/resources/v1/model.py @@ -11,6 +11,7 @@ from dbt.artifacts.resources.v1.config import NodeConfig from dbt_common.contracts.config.base import MergeBehavior from dbt_common.contracts.constraints import ModelLevelConstraint +from dbt_common.dataclass_schema import dbtClassMixin @dataclass @@ -21,6 +22,11 @@ class ModelConfig(NodeConfig): ) +@dataclass +class TimeSpine(dbtClassMixin): + standard_granularity_column: str + + @dataclass class Model(CompiledResource): resource_type: Literal[NodeType.Model] @@ -32,6 +38,7 @@ class Model(CompiledResource): deprecation_date: Optional[datetime] = None defer_relation: Optional[DeferRelation] = None primary_key: List[str] = field(default_factory=list) + time_spine: Optional[TimeSpine] = None def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): dct = super().__post_serialize__(dct, context) diff --git a/core/dbt/constants.py b/core/dbt/constants.py index 20dd8519855..0ff538910d5 100644 --- a/core/dbt/constants.py +++ b/core/dbt/constants.py @@ -1,3 +1,5 @@ +from dbt_semantic_interfaces.type_enums import TimeGranularity + DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER" SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$" @@ -15,6 +17,8 @@ PACKAGE_LOCK_FILE_NAME = "package-lock.yml" MANIFEST_FILE_NAME = "manifest.json" SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json" -TIME_SPINE_MODEL_NAME = "metricflow_time_spine" +LEGACY_TIME_SPINE_MODEL_NAME = "metricflow_time_spine" +LEGACY_TIME_SPINE_GRANULARITY = TimeGranularity.DAY +MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY = TimeGranularity.DAY PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack" PACKAGE_LOCK_HASH_KEY = "sha1_hash" diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 42d19e2c8dd..008d8d2b2ae 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -58,6 +58,7 @@ from dbt.artifacts.resources import Snapshot as SnapshotResource from dbt.artifacts.resources import SourceDefinition as SourceDefinitionResource from dbt.artifacts.resources import SqlOperation as SqlOperationResource +from dbt.artifacts.resources import TimeSpine from dbt.artifacts.resources import UnitTestDefinition as UnitTestDefinitionResource from dbt.contracts.graph.model_config import UnitTestNodeConfig from dbt.contracts.graph.node_args import ModelNodeArgs @@ -1625,6 +1626,7 @@ class ParsedNodePatch(ParsedPatch): latest_version: Optional[NodeVersion] constraints: List[Dict[str, Any]] deprecation_date: Optional[datetime] + time_spine: Optional[TimeSpine] = None @dataclass diff --git a/core/dbt/contracts/graph/semantic_manifest.py b/core/dbt/contracts/graph/semantic_manifest.py index 21680bbb4ae..04d822a72b5 100644 --- a/core/dbt/contracts/graph/semantic_manifest.py +++ b/core/dbt/contracts/graph/semantic_manifest.py @@ -1,10 +1,19 @@ -from dbt.constants import TIME_SPINE_MODEL_NAME +from typing import List, Optional + +from dbt.constants import ( + LEGACY_TIME_SPINE_GRANULARITY, + LEGACY_TIME_SPINE_MODEL_NAME, + MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY, +) +from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.graph.nodes import ModelNode from dbt.events.types import SemanticValidationFailure from dbt.exceptions import ParsingError from dbt_common.clients.system import write_file from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event from dbt_semantic_interfaces.implementations.metric import PydanticMetric +from dbt_semantic_interfaces.implementations.node_relation import PydanticNodeRelation from dbt_semantic_interfaces.implementations.project_configuration import ( PydanticProjectConfiguration, ) @@ -13,8 +22,12 @@ PydanticSemanticManifest, ) from dbt_semantic_interfaces.implementations.semantic_model import PydanticSemanticModel +from dbt_semantic_interfaces.implementations.time_spine import ( + PydanticTimeSpine, + PydanticTimeSpinePrimaryColumn, +) from dbt_semantic_interfaces.implementations.time_spine_table_configuration import ( - PydanticTimeSpineTableConfiguration, + PydanticTimeSpineTableConfiguration as LegacyTimeSpine, ) from dbt_semantic_interfaces.type_enums import TimeGranularity from dbt_semantic_interfaces.validations.semantic_manifest_validator import ( @@ -23,7 +36,7 @@ class SemanticManifest: - def __init__(self, manifest) -> None: + def __init__(self, manifest: Manifest) -> None: self.manifest = manifest def validate(self) -> bool: @@ -59,8 +72,50 @@ def write_json_to_file(self, file_path: str): write_file(file_path, json) def _get_pydantic_semantic_manifest(self) -> PydanticSemanticManifest: + pydantic_time_spines: List[PydanticTimeSpine] = [] + minimum_time_spine_granularity: Optional[TimeGranularity] = None + for node in self.manifest.nodes.values(): + if not (isinstance(node, ModelNode) and node.time_spine): + continue + time_spine = node.time_spine + standard_granularity_column = None + for column in node.columns.values(): + if column.name == time_spine.standard_granularity_column: + standard_granularity_column = column + break + # Assertions needed for type checking + if not standard_granularity_column: + raise ParsingError( + "Expected to find time spine standard granularity column in model columns, but did not. " + "This should have been caught in YAML parsing." + ) + if not standard_granularity_column.granularity: + raise ParsingError( + "Expected to find granularity set for time spine standard granularity column, but did not. " + "This should have been caught in YAML parsing." + ) + pydantic_time_spine = PydanticTimeSpine( + node_relation=PydanticNodeRelation( + alias=node.alias, + schema_name=node.schema, + database=node.database, + relation_name=node.relation_name, + ), + primary_column=PydanticTimeSpinePrimaryColumn( + name=time_spine.standard_granularity_column, + time_granularity=standard_granularity_column.granularity, + ), + ) + pydantic_time_spines.append(pydantic_time_spine) + if ( + not minimum_time_spine_granularity + or standard_granularity_column.granularity.to_int() + < minimum_time_spine_granularity.to_int() + ): + minimum_time_spine_granularity = standard_granularity_column.granularity + project_config = PydanticProjectConfiguration( - time_spine_table_configurations=[], + time_spine_table_configurations=[], time_spines=pydantic_time_spines ) pydantic_semantic_manifest = PydanticSemanticManifest( metrics=[], semantic_models=[], project_configuration=project_config @@ -79,24 +134,39 @@ def _get_pydantic_semantic_manifest(self) -> PydanticSemanticManifest: PydanticSavedQuery.parse_obj(saved_query.to_dict()) ) - # Look for time-spine table model and create time spine table configuration if self.manifest.semantic_models: - # Get model for time_spine_table - model = self.manifest.ref_lookup.find(TIME_SPINE_MODEL_NAME, None, None, self.manifest) - if not model: + legacy_time_spine_model = self.manifest.ref_lookup.find( + LEGACY_TIME_SPINE_MODEL_NAME, None, None, self.manifest + ) + if legacy_time_spine_model: + if ( + not minimum_time_spine_granularity + or LEGACY_TIME_SPINE_GRANULARITY.to_int() + < minimum_time_spine_granularity.to_int() + ): + minimum_time_spine_granularity = LEGACY_TIME_SPINE_GRANULARITY + + # If no time spines have been configured at DAY or smaller AND legacy time spine model does not exist, error. + if ( + not minimum_time_spine_granularity + or minimum_time_spine_granularity.to_int() + > MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY.to_int() + ): raise ParsingError( - "The semantic layer requires a 'metricflow_time_spine' model in the project, but none was found. " - "Guidance on creating this model can be found on our docs site (" - "https://docs.getdbt.com/docs/build/metricflow-time-spine) " + "The semantic layer requires a time spine model with granularity DAY or smaller in the project, " + "but none was found. Guidance on creating this model can be found on our docs site " + "(https://docs.getdbt.com/docs/build/metricflow-time-spine)." # TODO: update docs link when available! ) - # Create time_spine_table_config, set it in project_config, and add to semantic manifest - time_spine_table_config = PydanticTimeSpineTableConfiguration( - location=model.relation_name, - column_name="date_day", - grain=TimeGranularity.DAY, - ) - pydantic_semantic_manifest.project_configuration.time_spine_table_configurations = [ - time_spine_table_config - ] + + # For backward compatibility: if legacy time spine exists, include it in the manifest. + if legacy_time_spine_model: + legacy_time_spine = LegacyTimeSpine( + location=legacy_time_spine_model.relation_name, + column_name="date_day", + grain=LEGACY_TIME_SPINE_GRANULARITY, + ) + pydantic_semantic_manifest.project_configuration.time_spine_table_configurations = [ + legacy_time_spine + ] return pydantic_semantic_manifest diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index a64223df3b6..1dd00352ca8 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -116,6 +116,7 @@ class HasColumnAndTestProps(HasColumnProps): class UnparsedColumn(HasColumnAndTestProps): quote: Optional[bool] = None tags: List[str] = field(default_factory=list) + granularity: Optional[str] = None # str is really a TimeGranularity Enum @dataclass @@ -206,6 +207,11 @@ class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYa access: Optional[str] = None +@dataclass +class UnparsedTimeSpine(dbtClassMixin): + standard_granularity_column: str + + @dataclass class UnparsedModelUpdate(UnparsedNodeUpdate): quote_columns: Optional[bool] = None @@ -213,6 +219,7 @@ class UnparsedModelUpdate(UnparsedNodeUpdate): latest_version: Optional[NodeVersion] = None versions: Sequence[UnparsedVersion] = field(default_factory=list) deprecation_date: Optional[datetime.datetime] = None + time_spine: Optional[UnparsedTimeSpine] = None def __post_init__(self) -> None: if self.latest_version: @@ -234,6 +241,26 @@ def __post_init__(self) -> None: self.deprecation_date = normalize_date(self.deprecation_date) + if self.time_spine: + columns = ( + self.get_columns_for_version(self.latest_version) + if self.latest_version + else self.columns + ) + column_names_to_columns = {column.name: column for column in columns} + if self.time_spine.standard_granularity_column not in column_names_to_columns: + raise ParsingError( + f"Time spine standard granularity column must be defined on the model. Got invalid " + f"column name '{self.time_spine.standard_granularity_column}' for model '{self.name}'. Valid names" + f"{' for latest version' if self.latest_version else ''}: {list(column_names_to_columns.keys())}." + ) + column = column_names_to_columns[self.time_spine.standard_granularity_column] + if not column.granularity: + raise ParsingError( + f"Time spine standard granularity column must have a granularity defined. Please add one for " + f"column '{self.time_spine.standard_granularity_column}' in model '{self.name}'." + ) + def get_columns_for_version(self, version: NodeVersion) -> List[UnparsedColumn]: if version not in self._version_map: raise DbtInternalError( diff --git a/core/dbt/parser/common.py b/core/dbt/parser/common.py index 5e5807a0335..3bafbb9550f 100644 --- a/core/dbt/parser/common.py +++ b/core/dbt/parser/common.py @@ -18,6 +18,7 @@ from dbt.parser.search import FileBlock from dbt_common.contracts.constraints import ColumnLevelConstraint, ConstraintType from dbt_common.exceptions import DbtInternalError +from dbt_semantic_interfaces.type_enums import TimeGranularity def trimmed(inp: str) -> str: @@ -185,13 +186,12 @@ def __init__(self) -> None: self.column_info: Dict[str, ColumnInfo] = {} def _add(self, column: HasColumnProps) -> None: - tags: List[str] = [] - tags.extend(getattr(column, "tags", ())) - quote: Optional[bool] + tags: List[str] = getattr(column, "tags", []) + quote: Optional[bool] = None + granularity: Optional[TimeGranularity] = None if isinstance(column, UnparsedColumn): quote = column.quote - else: - quote = None + granularity = TimeGranularity(column.granularity) if column.granularity else None if any( c @@ -209,6 +209,7 @@ def _add(self, column: HasColumnProps) -> None: tags=tags, quote=quote, _extra=column.extra, + granularity=granularity, ) @classmethod diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 5e269fd385c..04f63d04e34 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -6,6 +6,7 @@ from dbt import deprecations from dbt.artifacts.resources import RefArgs +from dbt.artifacts.resources.v1.model import TimeSpine from dbt.clients.jinja_static import statically_parse_ref_or_source from dbt.clients.yaml_helper import load_yaml_text from dbt.config import RuntimeConfig @@ -619,9 +620,16 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: # could possibly skip creating one. Leaving here for now for # code consistency. deprecation_date: Optional[datetime.datetime] = None + time_spine: Optional[TimeSpine] = None if isinstance(block.target, UnparsedModelUpdate): deprecation_date = block.target.deprecation_date - + time_spine = ( + TimeSpine( + standard_granularity_column=block.target.time_spine.standard_granularity_column + ) + if block.target.time_spine + else None + ) patch = ParsedNodePatch( name=block.target.name, original_file_path=block.target.original_file_path, @@ -637,6 +645,7 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: latest_version=None, constraints=block.target.constraints, deprecation_date=deprecation_date, + time_spine=time_spine, ) assert isinstance(self.yaml.file, SchemaSourceFile) source_file: SchemaSourceFile = self.yaml.file @@ -915,6 +924,7 @@ def patch_node_properties(self, node, patch: "ParsedNodePatch") -> None: ) # These two will have to be reapplied after config is built for versioned models self.patch_constraints(node, patch.constraints) + self.patch_time_spine(node, patch.time_spine) node.build_contract_checksum() def patch_constraints(self, node, constraints: List[Dict[str, Any]]) -> None: @@ -953,6 +963,9 @@ def _process_constraints_refs_and_sources(self, model_node: ModelNode) -> None: else: model_node.sources.append(ref_or_source) + def patch_time_spine(self, node, time_spine: Optional[TimeSpine]) -> None: + node.time_spine = time_spine + def _validate_pk_constraints( self, model_node: ModelNode, constraints: List[Dict[str, Any]] ) -> None: diff --git a/core/setup.py b/core/setup.py index 1ebce19b6e0..d3d73d378cb 100644 --- a/core/setup.py +++ b/core/setup.py @@ -69,7 +69,7 @@ # Accept patches but avoid automatically updating past a set minor version range. "dbt-extractor>=0.5.0,<=0.6", "minimal-snowplow-tracker>=0.0.2,<0.1", - "dbt-semantic-interfaces>=0.6.8,<0.7", + "dbt-semantic-interfaces>=0.6.10,<0.7", # Minor versions for these are expected to be backwards-compatible "dbt-common>=1.6.0,<2.0", "dbt-adapters>=1.1.1,<2.0", diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 87e0db0756c..45679b9be96 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -537,6 +537,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -567,6 +584,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -1506,6 +1546,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -1536,6 +1593,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -2114,6 +2194,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -2144,6 +2241,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -2847,6 +2967,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -2877,6 +3014,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -3599,6 +3759,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -3629,6 +3806,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -3954,6 +4154,23 @@ "type": "boolean", "default": true }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } + }, "columns": { "type": "array", "items": { @@ -4392,6 +4609,27 @@ "items": { "type": "string" } + }, + "time_spine": { + "anyOf": [ + { + "type": "object", + "title": "TimeSpine", + "properties": { + "standard_granularity_column": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "standard_granularity_column" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -4825,6 +5063,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -4855,6 +5110,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -5433,6 +5711,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -5463,6 +5758,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -6282,6 +6600,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -6312,6 +6647,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -7403,6 +7761,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -7433,6 +7808,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -9754,6 +10152,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -9784,6 +10199,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -10723,6 +11161,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -10753,6 +11208,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -11331,6 +11809,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -11361,6 +11856,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -12064,6 +12582,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -12094,6 +12629,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -12816,6 +13374,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -12846,6 +13421,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -13171,6 +13769,23 @@ "type": "boolean", "default": true }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } + }, "columns": { "type": "array", "items": { @@ -13609,6 +14224,27 @@ "items": { "type": "string" } + }, + "time_spine": { + "anyOf": [ + { + "type": "object", + "title": "TimeSpine", + "properties": { + "standard_granularity_column": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "standard_granularity_column" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -14042,6 +14678,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -14072,6 +14725,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -14650,6 +15326,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -14680,6 +15373,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -15499,6 +16215,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -15529,6 +16262,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, @@ -16611,6 +17367,23 @@ "warn_unsupported": { "type": "boolean", "default": true + }, + "to": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + } } }, "additionalProperties": false, @@ -16641,6 +17414,29 @@ "propertyNames": { "type": "string" } + }, + "granularity": { + "anyOf": [ + { + "enum": [ + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": true, diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index 68e7799182b..7098a4f4fea 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -292,6 +292,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "first_name": { "name": "first_name", @@ -301,6 +302,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "email": { "name": "email", @@ -310,6 +312,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "ip_address": { "name": "ip_address", @@ -319,6 +322,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "updated_at": { "name": "updated_at", @@ -328,6 +332,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "contract": {"checksum": None, "enforced": False, "alias_types": True}, @@ -343,6 +348,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "access": "protected", "version": None, "latest_version": None, + "time_spine": None, }, "model.test.second_model": { "compiled_path": os.path.join(compiled_model_path, "second_model.sql"), @@ -385,6 +391,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "first_name": { "name": "first_name", @@ -394,6 +401,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "email": { "name": "email", @@ -403,6 +411,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "ip_address": { "name": "ip_address", @@ -412,6 +421,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "updated_at": { "name": "updated_at", @@ -421,6 +431,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "contract": {"checksum": None, "enforced": False, "alias_types": True}, @@ -436,6 +447,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "access": "protected", "version": None, "latest_version": None, + "time_spine": None, }, "seed.test.seed": { "build_path": None, @@ -468,6 +480,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "first_name": { "name": "first_name", @@ -477,6 +490,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "email": { "name": "email", @@ -486,6 +500,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "ip_address": { "name": "ip_address", @@ -495,6 +510,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "updated_at": { "name": "updated_at", @@ -504,6 +520,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "docs": {"node_color": None, "show": True}, @@ -730,6 +747,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "quote": None, "tags": [], "constraints": [], + "granularity": None, } }, "config": { @@ -957,6 +975,7 @@ def expected_references_manifest(project): "version": None, "latest_version": None, "constraints": [], + "time_spine": None, }, "model.test.ephemeral_summary": { "alias": "ephemeral_summary", @@ -972,6 +991,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "ct": { "description": "The number of instances of the first name", @@ -981,6 +1001,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "config": get_rendered_model_config(materialized="table", group="test_group"), @@ -1026,6 +1047,7 @@ def expected_references_manifest(project): "version": None, "latest_version": None, "constraints": [], + "time_spine": None, }, "model.test.view_summary": { "alias": "view_summary", @@ -1041,6 +1063,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "ct": { "description": "The number of instances of the first name", @@ -1050,6 +1073,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "config": get_rendered_model_config(), @@ -1091,6 +1115,7 @@ def expected_references_manifest(project): "version": None, "latest_version": None, "constraints": [], + "time_spine": None, }, "seed.test.seed": { "alias": "seed", @@ -1105,6 +1130,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "first_name": { "name": "first_name", @@ -1114,6 +1140,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "email": { "name": "email", @@ -1123,6 +1150,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "ip_address": { "name": "ip_address", @@ -1132,6 +1160,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "updated_at": { "name": "updated_at", @@ -1141,6 +1170,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "config": get_rendered_seed_config(), @@ -1219,6 +1249,7 @@ def expected_references_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, } }, "config": { @@ -1487,6 +1518,7 @@ def expected_versions_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "ct": { "description": "The number of instances of the first name", @@ -1496,6 +1528,7 @@ def expected_versions_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "config": get_rendered_model_config( @@ -1544,6 +1577,7 @@ def expected_versions_manifest(project): "access": "protected", "version": 1, "latest_version": 2, + "time_spine": None, }, "model.test.versioned_model.v2": { "alias": "versioned_model_v2", @@ -1559,6 +1593,7 @@ def expected_versions_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, "extra": { "description": "", @@ -1568,6 +1603,7 @@ def expected_versions_manifest(project): "quote": None, "tags": [], "constraints": [], + "granularity": None, }, }, "config": get_rendered_model_config( @@ -1612,6 +1648,7 @@ def expected_versions_manifest(project): "access": "protected", "version": 2, "latest_version": 2, + "time_spine": None, }, "model.test.ref_versioned_model": { "alias": "ref_versioned_model", @@ -1669,6 +1706,7 @@ def expected_versions_manifest(project): "access": "protected", "version": None, "latest_version": None, + "time_spine": None, }, "test.test.unique_versioned_model_v1_first_name.6138195dec": { "alias": "unique_versioned_model_v1_first_name", diff --git a/tests/functional/configs/test_contract_configs.py b/tests/functional/configs/test_contract_configs.py index 3af9415e3e3..179b0058a8d 100644 --- a/tests/functional/configs/test_contract_configs.py +++ b/tests/functional/configs/test_contract_configs.py @@ -331,7 +331,7 @@ def test__model_contract_true(self, project): assert contract_actual_config.enforced is True - expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[])], quote=True, tags=[], _extra={}), 'color': ColumnInfo(name='color', description='', meta={}, data_type='string', constraints=[], quote=None, tags=[], _extra={}), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={})}" + expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[])], quote=True, tags=[], _extra={}, granularity=None), 'color': ColumnInfo(name='color', description='', meta={}, data_type='string', constraints=[], quote=None, tags=[], _extra={}, granularity=None), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={}, granularity=None)}" assert expected_columns == str(my_model_columns) diff --git a/tests/functional/docs/test_good_docs_blocks.py b/tests/functional/docs/test_good_docs_blocks.py index 768e6201275..e1ed96c5eb7 100644 --- a/tests/functional/docs/test_good_docs_blocks.py +++ b/tests/functional/docs/test_good_docs_blocks.py @@ -91,6 +91,7 @@ def test_valid_doc_ref(self, project): "meta": {}, "quote": None, "tags": [], + "granularity": None, } == model_data["columns"]["id"] assert { @@ -101,6 +102,7 @@ def test_valid_doc_ref(self, project): "meta": {}, "quote": None, "tags": [], + "granularity": None, } == model_data["columns"]["first_name"] assert { @@ -111,6 +113,7 @@ def test_valid_doc_ref(self, project): "meta": {}, "quote": None, "tags": [], + "granularity": None, } == model_data["columns"]["last_name"] assert len(model_data["columns"]) == 3 @@ -152,6 +155,7 @@ def test_alternative_docs_path(self, project): "meta": {}, "quote": None, "tags": [], + "granularity": None, } == model_data["columns"]["id"] assert { @@ -162,6 +166,7 @@ def test_alternative_docs_path(self, project): "meta": {}, "quote": None, "tags": [], + "granularity": None, } == model_data["columns"]["first_name"] assert { @@ -172,6 +177,7 @@ def test_alternative_docs_path(self, project): "meta": {}, "quote": None, "tags": [], + "granularity": None, } == model_data["columns"]["last_name"] assert len(model_data["columns"]) == 3 diff --git a/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine_second.sql b/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine_second.sql new file mode 100644 index 00000000000..656724dbe97 --- /dev/null +++ b/tests/functional/fixtures/happy_path_project/models/metricflow_time_spine_second.sql @@ -0,0 +1,2 @@ +select + {{ dbt.date_trunc('second', dbt.current_timestamp()) }} as ts_second diff --git a/tests/functional/fixtures/happy_path_project/models/schema.yml b/tests/functional/fixtures/happy_path_project/models/schema.yml index ef6addf5b91..3d75adab62d 100644 --- a/tests/functional/fixtures/happy_path_project/models/schema.yml +++ b/tests/functional/fixtures/happy_path_project/models/schema.yml @@ -8,6 +8,16 @@ models: data_tests: - unique - not_null + - name: metricflow_time_spine + description: Day time spine + columns: + - name: date_day + granularity: day + - name: metricflow_time_spine_second + description: Second time spine + columns: + - name: ts_second + granularity: second sources: - name: my_source diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index 653021c608b..145b4e58cb9 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -133,12 +133,20 @@ def expect_analyses_output(self): def expect_model_output(self): expectations = { - "name": ("ephemeral", "incremental", "inner", "metricflow_time_spine", "outer"), + "name": ( + "ephemeral", + "incremental", + "inner", + "metricflow_time_spine", + "metricflow_time_spine_second", + "outer", + ), "selector": ( "test.ephemeral", "test.incremental", "test.sub.inner", "test.metricflow_time_spine", + "test.metricflow_time_spine_second", "test.outer", ), "json": ( @@ -294,6 +302,44 @@ def expect_model_output(self): "alias": "metricflow_time_spine", "resource_type": "model", }, + { + "name": "metricflow_time_spine_second", + "package_name": "test", + "depends_on": { + "nodes": [], + "macros": ["macro.dbt.current_timestamp", "macro.dbt.date_trunc"], + }, + "tags": [], + "config": { + "enabled": True, + "group": None, + "materialized": "view", + "post-hook": [], + "tags": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "persist_docs": {}, + "full_refresh": None, + "unique_key": None, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "database": None, + "schema": None, + "alias": None, + "meta": {}, + "grants": {}, + "packages": [], + "incremental_strategy": None, + "docs": {"node_color": None, "show": True}, + "contract": {"enforced": False, "alias_types": True}, + "access": "protected", + }, + "original_file_path": normalize("models/metricflow_time_spine_second.sql"), + "unique_id": "model.test.metricflow_time_spine_second", + "alias": "metricflow_time_spine_second", + "resource_type": "model", + }, { "name": "outer", "package_name": "test", @@ -338,6 +384,7 @@ def expect_model_output(self): self.dir("models/incremental.sql"), self.dir("models/sub/inner.sql"), self.dir("models/metricflow_time_spine.sql"), + self.dir("models/metricflow_time_spine_second.sql"), self.dir("models/outer.sql"), ), } @@ -573,6 +620,7 @@ def expect_all_output(self): "test.not_null_outer_id", "test.unique_outer_id", "test.metricflow_time_spine", + "test.metricflow_time_spine_second", "test.t", "semantic_model:test.my_sm", "metric:test.total_outer", @@ -618,6 +666,7 @@ def expect_select(self): "test.ephemeral", "test.outer", "test.metricflow_time_spine", + "test.metricflow_time_spine_second", "test.incremental", } @@ -638,6 +687,7 @@ def expect_resource_type_multiple(self): "test.outer", "test.sub.inner", "test.metricflow_time_spine", + "test.metricflow_time_spine_second", "test.t", "test.unique_outer_id", } @@ -658,6 +708,7 @@ def expect_resource_type_multiple(self): "test.not_null_outer_id", "test.outer", "test.metricflow_time_spine", + "test.metricflow_time_spine_second", "test.sub.inner", "test.t", } @@ -693,6 +744,7 @@ def expect_resource_type_env_var(self): "test.outer", "test.sub.inner", "test.metricflow_time_spine", + "test.metricflow_time_spine_second", "test.t", "test.unique_outer_id", } @@ -707,6 +759,7 @@ def expect_resource_type_env_var(self): "test.outer", "test.sub.inner", "test.metricflow_time_spine", + "test.metricflow_time_spine_second", } del os.environ["DBT_EXCLUDE_RESOURCE_TYPES"] diff --git a/tests/functional/time_spines/fixtures.py b/tests/functional/time_spines/fixtures.py new file mode 100644 index 00000000000..19711c5bb24 --- /dev/null +++ b/tests/functional/time_spines/fixtures.py @@ -0,0 +1,86 @@ +models_people_sql = """ +select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at +union all +select 2 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at +union all +select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at +""" + +semantic_model_people_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + dimensions: + - name: favorite_color + type: categorical + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + - name: people + agg: count + expr: id + entities: + - name: id + type: primary + defaults: + agg_time_dimension: created_at +""" + +metricflow_time_spine_sql = """ +SELECT to_date('02/20/2023, 'mm/dd/yyyy') as date_day +""" + +metricflow_time_spine_second_sql = """ +SELECT to_datetime('02/20/2023, 'mm/dd/yyyy hh:mm:ss') as ts_second +""" + +valid_time_spines_yml = """ +version: 2 + +models: + - name: metricflow_time_spine_second + time_spine: + standard_granularity_column: ts_second + columns: + - name: ts_second + granularity: second + - name: metricflow_time_spine + time_spine: + standard_granularity_column: date_day + columns: + - name: date_day + granularity: day +""" + +missing_time_spine_yml = """ +models: + - name: metricflow_time_spine + columns: + - name: ts_second + granularity: second +""" + +time_spine_missing_granularity_yml = """ +models: + - name: metricflow_time_spine_second + time_spine: + standard_granularity_column: ts_second + columns: + - name: ts_second +""" + +time_spine_missing_column_yml = """ +models: + - name: metricflow_time_spine_second + time_spine: + standard_granularity_column: ts_second + columns: + - name: date_day +""" diff --git a/tests/functional/time_spines/test_time_spines.py b/tests/functional/time_spines/test_time_spines.py new file mode 100644 index 00000000000..7b21ec0b35b --- /dev/null +++ b/tests/functional/time_spines/test_time_spines.py @@ -0,0 +1,198 @@ +from typing import Set + +import pytest + +from dbt.cli.main import dbtRunner +from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.graph.semantic_manifest import SemanticManifest +from dbt.exceptions import ParsingError +from dbt.tests.util import get_manifest +from dbt_semantic_interfaces.type_enums import TimeGranularity +from tests.functional.time_spines.fixtures import ( + metricflow_time_spine_second_sql, + metricflow_time_spine_sql, + models_people_sql, + semantic_model_people_yml, + time_spine_missing_column_yml, + time_spine_missing_granularity_yml, + valid_time_spines_yml, +) + + +class TestValidTimeSpines: + """Tests that YAML using current time spine configs parses as expected.""" + + @pytest.fixture(scope="class") + def models(self): + return { + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "metricflow_time_spine_second.sql": metricflow_time_spine_second_sql, + "time_spines.yml": valid_time_spines_yml, + "semantic_model_people.yml": semantic_model_people_yml, + "people.sql": models_people_sql, + } + + def test_time_spines(self, project): + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert result.success + assert isinstance(result.result, Manifest) + + manifest = get_manifest(project.project_root) + assert manifest + + # Test that models and columns are set as expected + time_spine_models = { + id.split(".")[-1]: node for id, node in manifest.nodes.items() if node.time_spine + } + day_model_name = "metricflow_time_spine" + second_model_name = "metricflow_time_spine_second" + day_column_name = "date_day" + second_column_name = "ts_second" + model_names_to_col_names = { + day_model_name: day_column_name, + second_model_name: second_column_name, + } + model_names_to_granularities = { + day_model_name: TimeGranularity.DAY, + second_model_name: TimeGranularity.SECOND, + } + assert len(time_spine_models) == 2 + expected_time_spine_aliases = {second_model_name, day_model_name} + assert set(time_spine_models.keys()) == expected_time_spine_aliases + for model in time_spine_models.values(): + assert ( + model.time_spine.standard_granularity_column + == model_names_to_col_names[model.name] + ) + assert len(model.columns) == 1 + assert ( + list(model.columns.values())[0].granularity + == model_names_to_granularities[model.name] + ) + + # Test that project configs are set as expected in semantic manifest + semantic_manifest = SemanticManifest(manifest) + assert semantic_manifest.validate() + project_config = semantic_manifest._get_pydantic_semantic_manifest().project_configuration + # Legacy config + assert len(project_config.time_spine_table_configurations) == 1 + legacy_time_spine_config = project_config.time_spine_table_configurations[0] + assert legacy_time_spine_config.column_name == day_column_name + assert legacy_time_spine_config.location.replace('"', "").split(".")[-1] == day_model_name + assert legacy_time_spine_config.grain == TimeGranularity.DAY + # Current configs + assert len(project_config.time_spines) == 2 + sl_time_spine_aliases: Set[str] = set() + for sl_time_spine in project_config.time_spines: + alias = sl_time_spine.node_relation.alias + sl_time_spine_aliases.add(alias) + assert sl_time_spine.primary_column.name == model_names_to_col_names[alias] + assert ( + sl_time_spine.primary_column.time_granularity + == model_names_to_granularities[alias] + ) + assert sl_time_spine_aliases == expected_time_spine_aliases + + +class TestValidLegacyTimeSpine: + """Tests that YAML using only legacy time spine config parses as expected.""" + + @pytest.fixture(scope="class") + def models(self): + return { + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_model_people.yml": semantic_model_people_yml, + "people.sql": models_people_sql, + } + + def test_time_spines(self, project): + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert result.success + assert isinstance(result.result, Manifest) + + manifest = get_manifest(project.project_root) + assert manifest + + # Test that project configs are set as expected in semantic manifest + semantic_manifest = SemanticManifest(manifest) + assert semantic_manifest.validate() + project_config = semantic_manifest._get_pydantic_semantic_manifest().project_configuration + # Legacy config + assert len(project_config.time_spine_table_configurations) == 1 + legacy_time_spine_config = project_config.time_spine_table_configurations[0] + assert legacy_time_spine_config.column_name == "date_day" + assert ( + legacy_time_spine_config.location.replace('"', "").split(".")[-1] + == "metricflow_time_spine" + ) + assert legacy_time_spine_config.grain == TimeGranularity.DAY + # Current configs + assert len(project_config.time_spines) == 0 + + +class TestMissingTimeSpine: + """Tests that YAML with semantic models but no time spines errors.""" + + @pytest.fixture(scope="class") + def models(self): + return { + "semantic_model_people.yml": semantic_model_people_yml, + "people.sql": models_people_sql, + } + + def test_time_spines(self, project): + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert isinstance(result.exception, ParsingError) + assert ( + "The semantic layer requires a time spine model with granularity DAY or smaller" + in result.exception.msg + ) + + +class TestTimeSpineColumnMissing: + """Tests that YAML with time spine column not in model errors.""" + + @pytest.fixture(scope="class") + def models(self): + return { + "semantic_model_people.yml": semantic_model_people_yml, + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "metricflow_time_spine_second.sql": metricflow_time_spine_second_sql, + "time_spines.yml": time_spine_missing_column_yml, + } + + def test_time_spines(self, project): + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert isinstance(result.exception, ParsingError) + assert ( + "Time spine standard granularity column must be defined on the model." + in result.exception.msg + ) + + +class TestTimeSpineGranularityMissing: + """Tests that YAML with time spine column without granularity errors.""" + + @pytest.fixture(scope="class") + def models(self): + return { + "semantic_model_people.yml": semantic_model_people_yml, + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "metricflow_time_spine_second.sql": metricflow_time_spine_second_sql, + "time_spines.yml": time_spine_missing_granularity_yml, + } + + def test_time_spines(self, project): + runner = dbtRunner() + result = runner.invoke(["parse"]) + assert isinstance(result.exception, ParsingError) + assert ( + "Time spine standard granularity column must have a granularity defined." + in result.exception.msg + ) diff --git a/tests/unit/contracts/graph/test_manifest.py b/tests/unit/contracts/graph/test_manifest.py index dc81fa4b7dc..5eef57324b7 100644 --- a/tests/unit/contracts/graph/test_manifest.py +++ b/tests/unit/contracts/graph/test_manifest.py @@ -94,6 +94,7 @@ "constraints", "deprecation_date", "defer_relation", + "time_spine", } )