diff --git a/.changes/unreleased/Features-20240716-104215.yaml b/.changes/unreleased/Features-20240716-104215.yaml new file mode 100644 index 00000000..d86e4e43 --- /dev/null +++ b/.changes/unreleased/Features-20240716-104215.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support for configuring multiple time spines at different granularities. +time: 2024-07-16T10:42:15.662883-07:00 +custom: + Author: courtneyholcomb + Issue: "280" diff --git a/dbt_semantic_interfaces/implementations/node_relation.py b/dbt_semantic_interfaces/implementations/node_relation.py new file mode 100644 index 00000000..9963abaf --- /dev/null +++ b/dbt_semantic_interfaces/implementations/node_relation.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import Any, Optional + +from typing_extensions import override + +from dbt_semantic_interfaces.implementations.base import HashableBaseModel +from dbt_semantic_interfaces.protocols import ProtocolHint +from dbt_semantic_interfaces.protocols.node_relation import NodeRelation +from dsi_pydantic_shim import validator + + +class PydanticNodeRelation(HashableBaseModel, ProtocolHint[NodeRelation]): + """Path object to where the data should be.""" + + alias: str + schema_name: str + database: Optional[str] = None + relation_name: str = "" + + @override + def _implements_protocol(self) -> NodeRelation: # noqa: D + return self + + @validator("relation_name", always=True) + @classmethod + def __create_default_relation_name(cls, value: Any, values: Any) -> str: # type: ignore[misc] + """Dynamically build the dot path for `relation_name`, if not specified.""" + if value: + # Only build the relation_name if it was not present in config. + return value + + alias, schema, database = values.get("alias"), values.get("schema_name"), values.get("database") + if alias is None or schema is None: + raise ValueError( + f"Failed to build relation_name because alias and/or schema was None. schema: {schema}, alias: {alias}" + ) + + if database is not None: + value = f"{database}.{schema}.{alias}" + else: + value = f"{schema}.{alias}" + return value + + @staticmethod + def from_string(sql_str: str) -> PydanticNodeRelation: # noqa: D + sql_str_split = sql_str.split(".") + if len(sql_str_split) == 2: + return PydanticNodeRelation(schema_name=sql_str_split[0], alias=sql_str_split[1]) + elif len(sql_str_split) == 3: + return PydanticNodeRelation(database=sql_str_split[0], schema_name=sql_str_split[1], alias=sql_str_split[2]) + raise RuntimeError( + f"Invalid input for a SQL table, expected form '.' or '..
' " + f"but got: {sql_str}" + ) diff --git a/dbt_semantic_interfaces/implementations/project_configuration.py b/dbt_semantic_interfaces/implementations/project_configuration.py index 1be46fcf..f47bfe5b 100644 --- a/dbt_semantic_interfaces/implementations/project_configuration.py +++ b/dbt_semantic_interfaces/implementations/project_configuration.py @@ -14,6 +14,7 @@ UNKNOWN_VERSION_SENTINEL, PydanticSemanticVersion, ) +from dbt_semantic_interfaces.implementations.time_spine import PydanticTimeSpine from dbt_semantic_interfaces.implementations.time_spine_table_configuration import ( PydanticTimeSpineTableConfiguration, ) @@ -32,6 +33,7 @@ def _implements_protocol(self) -> ProjectConfiguration: time_spine_table_configurations: List[PydanticTimeSpineTableConfiguration] metadata: Optional[PydanticMetadata] = None dsi_package_version: PydanticSemanticVersion = UNKNOWN_VERSION_SENTINEL + time_spines: List[PydanticTimeSpine] = [] @validator("dsi_package_version", always=True) @classmethod diff --git a/dbt_semantic_interfaces/implementations/semantic_model.py b/dbt_semantic_interfaces/implementations/semantic_model.py index 10c85d33..9ce62775 100644 --- a/dbt_semantic_interfaces/implementations/semantic_model.py +++ b/dbt_semantic_interfaces/implementations/semantic_model.py @@ -12,6 +12,7 @@ from dbt_semantic_interfaces.implementations.elements.entity import PydanticEntity from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure from dbt_semantic_interfaces.implementations.metadata import PydanticMetadata +from dbt_semantic_interfaces.implementations.node_relation import PydanticNodeRelation from dbt_semantic_interfaces.protocols import ( ProtocolHint, SemanticModel, @@ -26,48 +27,7 @@ SemanticModelReference, TimeDimensionReference, ) -from dsi_pydantic_shim import Field, validator - - -class NodeRelation(HashableBaseModel): - """Path object to where the data should be.""" - - alias: str - schema_name: str - database: Optional[str] = None - relation_name: str = "" - - @validator("relation_name", always=True) - @classmethod - def __create_default_relation_name(cls, value: Any, values: Any) -> str: # type: ignore[misc] - """Dynamically build the dot path for `relation_name`, if not specified.""" - if value: - # Only build the relation_name if it was not present in config. - return value - - alias, schema, database = values.get("alias"), values.get("schema_name"), values.get("database") - if alias is None or schema is None: - raise ValueError( - f"Failed to build relation_name because alias and/or schema was None. schema: {schema}, alias: {alias}" - ) - - if database is not None: - value = f"{database}.{schema}.{alias}" - else: - value = f"{schema}.{alias}" - return value - - @staticmethod - def from_string(sql_str: str) -> NodeRelation: # noqa: D - sql_str_split = sql_str.split(".") - if len(sql_str_split) == 2: - return NodeRelation(schema_name=sql_str_split[0], alias=sql_str_split[1]) - elif len(sql_str_split) == 3: - return NodeRelation(database=sql_str_split[0], schema_name=sql_str_split[1], alias=sql_str_split[2]) - raise RuntimeError( - f"Invalid input for a SQL table, expected form '.
' or '..
' " - f"but got: {sql_str}" - ) +from dsi_pydantic_shim import Field class PydanticSemanticModelDefaults(HashableBaseModel, ProtocolHint[SemanticModelDefaults]): # noqa: D @@ -96,7 +56,7 @@ def _implements_protocol(self) -> SemanticModel: name: str defaults: Optional[PydanticSemanticModelDefaults] description: Optional[str] - node_relation: NodeRelation + node_relation: PydanticNodeRelation primary_entity: Optional[str] entities: Sequence[PydanticEntity] = [] diff --git a/dbt_semantic_interfaces/implementations/time_spine.py b/dbt_semantic_interfaces/implementations/time_spine.py new file mode 100644 index 00000000..dcb836c0 --- /dev/null +++ b/dbt_semantic_interfaces/implementations/time_spine.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing_extensions import override + +from dbt_semantic_interfaces.implementations.base import HashableBaseModel +from dbt_semantic_interfaces.implementations.semantic_model import PydanticNodeRelation +from dbt_semantic_interfaces.protocols import ProtocolHint +from dbt_semantic_interfaces.protocols.time_spine import ( + TimeSpine, + TimeSpinePrimaryColumn, +) +from dbt_semantic_interfaces.type_enums import TimeGranularity + + +class PydanticTimeSpinePrimaryColumn(HashableBaseModel, ProtocolHint[TimeSpinePrimaryColumn]): + """Legacy Pydantic implementation of SemanticVersion. In the process of deprecation.""" + + @override + def _implements_protocol(self) -> TimeSpinePrimaryColumn: + return self + + name: str + time_granularity: TimeGranularity + + +class PydanticTimeSpine(HashableBaseModel, ProtocolHint[TimeSpine]): + """Legacy Pydantic implementation of SemanticVersion. In the process of deprecation.""" + + @override + def _implements_protocol(self) -> TimeSpine: + return self + + name: str + node_relation: PydanticNodeRelation + primary_column: PydanticTimeSpinePrimaryColumn diff --git a/dbt_semantic_interfaces/implementations/time_spine_table_configuration.py b/dbt_semantic_interfaces/implementations/time_spine_table_configuration.py index 9bab71f8..b8f8c423 100644 --- a/dbt_semantic_interfaces/implementations/time_spine_table_configuration.py +++ b/dbt_semantic_interfaces/implementations/time_spine_table_configuration.py @@ -16,7 +16,7 @@ class PydanticTimeSpineTableConfiguration( HashableBaseModel, ModelWithMetadataParsing, ProtocolHint[TimeSpineTableConfiguration] ): - """Pydantic implementation of SemanticVersion.""" + """Legacy Pydantic implementation of SemanticVersion. In the process of deprecation.""" @override def _implements_protocol(self) -> TimeSpineTableConfiguration: diff --git a/dbt_semantic_interfaces/parsing/generated_json_schemas/default_explicit_schema.json b/dbt_semantic_interfaces/parsing/generated_json_schemas/default_explicit_schema.json index ff23a4dc..9ff350b8 100644 --- a/dbt_semantic_interfaces/parsing/generated_json_schemas/default_explicit_schema.json +++ b/dbt_semantic_interfaces/parsing/generated_json_schemas/default_explicit_schema.json @@ -621,11 +621,15 @@ "$ref": "#/definitions/time_spine_table_configuration_schema" }, "type": "array" + }, + "time_spines": { + "items": { + "$ref": "#/definitions/time_spine_schema" + }, + "type": "array" } }, - "required": [ - "time_spine_table_configurations" - ], + "required": [], "type": "object" }, "saved_query_query_params_schema": { @@ -756,6 +760,67 @@ ], "type": "object" }, + "time_spine_primary_column_schema": { + "$id": "time_spine_primary_column_schema", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "time_granularity": { + "enum": [ + "NANOSECOND", + "MICROSECOND", + "MILLISECOND", + "SECOND", + "MINUTE", + "HOUR", + "DAY", + "WEEK", + "MONTH", + "QUARTER", + "YEAR", + "nanosecond", + "microsecond", + "millisecond", + "second", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ] + } + }, + "required": [ + "name", + "time_granularity" + ], + "type": "object" + }, + "time_spine_schema": { + "$id": "time_spine_schema", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "node_relation": { + "$ref": "#/definitions/node_relation_schema" + }, + "primary_column": { + "$ref": "#/definitions/time_spine_primary_column_schema" + } + }, + "required": [ + "name", + "node_relation", + "primary_column" + ], + "type": "object" + }, "time_spine_table_configuration_schema": { "$id": "time_spine_table_configuration_schema", "additionalProperties": false, diff --git a/dbt_semantic_interfaces/parsing/schemas.py b/dbt_semantic_interfaces/parsing/schemas.py index 26d35205..579b0ce0 100644 --- a/dbt_semantic_interfaces/parsing/schemas.py +++ b/dbt_semantic_interfaces/parsing/schemas.py @@ -347,6 +347,29 @@ "required": ["location", "column_name", "grain"], } +time_spine_primary_column_schema = { + "$id": "time_spine_primary_column_schema", + "type": "object", + "properties": { + "name": {"type": "string"}, + "time_granularity": {"enum": time_granularity_values}, + }, + "additionalProperties": False, + "required": ["name", "time_granularity"], +} + +time_spine_schema = { + "$id": "time_spine_schema", + "type": "object", + "properties": { + "name": {"type": "string"}, + "node_relation": {"$ref": "node_relation_schema"}, + "primary_column": {"$ref": "time_spine_primary_column_schema"}, + }, + "additionalProperties": False, + "required": ["name", "node_relation", "primary_column"], +} + project_configuration_schema = { "$id": "project_configuration_schema", @@ -356,9 +379,13 @@ "type": "array", "items": {"$ref": "time_spine_table_configuration_schema"}, }, + "time_spines": { + "type": "array", + "items": {"$ref": "time_spine_schema"}, + }, }, "additionalProperties": False, - "required": ["time_spine_table_configurations"], + "required": [], } export_config_schema = { @@ -475,6 +502,8 @@ node_relation_schema["$id"]: node_relation_schema, semantic_model_defaults_schema["$id"]: semantic_model_defaults_schema, time_spine_table_configuration_schema["$id"]: time_spine_table_configuration_schema, + time_spine_schema["$id"]: time_spine_schema, + time_spine_primary_column_schema["$id"]: time_spine_primary_column_schema, export_schema["$id"]: export_schema, export_config_schema["$id"]: export_config_schema, saved_query_query_params_schema["$id"]: saved_query_query_params_schema, diff --git a/dbt_semantic_interfaces/protocols/node_relation.py b/dbt_semantic_interfaces/protocols/node_relation.py new file mode 100644 index 00000000..b7246035 --- /dev/null +++ b/dbt_semantic_interfaces/protocols/node_relation.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from abc import abstractmethod +from typing import Optional, Protocol + + +class NodeRelation(Protocol): + """Path object to where the data should be.""" + + @property + @abstractmethod + def alias(self) -> str: # noqa: D + pass + + @property + @abstractmethod + def schema_name(self) -> str: # noqa: D + pass + + @property + @abstractmethod + def database(self) -> Optional[str]: # noqa: D + pass + + @property + @abstractmethod + def relation_name(self) -> str: # noqa: D + pass diff --git a/dbt_semantic_interfaces/protocols/project_configuration.py b/dbt_semantic_interfaces/protocols/project_configuration.py index e2248bb3..d15a9c22 100644 --- a/dbt_semantic_interfaces/protocols/project_configuration.py +++ b/dbt_semantic_interfaces/protocols/project_configuration.py @@ -2,6 +2,7 @@ from typing import Protocol, Sequence from dbt_semantic_interfaces.protocols.semantic_version import SemanticVersion +from dbt_semantic_interfaces.protocols.time_spine import TimeSpine from dbt_semantic_interfaces.protocols.time_spine_configuration import ( TimeSpineTableConfiguration, ) @@ -18,6 +19,12 @@ def dsi_package_version(self) -> SemanticVersion: @property @abstractmethod - def time_spine_table_configurations(self) -> Sequence[TimeSpineTableConfiguration]: + def time_spines(self) -> Sequence[TimeSpine]: """The time spine table configurations. Multiple allowed for different time grains.""" pass + + @property + @abstractmethod + def time_spine_table_configurations(self) -> Sequence[TimeSpineTableConfiguration]: + """Legacy time spine table configurations. In the process of deprecation.""" + pass diff --git a/dbt_semantic_interfaces/protocols/semantic_model.py b/dbt_semantic_interfaces/protocols/semantic_model.py index 4617ec79..d253b39a 100644 --- a/dbt_semantic_interfaces/protocols/semantic_model.py +++ b/dbt_semantic_interfaces/protocols/semantic_model.py @@ -7,6 +7,7 @@ from dbt_semantic_interfaces.protocols.entity import Entity from dbt_semantic_interfaces.protocols.measure import Measure from dbt_semantic_interfaces.protocols.metadata import Metadata +from dbt_semantic_interfaces.protocols.node_relation import NodeRelation from dbt_semantic_interfaces.references import ( EntityReference, LinkableElementReference, @@ -16,30 +17,6 @@ ) -class NodeRelation(Protocol): - """Path object to where the data should be.""" - - @property - @abstractmethod - def alias(self) -> str: # noqa: D - pass - - @property - @abstractmethod - def schema_name(self) -> str: # noqa: D - pass - - @property - @abstractmethod - def database(self) -> Optional[str]: # noqa: D - pass - - @property - @abstractmethod - def relation_name(self) -> str: # noqa: D - pass - - class SemanticModelDefaults(Protocol): """Path object to where the data should be.""" diff --git a/dbt_semantic_interfaces/protocols/time_spine.py b/dbt_semantic_interfaces/protocols/time_spine.py new file mode 100644 index 00000000..eaf23409 --- /dev/null +++ b/dbt_semantic_interfaces/protocols/time_spine.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from abc import abstractmethod +from typing import Protocol + +from dbt_semantic_interfaces.implementations.node_relation import NodeRelation +from dbt_semantic_interfaces.type_enums import TimeGranularity + + +class TimeSpine(Protocol): + """Describes a table that contains dates at a specific time grain. + + One column must map to a standard granularity (one of the TimeGranularity enum members). Others might represent + custom granularity columns. Custom granularity columns are not yet implemented. + """ + + @property + @abstractmethod + def name(self) -> str: + """A name the user assigns to this time spine.""" + pass + + @property + @abstractmethod + def node_relation(self) -> NodeRelation: + """dbt model where this time spine lives.""" # noqa: D403 + pass + + @property + @abstractmethod + def primary_column(self) -> TimeSpinePrimaryColumn: + """The column in the time spine that maps to one of our standard granularities.""" + pass + + +class TimeSpinePrimaryColumn(Protocol): + """The column in the time spine that maps to one of our standard granularities.""" + + @property + @abstractmethod + def name(self) -> str: + """The column name.""" + pass + + @property + @abstractmethod + def time_granularity(self) -> TimeGranularity: + """The column name.""" + pass diff --git a/dbt_semantic_interfaces/protocols/time_spine_configuration.py b/dbt_semantic_interfaces/protocols/time_spine_configuration.py index de0579a7..855ccc4f 100644 --- a/dbt_semantic_interfaces/protocols/time_spine_configuration.py +++ b/dbt_semantic_interfaces/protocols/time_spine_configuration.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from abc import abstractmethod from typing import Protocol @@ -5,10 +7,10 @@ class TimeSpineTableConfiguration(Protocol): - """Describes the configuration for a time spine table. + """Legacy time spine class that will eventually be deprecated in favor of TimeSpine. + Describes the configuration for a time spine table. A time spine table is a table with a single column containing dates at a specific grain. - e.g. with day granularity: ... 2020-01-01 diff --git a/dbt_semantic_interfaces/test_utils.py b/dbt_semantic_interfaces/test_utils.py index c22e1f1a..ced0cd21 100644 --- a/dbt_semantic_interfaces/test_utils.py +++ b/dbt_semantic_interfaces/test_utils.py @@ -20,7 +20,7 @@ PydanticSemanticManifest, ) from dbt_semantic_interfaces.implementations.semantic_model import ( - NodeRelation, + PydanticNodeRelation, PydanticSemanticModel, ) from dbt_semantic_interfaces.parsing.objects import YamlConfigFile @@ -143,7 +143,7 @@ def metric_with_guaranteed_meta( def semantic_model_with_guaranteed_meta( name: str, description: Optional[str] = None, - node_relation: Optional[NodeRelation] = None, + node_relation: Optional[PydanticNodeRelation] = None, metadata: PydanticMetadata = default_meta(), entities: Sequence[PydanticEntity] = (), measures: Sequence[PydanticMeasure] = (), @@ -155,7 +155,7 @@ def semantic_model_with_guaranteed_meta( """ created_node_relation = node_relation if created_node_relation is None: - created_node_relation = NodeRelation( + created_node_relation = PydanticNodeRelation( schema_name="schema", alias="table", ) diff --git a/pyproject.toml b/pyproject.toml index d94278ca..ddce1080 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dbt-semantic-interfaces" -version = "0.6.8" +version = "0.6.9" description = 'The shared semantic layer definitions that dbt-core and MetricFlow use' readme = "README.md" requires-python = ">=3.8" diff --git a/tests/example_project_configuration.py b/tests/example_project_configuration.py index b7cd99aa..1eb3af52 100644 --- a/tests/example_project_configuration.py +++ b/tests/example_project_configuration.py @@ -1,8 +1,13 @@ import textwrap +from dbt_semantic_interfaces.implementations.node_relation import PydanticNodeRelation from dbt_semantic_interfaces.implementations.project_configuration import ( PydanticProjectConfiguration, ) +from dbt_semantic_interfaces.implementations.time_spine import ( + PydanticTimeSpine, + PydanticTimeSpinePrimaryColumn, +) from dbt_semantic_interfaces.implementations.time_spine_table_configuration import ( PydanticTimeSpineTableConfiguration, ) @@ -17,6 +22,13 @@ grain=TimeGranularity.DAY, ) ], + time_spines=[ + PydanticTimeSpine( + name="day_time_spine", + node_relation=PydanticNodeRelation(alias="day_time_spine", schema_name="stuff"), + primary_column=PydanticTimeSpinePrimaryColumn(name="ds_day", time_granularity=TimeGranularity.DAY), + ) + ], ) EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE = YamlConfigFile( @@ -28,6 +40,14 @@ - location: example_schema.example_table column_name: ds grain: day + time_spines: + - name: day_time_spine + node_relation: + schema_name: stuff + alias: day_time_spine + primary_column: + name: ds_day + time_granularity: day """ ), ) diff --git a/tests/fixtures/semantic_manifest_yamls/simple_semantic_manifest/project_configuration.yaml b/tests/fixtures/semantic_manifest_yamls/simple_semantic_manifest/project_configuration.yaml index 7840fe9f..1530052f 100644 --- a/tests/fixtures/semantic_manifest_yamls/simple_semantic_manifest/project_configuration.yaml +++ b/tests/fixtures/semantic_manifest_yamls/simple_semantic_manifest/project_configuration.yaml @@ -4,3 +4,11 @@ project_configuration: - location: example_schema.example_table column_name: ds grain: day + time_spines: + - name: day_time_spine + node_relation: + schema_name: stuff + alias: day_time_spine + primary_column: + name: ds_day + time_granularity: day diff --git a/tests/validations/test_reserved_keywords.py b/tests/validations/test_reserved_keywords.py index e1d477d5..85cd8b02 100644 --- a/tests/validations/test_reserved_keywords.py +++ b/tests/validations/test_reserved_keywords.py @@ -4,7 +4,7 @@ from dbt_semantic_interfaces.implementations.semantic_manifest import ( PydanticSemanticManifest, ) -from dbt_semantic_interfaces.implementations.semantic_model import NodeRelation +from dbt_semantic_interfaces.implementations.semantic_model import PydanticNodeRelation from dbt_semantic_interfaces.test_utils import find_semantic_model_with from dbt_semantic_interfaces.validations.reserved_keywords import ( RESERVED_KEYWORDS, @@ -76,7 +76,7 @@ def test_reserved_keywords_in_node_relation( # noqa: D (semantic_model_with_node_relation, _index) = find_semantic_model_with( model=model, function=lambda semantic_model: semantic_model.node_relation is not None ) - semantic_model_with_node_relation.node_relation = NodeRelation( + semantic_model_with_node_relation.node_relation = PydanticNodeRelation( alias=random_keyword(), schema_name="some_schema", )