Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add time spine table configuration to semantic manifest #7996

Merged
merged 7 commits into from
Jun 30, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20230629-172524.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Store time_spline table configuration in semantic manifest
time: 2023-06-29T17:25:24.265366-04:00
custom:
Author: gshank
Issue: "7938"
34 changes: 33 additions & 1 deletion core/dbt/contracts/graph/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
DuplicateMacroInPackageError,
DuplicateMaterializationNameError,
AmbiguousResourceNameRefError,
ParsingError,
)
from dbt.helper_types import PathSet
from dbt.events.functions import fire_event
Expand All @@ -62,6 +63,13 @@
from dbt_semantic_interfaces.implementations.metric import PydanticMetric
from dbt_semantic_interfaces.implementations.semantic_manifest import PydanticSemanticManifest
from dbt_semantic_interfaces.implementations.semantic_model import PydanticSemanticModel
from dbt_semantic_interfaces.implementations.project_configuration import (
PydanticProjectConfiguration,
)
from dbt_semantic_interfaces.implementations.time_spine_table_configuration import (
PydanticTimeSpineTableConfiguration,
)
from dbt_semantic_interfaces.type_enums import TimeGranularity

NodeEdgeMap = Dict[str, List[str]]
PackageName = str
Expand Down Expand Up @@ -982,7 +990,12 @@ def analysis_lookup(self) -> AnalysisLookup:

@property
def pydantic_semantic_manifest(self) -> PydanticSemanticManifest:
pydantic_semantic_manifest = PydanticSemanticManifest(metrics=[], semantic_models=[])
project_config = PydanticProjectConfiguration(
time_spine_table_configurations=[],
)
pydantic_semantic_manifest = PydanticSemanticManifest(
metrics=[], semantic_models=[], project_configuration=project_config
)

for semantic_model in self.semantic_models.values():
pydantic_semantic_manifest.semantic_models.append(
Expand All @@ -992,6 +1005,25 @@ def pydantic_semantic_manifest(self) -> PydanticSemanticManifest:
for metric in self.metrics.values():
pydantic_semantic_manifest.metrics.append(PydanticMetric.parse_obj(metric.to_dict()))

# Look for time-spine table model and create time spine table configuration
if self.semantic_models:
# Get model for time_spine_table
time_spine_model_name = "metricflow_time_spine"
model = self.ref_lookup.find(time_spine_model_name, None, None, self)
if not model:
raise ParsingError(
"The semantic layer requires a 'metricflow_time_spine' model in the project, but none was found. Guidance on creating this model can be found on our docs site (https://docs.getdbt.com/docs/build/metricflow-time-spine)"
)
# Create time_spine_table_config, set it in project_config, and add to semantic manifest
time_spine_table_config = PydanticTimeSpineTableConfiguration(
location=model.relation_name,
column_name="date_day",
grain=TimeGranularity.DAY,
)
pydantic_semantic_manifest.project_configuration.time_spine_table_configurations = [
time_spine_table_config
]

return pydantic_semantic_manifest

@property
Expand Down
2 changes: 1 addition & 1 deletion core/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
"minimal-snowplow-tracker~=0.0.2",
# DSI is under active development, so we're pinning to specific dev versions for now.
# TODO: Before RC/final release, update to use ~= pinning.
"dbt-semantic-interfaces==0.1.0.dev7",
"dbt-semantic-interfaces==0.1.0.dev8",
# ----
# Expect compatibility with all new versions of these packages, so lower bounds only.
"packaging>20.9",
Expand Down
22 changes: 20 additions & 2 deletions tests/functional/semantic_models/test_semantic_model_parsing.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,32 @@
1000 as revenue,
current_timestamp as created_at"""

metricflow_time_spine_sql = """
with days as (
{{dbt_utils.date_spine('day'
, "to_date('01/01/2000','mm/dd/yyyy')"
, "to_date('01/01/2027','mm/dd/yyyy')"
)
}}
),

final as (
select cast(date_day as date) as date_day
from days
)

select *
from final
"""


class TestSemanticModelParsing:
@pytest.fixture(scope="class")
def models(self):
return {
"schema.yml": schema_yml,
"fct_revenue.sql": fct_revenue_sql,
"metricflow_time_spine.sql": metricflow_time_spine_sql,
}

def test_semantic_model_parsing(self, project):
Expand Down Expand Up @@ -111,7 +130,6 @@ def test_semantic_model_changed_partial_parsing(self, project):
semantic_model = manifest.semantic_models["semantic_model.test.revenue"]
assert semantic_model.dimensions[0].type_params.time_granularity == TimeGranularity.WEEK

@pytest.mark.skip(reason="Tracking in https://github.com/dbt-labs/dbt-core/issues/7977")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚀

def test_semantic_model_deleted_partial_parsing(self, project):
# First, use the default schema.yml to define our semantic model, and
# run the dbt parse command
Expand All @@ -128,4 +146,4 @@ def test_semantic_model_deleted_partial_parsing(self, project):
assert result.success

# Finally, verify that the manifest reflects the deletion
assert "semanticmodel.test.revenue" not in result.result.semantic_models
assert "semantic_model.test.revenue" not in result.result.semantic_models