Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge semantic model meta config #367

Merged
merged 3 commits into from
Nov 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions dbt_semantic_interfaces/parsing/dir_to_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,17 @@
import traceback
from dataclasses import dataclass
from string import Template
from typing import Dict, List, Optional, Type, Union
from typing import Dict, List, Optional, Sequence, Type, Union

from jsonschema import exceptions

from dbt_semantic_interfaces.errors import ParsingException
from dbt_semantic_interfaces.implementations.element_config import (
PydanticSemanticLayerElementConfig,
)
from dbt_semantic_interfaces.implementations.elements.dimension import PydanticDimension
from dbt_semantic_interfaces.implementations.elements.entity import PydanticEntity
from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure
from dbt_semantic_interfaces.implementations.metric import PydanticMetric
from dbt_semantic_interfaces.implementations.project_configuration import (
PydanticProjectConfiguration,
Expand Down Expand Up @@ -334,7 +340,20 @@ def parse_config_yaml(
results.append(metric_class.parse_obj(object_cfg))
elif document_type == SEMANTIC_MODEL_TYPE:
semantic_model_validator.validate(config_document[document_type])
results.append(semantic_model_class.parse_obj(object_cfg))
sm = semantic_model_class.parse_obj(object_cfg)
# Combine configs according to the behavior documented here https://docs.getdbt.com/reference/configs-and-properties#combining-configs
elements: Sequence[Union[PydanticDimension, PydanticEntity, PydanticMeasure]] = [
*sm.dimensions,
*sm.entities,
*sm.measures,
]
for element in elements:
if sm.config is not None:
if element.config is None:
element.config = PydanticSemanticLayerElementConfig(meta=sm.config.meta)
else:
element.config.meta = {**sm.config.meta, **element.config.meta}
results.append(sm)
elif document_type == PROJECT_CONFIGURATION_TYPE:
project_configuration_validator.validate(config_document[document_type])
results.append(project_configuration_class.parse_obj(object_cfg))
Expand Down
82 changes: 82 additions & 0 deletions tests/parsing/test_semantic_model_parsing.py
Original file line number Diff line number Diff line change
Expand Up @@ -538,3 +538,85 @@ def test_semantic_model_dimension_validity_params_parsing() -> None:
assert end_dimension.type_params.validity_params is not None
assert end_dimension.type_params.validity_params.is_start is False
assert end_dimension.type_params.validity_params.is_end is True


def test_semantic_model_element_config_merging() -> None:
"""Test for merging element config metadata from semantic model into dimension, entity, and measure objects."""
yaml_contents = textwrap.dedent(
"""\
semantic_model:
name: sm
config:
meta:
sm_metadata: asdf
node_relation:
alias: source_table
schema_name: some_schema
dimensions:
- name: dim_0
type: time
type_params:
time_granularity: day
config:
meta:
sm_metadata: qwer
dim_metadata: fdsa
- name: dim_1
type: time
type_params:
time_granularity: day
config:
meta:
dim_metadata: mlkj
sm_metadata: zxcv
- name: dim_2
type: time
type_params:
time_granularity: day
- name: dim_3
type: time
type_params:
time_granularity: day
config:
meta:
dim_metadata: gfds
entities:
- name: entity_0
type: primary
config:
meta:
sm_metadata: hjkl
measures:
- name: measure_0
agg: count_distinct
config:
meta:
sm_metadata: ijkl
"""
)
file = YamlConfigFile(filepath="test_dir/inline_for_test", contents=yaml_contents)

build_result = parse_yaml_files_to_semantic_manifest(files=[file, EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE])

assert len(build_result.semantic_manifest.semantic_models) == 1
semantic_model = build_result.semantic_manifest.semantic_models[0]
assert semantic_model.config is not None
assert semantic_model.config.meta["sm_metadata"] == "asdf"
assert len(semantic_model.dimensions) == 4
assert semantic_model.dimensions[0].config is not None
assert semantic_model.dimensions[0].config.meta["sm_metadata"] == "qwer"
assert semantic_model.dimensions[0].config.meta["dim_metadata"] == "fdsa"
assert semantic_model.dimensions[1].config is not None
assert semantic_model.dimensions[1].config.meta["sm_metadata"] == "zxcv"
assert semantic_model.dimensions[1].config.meta["dim_metadata"] == "mlkj"
assert semantic_model.dimensions[2].config is not None
assert semantic_model.dimensions[2].config.meta["sm_metadata"] == "asdf"
assert semantic_model.dimensions[3].config is not None
assert semantic_model.dimensions[3].config.meta["dim_metadata"] == "gfds"
assert semantic_model.dimensions[3].config.meta["sm_metadata"] == "asdf"
Comment on lines +615 to +616

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These lines cover the test cases I was thinking about here: #367 (review)

Namely, it verifies that it merged the two dictionaries and has the sm_metadata key/value inherited from the semantic model as well as the dim_metadata key/value defined on the dimension.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per discussion with @DevonFulcher, we decided that enumerating similar test cases for entities and measures would be overkill because they are covered by the tests for dimensions.

The reason is that dimensions, entities, and measures are all treated the same here:

                    elements: Sequence[Union[PydanticDimension, PydanticEntity, PydanticMeasure]] = [
                        *sm.dimensions,
                        *sm.entities,
                        *sm.measures,

assert len(semantic_model.entities) == 1
assert semantic_model.entities[0].config is not None
assert semantic_model.entities[0].config.meta["sm_metadata"] == "hjkl"
assert len(semantic_model.measures) == 1
assert semantic_model.measures[0].config is not None
assert semantic_model.measures[0].config.meta["sm_metadata"] == "ijkl"
Loading