From c1641de0aed72d9bcf386c4324cbad261ced6a7c Mon Sep 17 00:00:00 2001 From: Devon Fulcher <24593113+DevonFulcher@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:33:16 -0600 Subject: [PATCH 1/2] Merge semantic model meta config --- .../parsing/dir_to_model.py | 23 +++++- tests/parsing/test_semantic_model_parsing.py | 72 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/dbt_semantic_interfaces/parsing/dir_to_model.py b/dbt_semantic_interfaces/parsing/dir_to_model.py index 418ca254..c347b8a4 100644 --- a/dbt_semantic_interfaces/parsing/dir_to_model.py +++ b/dbt_semantic_interfaces/parsing/dir_to_model.py @@ -3,11 +3,17 @@ import traceback from dataclasses import dataclass from string import Template -from typing import Dict, List, Optional, Type, Union +from typing import Dict, List, Optional, Sequence, Type, Union from jsonschema import exceptions from dbt_semantic_interfaces.errors import ParsingException +from dbt_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) +from dbt_semantic_interfaces.implementations.elements.dimension import PydanticDimension +from dbt_semantic_interfaces.implementations.elements.entity import PydanticEntity +from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure from dbt_semantic_interfaces.implementations.metric import PydanticMetric from dbt_semantic_interfaces.implementations.project_configuration import ( PydanticProjectConfiguration, @@ -334,7 +340,20 @@ def parse_config_yaml( results.append(metric_class.parse_obj(object_cfg)) elif document_type == SEMANTIC_MODEL_TYPE: semantic_model_validator.validate(config_document[document_type]) - results.append(semantic_model_class.parse_obj(object_cfg)) + sm = semantic_model_class.parse_obj(object_cfg) + # Combine configs according to the behavior documented here https://docs.getdbt.com/reference/configs-and-properties#combining-configs + elements: Sequence[Union[PydanticDimension, PydanticEntity, PydanticMeasure]] = [ + *sm.dimensions, + *sm.entities, + *sm.measures, + ] + for element in elements: + if sm.config is not None: + if element.config is None: + element.config = PydanticSemanticLayerElementConfig(meta=sm.config.meta) + else: + element.config.meta = {**sm.config.meta, **element.config.meta} + results.append(sm) elif document_type == PROJECT_CONFIGURATION_TYPE: project_configuration_validator.validate(config_document[document_type]) results.append(project_configuration_class.parse_obj(object_cfg)) diff --git a/tests/parsing/test_semantic_model_parsing.py b/tests/parsing/test_semantic_model_parsing.py index 34ad851f..631dd634 100644 --- a/tests/parsing/test_semantic_model_parsing.py +++ b/tests/parsing/test_semantic_model_parsing.py @@ -538,3 +538,75 @@ def test_semantic_model_dimension_validity_params_parsing() -> None: assert end_dimension.type_params.validity_params is not None assert end_dimension.type_params.validity_params.is_start is False assert end_dimension.type_params.validity_params.is_end is True + + +def test_semantic_model_element_config_merging() -> None: + """Test for merging element config metadata from semantic model into dimension, entity, and measure objects.""" + yaml_contents = textwrap.dedent( + """\ + semantic_model: + name: sm + config: + meta: + sm_metadata: asdf + node_relation: + alias: source_table + schema_name: some_schema + dimensions: + - name: dim_0 + type: time + type_params: + time_granularity: day + config: + meta: + sm_metadata: qwer + dim_metadata: fdsa + - name: dim_1 + type: time + type_params: + time_granularity: day + config: + meta: + dim_metadata: mlkj + sm_metadata: zxcv + - name: dim_2 + type: time + type_params: + time_granularity: day + entities: + - name: entity_0 + type: primary + config: + meta: + sm_metadata: hjkl + measures: + - name: measure_0 + agg: count_distinct + config: + meta: + sm_metadata: ijkl + """ + ) + file = YamlConfigFile(filepath="test_dir/inline_for_test", contents=yaml_contents) + + build_result = parse_yaml_files_to_semantic_manifest(files=[file, EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE]) + + assert len(build_result.semantic_manifest.semantic_models) == 1 + semantic_model = build_result.semantic_manifest.semantic_models[0] + assert semantic_model.config is not None + assert semantic_model.config.meta["sm_metadata"] == "asdf" + assert len(semantic_model.dimensions) == 3 + assert semantic_model.dimensions[0].config is not None + assert semantic_model.dimensions[0].config.meta["sm_metadata"] == "qwer" + assert semantic_model.dimensions[0].config.meta["dim_metadata"] == "fdsa" + assert semantic_model.dimensions[1].config is not None + assert semantic_model.dimensions[1].config.meta["sm_metadata"] == "zxcv" + assert semantic_model.dimensions[1].config.meta["dim_metadata"] == "mlkj" + assert semantic_model.dimensions[2].config is not None + assert semantic_model.dimensions[2].config.meta["sm_metadata"] == "asdf" + assert len(semantic_model.entities) == 1 + assert semantic_model.entities[0].config is not None + assert semantic_model.entities[0].config.meta["sm_metadata"] == "hjkl" + assert len(semantic_model.measures) == 1 + assert semantic_model.measures[0].config is not None + assert semantic_model.measures[0].config.meta["sm_metadata"] == "ijkl" From 1c54b875474e108c2c7eeb4bc04a9a7d7c538a07 Mon Sep 17 00:00:00 2001 From: Devon Fulcher <24593113+DevonFulcher@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:22:12 -0600 Subject: [PATCH 2/2] Added test assertion for merging meta --- tests/parsing/test_semantic_model_parsing.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/parsing/test_semantic_model_parsing.py b/tests/parsing/test_semantic_model_parsing.py index 631dd634..61839360 100644 --- a/tests/parsing/test_semantic_model_parsing.py +++ b/tests/parsing/test_semantic_model_parsing.py @@ -573,6 +573,13 @@ def test_semantic_model_element_config_merging() -> None: type: time type_params: time_granularity: day + - name: dim_3 + type: time + type_params: + time_granularity: day + config: + meta: + dim_metadata: gfds entities: - name: entity_0 type: primary @@ -595,7 +602,7 @@ def test_semantic_model_element_config_merging() -> None: semantic_model = build_result.semantic_manifest.semantic_models[0] assert semantic_model.config is not None assert semantic_model.config.meta["sm_metadata"] == "asdf" - assert len(semantic_model.dimensions) == 3 + assert len(semantic_model.dimensions) == 4 assert semantic_model.dimensions[0].config is not None assert semantic_model.dimensions[0].config.meta["sm_metadata"] == "qwer" assert semantic_model.dimensions[0].config.meta["dim_metadata"] == "fdsa" @@ -604,6 +611,9 @@ def test_semantic_model_element_config_merging() -> None: assert semantic_model.dimensions[1].config.meta["dim_metadata"] == "mlkj" assert semantic_model.dimensions[2].config is not None assert semantic_model.dimensions[2].config.meta["sm_metadata"] == "asdf" + assert semantic_model.dimensions[3].config is not None + assert semantic_model.dimensions[3].config.meta["dim_metadata"] == "gfds" + assert semantic_model.dimensions[3].config.meta["sm_metadata"] == "asdf" assert len(semantic_model.entities) == 1 assert semantic_model.entities[0].config is not None assert semantic_model.entities[0].config.meta["sm_metadata"] == "hjkl"