From e595857d7b7aa0054388343262927cfa5c7a7f2e Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 26 Sep 2023 00:13:17 +0100 Subject: [PATCH 1/4] Fix #8509: Support doc blocks in nested semantic model YAML --- core/dbt/parser/manifest.py | 29 +++++++++++++ core/dbt/parser/schema_renderer.py | 15 +------ tests/functional/semantic_models/fixtures.py | 43 ++++++++++++++++++- .../semantic_models/test_semantic_models.py | 25 ++++++++++- 4 files changed, 95 insertions(+), 17 deletions(-) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 4398fa75bc7..fbf2adde746 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1172,6 +1172,16 @@ def process_docs(self, config: RuntimeConfig): config.project_name, ) _process_docs_for_metrics(ctx, metric) + for semantic_model in self.manifest.semantic_models.values(): + if semantic_model.created_at < self.started_at: + continue + ctx = generate_runtime_docs_context( + config, + semantic_model, + self.manifest, + config.project_name, + ) + _process_docs_for_semantic_model(ctx, semantic_model) # Loops through all nodes and exposures, for each element in # 'sources' array finds the source node and updates the @@ -1406,6 +1416,25 @@ def _process_docs_for_metrics(context: Dict[str, Any], metric: Metric) -> None: metric.description = get_rendered(metric.description, context) +def _process_docs_for_semantic_model( + context: Dict[str, Any], semantic_model: SemanticModel +) -> None: + if semantic_model.description: + semantic_model.description = get_rendered(semantic_model.description, context) + + for dimension in semantic_model.dimensions: + if dimension.description: + dimension.description = get_rendered(dimension.description, context) + + for measure in semantic_model.measures: + if measure.description: + measure.description = get_rendered(measure.description, context) + + for entity in semantic_model.entities: + if entity.description: + entity.description = get_rendered(entity.description, context) + + def _process_refs( manifest: Manifest, current_project: str, node, dependencies: Optional[Mapping[str, Project]] ) -> None: diff --git a/core/dbt/parser/schema_renderer.py b/core/dbt/parser/schema_renderer.py index e0c54f247da..0ab7d622bf1 100644 --- a/core/dbt/parser/schema_renderer.py +++ b/core/dbt/parser/schema_renderer.py @@ -34,20 +34,7 @@ def _is_norender_key(self, keypath: Keypath) -> bool: Return True if it's tests or description - those aren't rendered now because they're rendered later in parse_generic_tests or process_docs. """ - if len(keypath) >= 1 and keypath[0] in ("tests", "description"): - return True - - if len(keypath) == 2 and keypath[1] in ("tests", "description"): - return True - - if ( - len(keypath) >= 3 - and keypath[0] == "columns" - and keypath[2] in ("tests", "description") - ): - return True - - return False + return keypath[-1] in ("tests", "description") # don't render descriptions or test keyword arguments def should_render_keypath(self, keypath: Keypath) -> bool: diff --git a/tests/functional/semantic_models/fixtures.py b/tests/functional/semantic_models/fixtures.py index d19ba42f1a6..81b8e64932b 100644 --- a/tests/functional/semantic_models/fixtures.py +++ b/tests/functional/semantic_models/fixtures.py @@ -86,6 +86,44 @@ agg_time_dimension: created_at """ +semantic_model_descriptions = """ +{% docs semantic_model_description %} foo {% enddocs %} +{% docs dimension_description %} bar {% enddocs %} +{% docs measure_description %} baz {% enddocs %} +{% docs entity_description %} qux {% enddocs %} +""" + +semantic_model_people_yml_with_docs = """ +version: 2 + +semantic_models: + - name: semantic_people + model: ref('people') + description: "{{ doc('semantic_model_description') }}" + dimensions: + - name: favorite_color + type: categorical + description: "{{ doc('dimension_description') }}" + - name: created_at + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + agg: SUM + expr: tenure + description: "{{ doc('measure_description') }}" + - name: people + agg: count + expr: id + entities: + - name: id + description: "{{ doc('entity_description') }}" + type: primary + defaults: + agg_time_dimension: created_at +""" + enabled_semantic_model_people_yml = """ version: 2 @@ -218,7 +256,10 @@ schema_without_semantic_model_yml = """models: - name: fct_revenue - description: This is the model fct_revenue. It should be able to use doc blocks + description: "{{ doc('dimension_description') }}" + columns: + - name: revenue + description: "{{ doc('dimension_description') }}" """ fct_revenue_sql = """select diff --git a/tests/functional/semantic_models/test_semantic_models.py b/tests/functional/semantic_models/test_semantic_models.py index 3d3be94b341..11fdfc32456 100644 --- a/tests/functional/semantic_models/test_semantic_models.py +++ b/tests/functional/semantic_models/test_semantic_models.py @@ -3,13 +3,13 @@ from dbt.contracts.graph.manifest import Manifest from dbt.exceptions import CompilationError from dbt.tests.util import run_dbt - - from tests.functional.semantic_models.fixtures import ( models_people_sql, simple_metricflow_time_spine_sql, semantic_model_people_yml, models_people_metrics_yml, + semantic_model_people_yml_with_docs, + semantic_model_descriptions, ) @@ -36,6 +36,27 @@ def test_depends_on(self, project): ) +class TestSemanticModelNestedDocs: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": simple_metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_people_yml_with_docs, + "people_metrics.yml": models_people_metrics_yml, + "docs.md": semantic_model_descriptions, + } + + def test_depends_on(self, project): + manifest = run_dbt(["parse"]) + node = manifest.semantic_models["semantic_model.test.semantic_people"] + + assert node.description == "foo" + assert node.dimensions[0].description == "bar" + assert node.measures[0].description == "baz" + assert node.entities[0].description == "qux" + + class TestSemanticModelUnknownModel: @pytest.fixture(scope="class") def models(self): From 11d819582bfb7bb9460388a4a79ea53a4ebbc70d Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 26 Sep 2023 00:15:50 +0100 Subject: [PATCH 2/4] add changie --- .changes/unreleased/Fixes-20230926-001527.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20230926-001527.yaml diff --git a/.changes/unreleased/Fixes-20230926-001527.yaml b/.changes/unreleased/Fixes-20230926-001527.yaml new file mode 100644 index 00000000000..53d6b9151fd --- /dev/null +++ b/.changes/unreleased/Fixes-20230926-001527.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Support doc blocks in nested semantic model YAML +time: 2023-09-26T00:15:27.328363+01:00 +custom: + Author: aranke + Issue: "8509" From 58aee26d39beee55553393315c526f6a34114835 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 26 Sep 2023 00:18:48 +0100 Subject: [PATCH 3/4] revert schema_without_semantic_model_yml --- tests/functional/semantic_models/fixtures.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/functional/semantic_models/fixtures.py b/tests/functional/semantic_models/fixtures.py index 81b8e64932b..3fb65a3a4fb 100644 --- a/tests/functional/semantic_models/fixtures.py +++ b/tests/functional/semantic_models/fixtures.py @@ -256,10 +256,7 @@ schema_without_semantic_model_yml = """models: - name: fct_revenue - description: "{{ doc('dimension_description') }}" - columns: - - name: revenue - description: "{{ doc('dimension_description') }}" + description: This is the model fct_revenue. It should be able to use doc blocks """ fct_revenue_sql = """select From eb253b4563ee323162ab2fd1972541ba1d2259f0 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 26 Sep 2023 03:39:00 +0100 Subject: [PATCH 4/4] revert clever commit --- core/dbt/parser/schema_renderer.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/core/dbt/parser/schema_renderer.py b/core/dbt/parser/schema_renderer.py index 0ab7d622bf1..66b91fee1b4 100644 --- a/core/dbt/parser/schema_renderer.py +++ b/core/dbt/parser/schema_renderer.py @@ -34,7 +34,20 @@ def _is_norender_key(self, keypath: Keypath) -> bool: Return True if it's tests or description - those aren't rendered now because they're rendered later in parse_generic_tests or process_docs. """ - return keypath[-1] in ("tests", "description") + if len(keypath) >= 1 and keypath[0] in ("tests", "description"): + return True + + if len(keypath) == 2 and keypath[1] in ("tests", "description"): + return True + + if ( + len(keypath) >= 3 + and keypath[0] in ("columns", "dimensions", "measures", "entities") + and keypath[2] in ("tests", "description") + ): + return True + + return False # don't render descriptions or test keyword arguments def should_render_keypath(self, keypath: Keypath) -> bool: