From e436c822609c34204ed1392e8557ea4c9dc12dad Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Fri, 7 Jun 2024 19:32:18 +0000 Subject: [PATCH 1/5] Use alias instead of name when adding ephemeral model prefixes --- core/dbt/compilation.py | 6 +++--- core/dbt/context/providers.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index d03407b2a4c..50b3a613145 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -294,10 +294,10 @@ def _create_node_context( return context - def add_ephemeral_prefix(self, name: str): + def add_ephemeral_prefix(self, alias: str): adapter = get_adapter(self.config) relation_cls = adapter.Relation - return relation_cls.add_ephemeral_prefix(name) + return relation_cls.add_ephemeral_prefix(alias) def _recursively_prepend_ctes( self, @@ -371,7 +371,7 @@ def _recursively_prepend_ctes( _extend_prepended_ctes(prepended_ctes, new_prepended_ctes) - new_cte_name = self.add_ephemeral_prefix(cte_model.name) + new_cte_name = self.add_ephemeral_prefix(cte_model.alias) rendered_sql = cte_model._pre_injected_sql or cte_model.compiled_code sql = f" {new_cte_name} as (\n{rendered_sql}\n)" diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 15be73b535d..241ffdb6373 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -1599,7 +1599,7 @@ def this(self) -> Optional[str]: if self.model.this_input_node_unique_id: this_node = self.manifest.expect(self.model.this_input_node_unique_id) self.model.set_cte(this_node.unique_id, None) # type: ignore - return self.adapter.Relation.add_ephemeral_prefix(this_node.name) + return self.adapter.Relation.add_ephemeral_prefix(this_node.alias) return None From f81fe6fc5f46b0adc8cd5a1711b4aac43b7c63d1 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Fri, 7 Jun 2024 21:55:23 +0000 Subject: [PATCH 2/5] Adjust TestCustomSchemaWithCustomMacroFromModelName to test ephemeral models --- tests/functional/schema/fixtures/sql.py | 10 +++++++++- tests/functional/schema/test_custom_schema.py | 8 ++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/functional/schema/fixtures/sql.py b/tests/functional/schema/fixtures/sql.py index ca9fe2a0cdf..daed6d0c741 100644 --- a/tests/functional/schema/fixtures/sql.py +++ b/tests/functional/schema/fixtures/sql.py @@ -18,7 +18,15 @@ ) _TABLE_TWO_DOT_MODEL_SCHEMA = "second_schema" _TABLE_TWO_DOT_MODEL_NAME = f"{_TABLE_TWO_DOT_MODEL_SCHEMA}.view_2" -_TABLE_TWO_DOT_MODEL = "select * from {{ ref('" + _TABLE_ONE_DOT_MODEL_NAME + "') }}" +_TABLE_TWO_DOT_MODEL = ( + """ +{{ config(materialized='ephemeral') }} + +select * from {{ ref('""" + + _TABLE_ONE_DOT_MODEL_NAME + + """') }} +""" +) _TABLE_THREE_SCHEMA = "test" _TABLE_THREE = ( diff --git a/tests/functional/schema/test_custom_schema.py b/tests/functional/schema/test_custom_schema.py index 5a9969e4284..5a9999abbde 100644 --- a/tests/functional/schema/test_custom_schema.py +++ b/tests/functional/schema/test_custom_schema.py @@ -17,7 +17,6 @@ _TABLE_TWO, _TABLE_TWO_DOT_MODEL, _TABLE_TWO_DOT_MODEL_NAME, - _TABLE_TWO_DOT_MODEL_SCHEMA, _TABLE_TWO_SCHEMA, _VALIDATION_SQL, ) @@ -202,18 +201,15 @@ def test__postgres__custom_schema_from_model_name( project.run_sql(_VALIDATION_SQL) run_dbt(["seed"]) results = run_dbt(["run"]) - assert len(results) == 3 + # Model 2 is ephemeral, so it should not show up in the results list + assert len(results) == 2 table_results = {r.node.name: r.node.schema for r in results.results} assert table_results[_TABLE_ONE_DOT_MODEL_NAME] == _TABLE_ONE_DOT_MODEL_SCHEMA - assert table_results[_TABLE_TWO_DOT_MODEL_NAME] == _TABLE_TWO_DOT_MODEL_SCHEMA assert table_results["table_3"] == f"{project.test_schema}" check_relations_equal( adapter=project.adapter, relation_names=("seed", _TABLE_ONE_DOT_MODEL_NAME) ) - check_relations_equal( - adapter=project.adapter, relation_names=("seed", _TABLE_TWO_DOT_MODEL_NAME) - ) check_relations_equal( adapter=project.adapter, relation_names=("agg", f"{project.test_schema}.table_3") ) From ddaeffc3b68660e579a076473058722c4ed87e38 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 10 Jun 2024 20:05:41 +0000 Subject: [PATCH 3/5] Add changelog entry for ephemeral model CTE identifier fix --- .changes/unreleased/Fixes-20240610-200522.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240610-200522.yaml diff --git a/.changes/unreleased/Fixes-20240610-200522.yaml b/.changes/unreleased/Fixes-20240610-200522.yaml new file mode 100644 index 00000000000..456575644ac --- /dev/null +++ b/.changes/unreleased/Fixes-20240610-200522.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use model alias for the CTE identifier generated during ephemeral materialization +time: 2024-06-10T20:05:22.510814008Z +custom: + Author: jeancochrane + Issue: "5273" From 582f972c5e92e439eac51e89679454d2457fab71 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Tue, 16 Jul 2024 17:17:47 -0500 Subject: [PATCH 4/5] Reference model.identifier and model.name where appropriate to resolve typing errors --- core/dbt/compilation.py | 6 +++--- core/dbt/context/providers.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index 50b3a613145..08de60490d3 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -294,10 +294,10 @@ def _create_node_context( return context - def add_ephemeral_prefix(self, alias: str): + def add_ephemeral_prefix(self, name: str): adapter = get_adapter(self.config) relation_cls = adapter.Relation - return relation_cls.add_ephemeral_prefix(alias) + return relation_cls.add_ephemeral_prefix(name) def _recursively_prepend_ctes( self, @@ -371,7 +371,7 @@ def _recursively_prepend_ctes( _extend_prepended_ctes(prepended_ctes, new_prepended_ctes) - new_cte_name = self.add_ephemeral_prefix(cte_model.alias) + new_cte_name = self.add_ephemeral_prefix(cte_model.identifier) rendered_sql = cte_model._pre_injected_sql or cte_model.compiled_code sql = f" {new_cte_name} as (\n{rendered_sql}\n)" diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 241ffdb6373..15be73b535d 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -1599,7 +1599,7 @@ def this(self) -> Optional[str]: if self.model.this_input_node_unique_id: this_node = self.manifest.expect(self.model.this_input_node_unique_id) self.model.set_cte(this_node.unique_id, None) # type: ignore - return self.adapter.Relation.add_ephemeral_prefix(this_node.alias) + return self.adapter.Relation.add_ephemeral_prefix(this_node.name) return None From 5da895c2f3174d1fc5f310b532b441c18f3efcc5 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 17 Jul 2024 10:55:52 -0500 Subject: [PATCH 5/5] Move test for ephemeral model with alias to dedicated test in test_compile.py --- tests/functional/compile/fixtures.py | 9 +++++++++ tests/functional/compile/test_compile.py | 20 +++++++++++++++++++ tests/functional/schema/fixtures/sql.py | 10 +--------- tests/functional/schema/test_custom_schema.py | 8 ++++++-- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/functional/compile/fixtures.py b/tests/functional/compile/fixtures.py index e0be7c895bf..97ccd6d16b6 100644 --- a/tests/functional/compile/fixtures.py +++ b/tests/functional/compile/fixtures.py @@ -42,6 +42,15 @@ select sum(n) from t; """ +first_ephemeral_model_with_alias_sql = """ +{{ config(materialized = 'ephemeral', alias = 'first_alias') }} +select 1 as fun +""" + +second_ephemeral_model_with_alias_sql = """ +select * from {{ ref('first_ephemeral_model_with_alias') }} +""" + schema_yml = """ version: 2 diff --git a/tests/functional/compile/test_compile.py b/tests/functional/compile/test_compile.py index e7732b09a8c..956169eddf1 100644 --- a/tests/functional/compile/test_compile.py +++ b/tests/functional/compile/test_compile.py @@ -10,10 +10,12 @@ from tests.functional.assertions.test_runner import dbtTestRunner from tests.functional.compile.fixtures import ( first_ephemeral_model_sql, + first_ephemeral_model_with_alias_sql, first_model_sql, model_multiline_jinja, schema_yml, second_ephemeral_model_sql, + second_ephemeral_model_with_alias_sql, second_model_sql, third_ephemeral_model_sql, with_recursive_model_sql, @@ -128,6 +130,24 @@ def test_with_recursive_cte(self, project): ] +class TestEphemeralModelWithAlias: + @pytest.fixture(scope="class") + def models(self): + return { + "first_ephemeral_model_with_alias.sql": first_ephemeral_model_with_alias_sql, + "second_ephemeral_model_with_alias.sql": second_ephemeral_model_with_alias_sql, + } + + def test_compile(self, project): + run_dbt(["compile"]) + + assert get_lines("second_ephemeral_model_with_alias") == [ + "with __dbt__cte__first_alias as (", + "select 1 as fun", + ") select * from __dbt__cte__first_alias", + ] + + class TestCompile: @pytest.fixture(scope="class") def models(self): diff --git a/tests/functional/schema/fixtures/sql.py b/tests/functional/schema/fixtures/sql.py index daed6d0c741..ca9fe2a0cdf 100644 --- a/tests/functional/schema/fixtures/sql.py +++ b/tests/functional/schema/fixtures/sql.py @@ -18,15 +18,7 @@ ) _TABLE_TWO_DOT_MODEL_SCHEMA = "second_schema" _TABLE_TWO_DOT_MODEL_NAME = f"{_TABLE_TWO_DOT_MODEL_SCHEMA}.view_2" -_TABLE_TWO_DOT_MODEL = ( - """ -{{ config(materialized='ephemeral') }} - -select * from {{ ref('""" - + _TABLE_ONE_DOT_MODEL_NAME - + """') }} -""" -) +_TABLE_TWO_DOT_MODEL = "select * from {{ ref('" + _TABLE_ONE_DOT_MODEL_NAME + "') }}" _TABLE_THREE_SCHEMA = "test" _TABLE_THREE = ( diff --git a/tests/functional/schema/test_custom_schema.py b/tests/functional/schema/test_custom_schema.py index 5a9999abbde..5a9969e4284 100644 --- a/tests/functional/schema/test_custom_schema.py +++ b/tests/functional/schema/test_custom_schema.py @@ -17,6 +17,7 @@ _TABLE_TWO, _TABLE_TWO_DOT_MODEL, _TABLE_TWO_DOT_MODEL_NAME, + _TABLE_TWO_DOT_MODEL_SCHEMA, _TABLE_TWO_SCHEMA, _VALIDATION_SQL, ) @@ -201,15 +202,18 @@ def test__postgres__custom_schema_from_model_name( project.run_sql(_VALIDATION_SQL) run_dbt(["seed"]) results = run_dbt(["run"]) - # Model 2 is ephemeral, so it should not show up in the results list - assert len(results) == 2 + assert len(results) == 3 table_results = {r.node.name: r.node.schema for r in results.results} assert table_results[_TABLE_ONE_DOT_MODEL_NAME] == _TABLE_ONE_DOT_MODEL_SCHEMA + assert table_results[_TABLE_TWO_DOT_MODEL_NAME] == _TABLE_TWO_DOT_MODEL_SCHEMA assert table_results["table_3"] == f"{project.test_schema}" check_relations_equal( adapter=project.adapter, relation_names=("seed", _TABLE_ONE_DOT_MODEL_NAME) ) + check_relations_equal( + adapter=project.adapter, relation_names=("seed", _TABLE_TWO_DOT_MODEL_NAME) + ) check_relations_equal( adapter=project.adapter, relation_names=("agg", f"{project.test_schema}.table_3") )