diff --git a/.changes/unreleased/Features-20230907-161831.yaml b/.changes/unreleased/Features-20230907-161831.yaml new file mode 100644 index 00000000000..bf8f64a47b0 --- /dev/null +++ b/.changes/unreleased/Features-20230907-161831.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Use translate_type on data_type in model.columns in templates by default, remove no op `TYPE_LABELS` +time: 2023-09-07T16:18:31.428161-04:00 +custom: + Author: gshank + Issue: "8007" diff --git a/core/dbt/adapters/base/column.py b/core/dbt/adapters/base/column.py index aa0ce7dc63f..50a687c2f5d 100644 --- a/core/dbt/adapters/base/column.py +++ b/core/dbt/adapters/base/column.py @@ -7,12 +7,12 @@ @dataclass class Column: + # Note: This is automatically used by contract code + # No-op conversions (INTEGER => INT) have been removed. + # Any adapter that wants to take advantage of "translate_type" + # should create a ClassVar with the appropriate conversions. TYPE_LABELS: ClassVar[Dict[str, str]] = { "STRING": "TEXT", - "TIMESTAMP": "TIMESTAMP", - "FLOAT": "FLOAT", - "INTEGER": "INT", - "BOOLEAN": "BOOLEAN", } column: str dtype: str diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 996d5027c58..1b838e13b5c 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -1227,12 +1227,24 @@ def graph(self) -> Dict[str, Any]: @contextproperty("model") def ctx_model(self) -> Dict[str, Any]: - ret = self.model.to_dict(omit_none=True) + model_dct = self.model.to_dict(omit_none=True) # Maintain direct use of compiled_sql # TODO add depreciation logic[CT-934] - if "compiled_code" in ret: - ret["compiled_sql"] = ret["compiled_code"] - return ret + if "compiled_code" in model_dct: + model_dct["compiled_sql"] = model_dct["compiled_code"] + + if ( + hasattr(self.model, "contract") + and self.model.contract.alias_types is True + and "columns" in model_dct + ): + for column in model_dct["columns"].values(): + if "data_type" in column: + orig_data_type = column["data_type"] + # translate data_type to value in Column.TYPE_LABELS + new_data_type = self.adapter.Column.translate_type(orig_data_type) + column["data_type"] = new_data_type + return model_dct @contextproperty() def pre_hooks(self) -> Optional[List[Dict[str, Any]]]: diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index f44fee5d50c..300ddc25056 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -202,6 +202,7 @@ def default(cls) -> "OnConfigurationChangeOption": @dataclass class ContractConfig(dbtClassMixin, Replaceable): enforced: bool = False + alias_types: bool = True @dataclass diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 065a60a6d33..f5f15f7de8f 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -232,6 +232,7 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable @dataclass class Contract(dbtClassMixin, Replaceable): enforced: bool = False + alias_types: bool = True checksum: Optional[str] = None diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index dde4aa5035c..17e9218e62a 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -360,7 +360,9 @@ def update_parsed_node_config( # If we have contract in the config, copy to node level if "contract" in config_dict and config_dict["contract"]: - parsed_node.contract = Contract(enforced=config_dict["contract"]["enforced"]) + contract_dct = config_dict["contract"] + Contract.validate(contract_dct) + parsed_node.contract = Contract.from_dict(contract_dct) # unrendered_config is used to compare the original database/schema/alias # values and to handle 'same_config' and 'same_contents' calls diff --git a/schemas/dbt/manifest/v11.json b/schemas/dbt/manifest/v11.json index aafbab125d3..f8f0e6231f5 100644 --- a/schemas/dbt/manifest/v11.json +++ b/schemas/dbt/manifest/v11.json @@ -172,6 +172,10 @@ "enforced": { "type": "boolean", "default": false + }, + "alias_types": { + "type": "boolean", + "default": true } }, "additionalProperties": false @@ -560,6 +564,10 @@ "type": "boolean", "default": false }, + "alias_types": { + "type": "boolean", + "default": true + }, "checksum": { "anyOf": [ { diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index 6082ae4b8d4..6c1c3e9bb4f 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -36,7 +36,7 @@ def get_rendered_model_config(**updates): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "access": "protected", } result.update(updates) @@ -72,7 +72,7 @@ def get_rendered_seed_config(**updates): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, } result.update(updates) return result @@ -112,7 +112,7 @@ def get_rendered_snapshot_config(**updates): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, } result.update(updates) return result @@ -328,7 +328,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "constraints": [], }, }, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "constraints": [], "patch_path": "test://" + model_schema_yml_path, "docs": {"node_color": None, "show": False}, @@ -421,7 +421,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "constraints": [], }, }, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "constraints": [], "patch_path": "test://" + model_schema_yml_path, "docs": {"node_color": None, "show": False}, @@ -564,7 +564,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): }, "checksum": {"name": "none", "checksum": ""}, "unrendered_config": unrendered_test_config, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, }, "snapshot.test.snapshot_seed": { "alias": "snapshot_seed", @@ -576,7 +576,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "compiled": True, "compiled_code": ANY, "config": snapshot_config, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": project.database, "group": None, "deferred": False, @@ -625,7 +625,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "columns": {}, "config": test_config, "group": None, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "depends_on": { "macros": ["macro.test.test_nothing", "macro.dbt.get_where_subquery"], @@ -678,7 +678,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "columns": {}, "config": test_config, "group": None, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], @@ -949,7 +949,7 @@ def expected_references_manifest(project): "unique_id": "model.test.ephemeral_copy", "compiled": True, "compiled_code": ANY, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(ephemeral_copy_path), @@ -985,7 +985,7 @@ def expected_references_manifest(project): }, }, "config": get_rendered_model_config(materialized="table", group="test_group"), - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "depends_on": { "macros": [], @@ -1054,7 +1054,7 @@ def expected_references_manifest(project): }, }, "config": get_rendered_model_config(), - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": project.database, "depends_on": { "macros": [], @@ -1178,7 +1178,7 @@ def expected_references_manifest(project): "compiled": True, "compiled_code": ANY, "config": get_rendered_snapshot_config(target_schema=alternate_schema), - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": model_database, "deferred": False, "depends_on": {"macros": [], "nodes": ["seed.test.seed"]}, @@ -1533,7 +1533,7 @@ def expected_versions_manifest(project): "unique_id": "model.test.versioned_model.v1", "compiled": True, "compiled_code": ANY, - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "extra_ctes_injected": True, "extra_ctes": [], "checksum": checksum_file(versioned_model_v1_path), @@ -1575,7 +1575,7 @@ def expected_versions_manifest(project): materialized="view", group="test_group", meta={"size": "large", "color": "red"} ), "constraints": [], - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "depends_on": {"macros": [], "nodes": []}, "deferred": False, @@ -1622,7 +1622,7 @@ def expected_versions_manifest(project): "columns": {}, "config": get_rendered_model_config(), "constraints": [], - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "database": project.database, "depends_on": { "macros": [], @@ -1683,7 +1683,7 @@ def expected_versions_manifest(project): "columns": {}, "config": test_config, "group": "test_group", - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], @@ -1737,7 +1737,7 @@ def expected_versions_manifest(project): "columns": {}, "config": test_config, "group": "test_group", - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], @@ -1791,7 +1791,7 @@ def expected_versions_manifest(project): "columns": {}, "config": test_config, "group": "test_group", - "contract": {"checksum": None, "enforced": False}, + "contract": {"checksum": None, "enforced": False, "alias_types": True}, "sources": [], "depends_on": { "macros": ["macro.dbt.test_unique", "macro.dbt.get_where_subquery"], diff --git a/tests/functional/configs/test_contract_configs.py b/tests/functional/configs/test_contract_configs.py index 092fcd51824..0a6c6ab803e 100644 --- a/tests/functional/configs/test_contract_configs.py +++ b/tests/functional/configs/test_contract_configs.py @@ -104,7 +104,33 @@ def model(dbt, _): tests: - unique - name: color - data_type: text + data_type: string + - name: date_day + data_type: date +""" + +model_schema_alias_types_false_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + alias_types: false + columns: + - name: id + quote: true + data_type: integer + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: string - name: date_day data_type: date """ @@ -251,7 +277,7 @@ def test__model_contract_true(self, project): assert contract_actual_config.enforced is True - expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True), ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True), ColumnLevelConstraint(type=, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True)], quote=True, tags=[], _extra={}), 'color': ColumnInfo(name='color', description='', meta={}, data_type='text', constraints=[], quote=None, tags=[], _extra={}), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={})}" + expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True), ColumnLevelConstraint(type=, name=None, expression=None, warn_unenforced=True, warn_unsupported=True), ColumnLevelConstraint(type=, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True)], quote=True, tags=[], _extra={}), 'color': ColumnInfo(name='color', description='', meta={}, data_type='string', constraints=[], quote=None, tags=[], _extra={}), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={})}" assert expected_columns == str(my_model_columns) @@ -264,6 +290,15 @@ def test__model_contract_true(self, project): == cleaned_code ) + # set alias_types to false (should fail to compile) + write_file( + model_schema_alias_types_false_yml, + project.project_root, + "models", + "constraints_schema.yml", + ) + run_dbt(["run"], expect_pass=False) + class TestProjectContractEnabledConfigs: @pytest.fixture(scope="class") diff --git a/tests/functional/list/test_list.py b/tests/functional/list/test_list.py index 582258802a3..b9a20214589 100644 --- a/tests/functional/list/test_list.py +++ b/tests/functional/list/test_list.py @@ -98,7 +98,7 @@ def expect_snapshot_output(self, project): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, }, "unique_id": "snapshot.test.my_snapshot", "original_file_path": normalize("snapshots/snapshot.sql"), @@ -140,7 +140,7 @@ def expect_analyses_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, }, "unique_id": "analysis.test.a", "original_file_path": normalize("analyses/a.sql"), @@ -192,7 +192,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "access": "protected", }, "original_file_path": normalize("models/ephemeral.sql"), @@ -230,7 +230,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": "delete+insert", "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "access": "protected", }, "original_file_path": normalize("models/incremental.sql"), @@ -268,7 +268,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "access": "protected", }, "original_file_path": normalize("models/sub/inner.sql"), @@ -306,7 +306,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "access": "protected", }, "original_file_path": normalize("models/metricflow_time_spine.sql"), @@ -344,7 +344,7 @@ def expect_model_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "access": "protected", }, "original_file_path": normalize("models/outer.sql"), @@ -462,7 +462,7 @@ def expect_seed_output(self): "packages": [], "incremental_strategy": None, "docs": {"node_color": None, "show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, }, "depends_on": {"macros": []}, "unique_id": "seed.test.seed", diff --git a/tests/unit/test_contracts_graph_compiled.py b/tests/unit/test_contracts_graph_compiled.py index 1b4f0bbd045..18b2bdea7be 100644 --- a/tests/unit/test_contracts_graph_compiled.py +++ b/tests/unit/test_contracts_graph_compiled.py @@ -204,13 +204,13 @@ def basic_compiled_dict(): "meta": {}, "grants": {}, "packages": [], - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "docs": {"show": True}, "access": "protected", }, "docs": {"show": True}, "columns": {}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "meta": {}, "compiled": True, "extra_ctes": [{"id": "whatever", "sql": "select * from other"}], @@ -554,7 +554,7 @@ def basic_compiled_schema_test_dict(): }, "docs": {"show": True}, "columns": {}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "meta": {}, "compiled": True, "extra_ctes": [{"id": "whatever", "sql": "select * from other"}], diff --git a/tests/unit/test_contracts_graph_parsed.py b/tests/unit/test_contracts_graph_parsed.py index 1ef33149730..de0dbfd9cc5 100644 --- a/tests/unit/test_contracts_graph_parsed.py +++ b/tests/unit/test_contracts_graph_parsed.py @@ -96,7 +96,7 @@ def populated_node_config_dict(): "grants": {}, "packages": [], "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "access": "protected", } @@ -181,13 +181,13 @@ def base_parsed_model_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], "access": "protected", }, "deferred": False, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": { @@ -291,12 +291,12 @@ def complex_parsed_model_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], "access": "protected", }, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": { "a": { "name": "a", @@ -513,7 +513,7 @@ def basic_parsed_seed_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], }, "deferred": False, @@ -606,7 +606,7 @@ def complex_parsed_seed_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], }, "deferred": False, @@ -813,11 +813,11 @@ def base_parsed_hook_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], }, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": { @@ -896,11 +896,11 @@ def complex_parsed_hook_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], }, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": { "a": { "name": "a", @@ -1053,7 +1053,7 @@ def basic_parsed_schema_test_dict(): "schema": "dbt_test__audit", }, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": {}, "test_metadata": { "name": "foo", @@ -1133,7 +1133,7 @@ def complex_parsed_schema_test_dict(): "schema": "dbt_test__audit", }, "docs": {"show": False}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": { "a": { "name": "a", @@ -1254,7 +1254,7 @@ def basic_timestamp_snapshot_config_dict(): "grants": {}, "packages": [], "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, } @@ -1292,7 +1292,7 @@ def complex_timestamp_snapshot_config_dict(): "grants": {}, "packages": [], "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, } @@ -1358,7 +1358,7 @@ def basic_check_snapshot_config_dict(): "grants": {}, "packages": [], "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, } @@ -1396,7 +1396,7 @@ def complex_set_snapshot_config_dict(): "grants": {}, "packages": [], "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, } @@ -1512,11 +1512,11 @@ def basic_timestamp_snapshot_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], }, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": { @@ -1660,11 +1660,11 @@ def basic_check_snapshot_dict(): "meta": {}, "grants": {}, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "packages": [], }, "docs": {"show": True}, - "contract": {"enforced": False}, + "contract": {"enforced": False, "alias_types": True}, "columns": {}, "meta": {}, "checksum": {