From adf0f9de7130dea17b8be3a751010cacc812108b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 26 Nov 2024 13:37:33 -0500 Subject: [PATCH 01/10] always validate in "update_from" --- dbt_common/contracts/config/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dbt_common/contracts/config/base.py b/dbt_common/contracts/config/base.py index b571cb2..840d5e9 100644 --- a/dbt_common/contracts/config/base.py +++ b/dbt_common/contracts/config/base.py @@ -170,9 +170,10 @@ def update_from( dct.update(data) # any validation failures must have come from the update - if validate: - self.validate(dct) - return self.from_dict(dct) + # Always validate, otherwise the "from_dict" will cast and eliminate some errors + self.validate(dct) + obj = self.from_dict(dct) + return obj def finalize_and_validate(self: T) -> T: dct = self.to_dict(omit_none=False) From da68d6951aad0519f1198f4b0be353d708a0cf89 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 26 Nov 2024 14:10:40 -0500 Subject: [PATCH 02/10] Put back validate flag --- dbt_common/contracts/config/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_common/contracts/config/base.py b/dbt_common/contracts/config/base.py index 840d5e9..4e7ae10 100644 --- a/dbt_common/contracts/config/base.py +++ b/dbt_common/contracts/config/base.py @@ -170,8 +170,8 @@ def update_from( dct.update(data) # any validation failures must have come from the update - # Always validate, otherwise the "from_dict" will cast and eliminate some errors - self.validate(dct) + if validate: + self.validate(dct) obj = self.from_dict(dct) return obj From 1fc36bbe2e9d840a51a0efab6aba20adc1b8079c Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 26 Nov 2024 21:45:31 -0500 Subject: [PATCH 03/10] put back --- dbt_common/contracts/config/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt_common/contracts/config/base.py b/dbt_common/contracts/config/base.py index 4e7ae10..b571cb2 100644 --- a/dbt_common/contracts/config/base.py +++ b/dbt_common/contracts/config/base.py @@ -172,8 +172,7 @@ def update_from( # any validation failures must have come from the update if validate: self.validate(dct) - obj = self.from_dict(dct) - return obj + return self.from_dict(dct) def finalize_and_validate(self: T) -> T: dct = self.to_dict(omit_none=False) From b511cc0430ae39068a2833f4e0fa9205ffa69c41 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 2 Dec 2024 17:39:11 -0500 Subject: [PATCH 04/10] Change "update_from" to a class method --- dbt_common/contracts/config/base.py | 46 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/dbt_common/contracts/config/base.py b/dbt_common/contracts/config/base.py index b571cb2..65020a7 100644 --- a/dbt_common/contracts/config/base.py +++ b/dbt_common/contracts/config/base.py @@ -115,6 +115,28 @@ def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> boo "object": ["snapshot_meta_column_names"], } + @classmethod + def update_from( + cls, dct: Dict[str, Any], data: Dict[str, Any], adapter_config_cls: Type[BaseConfig] + ) -> Dict[str, Any]: + """Update and validate config given a dict. + + Given a dict of keys, update the current config from them, validate + it, and return a new config with the updated values + """ + # dct = self.to_dict(omit_none=False) + + self_merged = cls._merge_dicts(dct, data) + dct.update(self_merged) + + adapter_merged = adapter_config_cls._merge_dicts(dct, data) + dct.update(adapter_merged) + + # any remaining fields must be "clobber" + dct.update(data) + + return dct + @classmethod def _merge_dicts(cls, src: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: """Mutate input to return merge results. @@ -150,30 +172,6 @@ def _merge_dicts(cls, src: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, An ) return result - def update_from( - self: T, data: Dict[str, Any], config_cls: Type[BaseConfig], validate: bool = True - ) -> T: - """Update and validate config given a dict. - - Given a dict of keys, update the current config from them, validate - it, and return a new config with the updated values - """ - dct = self.to_dict(omit_none=False) - - self_merged = self._merge_dicts(dct, data) - dct.update(self_merged) - - adapter_merged = config_cls._merge_dicts(dct, data) - dct.update(adapter_merged) - - # any remaining fields must be "clobber" - dct.update(data) - - # any validation failures must have come from the update - if validate: - self.validate(dct) - return self.from_dict(dct) - def finalize_and_validate(self: T) -> T: dct = self.to_dict(omit_none=False) self.validate(dct) From 867bc1067aaab9a7e32bd88c2e8b2e3a4bc42bd3 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 3 Dec 2024 13:43:59 -0500 Subject: [PATCH 05/10] Use mashumaro 3.15 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5fa7fdd..42dfa22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "isodate>=0.6,<0.7", "jsonschema>=4.0,<5.0", "Jinja2>=3.1.3,<4", - "mashumaro[msgpack]>=3.9,<4.0", + "mashumaro[msgpack]>=3.15,<4.0", "pathspec>=0.9,<0.13", "protobuf>=5.0,<6.0", "python-dateutil>=2.0,<3.0", From bfb08a0658ebed6e108dda7c4ff90aa78da68f3d Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 5 Dec 2024 16:48:17 -0500 Subject: [PATCH 06/10] Cleanup --- dbt_common/contracts/config/base.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dbt_common/contracts/config/base.py b/dbt_common/contracts/config/base.py index 65020a7..74e80dc 100644 --- a/dbt_common/contracts/config/base.py +++ b/dbt_common/contracts/config/base.py @@ -117,25 +117,27 @@ def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> boo @classmethod def update_from( - cls, dct: Dict[str, Any], data: Dict[str, Any], adapter_config_cls: Type[BaseConfig] + cls, + orig_dict: Dict[str, Any], + new_dict: Dict[str, Any], + adapter_config_cls: Type[BaseConfig], ) -> Dict[str, Any]: """Update and validate config given a dict. Given a dict of keys, update the current config from them, validate it, and return a new config with the updated values """ - # dct = self.to_dict(omit_none=False) - self_merged = cls._merge_dicts(dct, data) - dct.update(self_merged) + self_merged = cls._merge_dicts(orig_dict, new_dict) + new_dict.update(self_merged) - adapter_merged = adapter_config_cls._merge_dicts(dct, data) - dct.update(adapter_merged) + adapter_merged = adapter_config_cls._merge_dicts(orig_dict, new_dict) + new_dict.update(adapter_merged) # any remaining fields must be "clobber" - dct.update(data) + orig_dict.update(new_dict) - return dct + return orig_dict @classmethod def _merge_dicts(cls, src: Dict[str, Any], data: Dict[str, Any]) -> Dict[str, Any]: From 56b3592e06b81baadfe3a827e1b5b9b4923f0103 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 5 Dec 2024 18:19:41 -0500 Subject: [PATCH 07/10] Fix unit test --- tests/unit/test_model_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_model_config.py b/tests/unit/test_model_config.py index 9d1de1b..d914213 100644 --- a/tests/unit/test_model_config.py +++ b/tests/unit/test_model_config.py @@ -90,7 +90,10 @@ def test_update_from() -> None: "grants": {"two": "fine", "+one": "some"}, "snapshot_table_column_names": {"first_column": "dbt_ack", "second_column": "dbt_more"}, } - updated_obj = initial_obj.update_from(update_dct.copy(), SubstituteAdapterConfig) + updated_dict = initial_obj.update_from( + initial_obj.to_dict(), update_dct.copy(), SubstituteAdapterConfig + ) + updated_obj = ThingWithMergeBehavior.from_dict(updated_dict) assert updated_obj.default_behavior == 3 assert updated_obj.tags == ["one", "two", "five"] From c3f3284fa9b12c1c91c1493299b47e4bff37c414 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 12 Dec 2024 10:03:16 -0500 Subject: [PATCH 08/10] Bump down mashumaro requirement --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42dfa22..38199cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "isodate>=0.6,<0.7", "jsonschema>=4.0,<5.0", "Jinja2>=3.1.3,<4", - "mashumaro[msgpack]>=3.15,<4.0", + "mashumaro[msgpack]>=3.9,<3.15", "pathspec>=0.9,<0.13", "protobuf>=5.0,<6.0", "python-dateutil>=2.0,<3.0", From b7dd3aed99d18d645fd34cf3d3bfc2a775608173 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 19 Dec 2024 13:17:36 -0500 Subject: [PATCH 09/10] Revert "Bump down mashumaro requirement" This reverts commit c3f3284fa9b12c1c91c1493299b47e4bff37c414. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 38199cf..42dfa22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "isodate>=0.6,<0.7", "jsonschema>=4.0,<5.0", "Jinja2>=3.1.3,<4", - "mashumaro[msgpack]>=3.9,<3.15", + "mashumaro[msgpack]>=3.15,<4.0", "pathspec>=0.9,<0.13", "protobuf>=5.0,<6.0", "python-dateutil>=2.0,<3.0", From d77bfa8dbd5a369c9ae7f0efb07c110629d0b7a6 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 19 Dec 2024 13:22:44 -0500 Subject: [PATCH 10/10] Add mashumaro no_copy_collection() to dialect --- dbt_common/dataclass_schema.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbt_common/dataclass_schema.py b/dbt_common/dataclass_schema.py index 0aad4d5..1c5b515 100644 --- a/dbt_common/dataclass_schema.py +++ b/dbt_common/dataclass_schema.py @@ -15,11 +15,15 @@ from mashumaro.jsonschema import build_json_schema # following includes DataClassDictMixin -from mashumaro.mixins.msgpack import DataClassMessagePackMixin +from mashumaro.mixins.msgpack import DataClassMessagePackMixin, MessagePackDialect import functools +class FixedMessagePackDialect(MessagePackDialect): + no_copy_collections = () + + class ValidationError(jsonschema.ValidationError): pass @@ -49,6 +53,7 @@ class dbtMashConfig(MashBaseConfig): } serialize_by_alias = True lazy_compilation = True + dialect = FixedMessagePackDialect # This class pulls in DataClassDictMixin from Mashumaro. The 'to_dict'