diff --git a/.spelling b/.spelling index 8efb3ebb52..405acc5c29 100644 --- a/.spelling +++ b/.spelling @@ -3,6 +3,7 @@ # global dictionary is at the start, file overrides afterwards # one word per line, to define a file override use ' - filename' # where filename is relative to this configuration file +fraxtal Flashbots liveness pre-shared diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py index 568194e311..838c02b086 100644 --- a/aea/configurations/validation.py +++ b/aea/configurations/validation.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022-2023 Valory AG +# Copyright 2022-2024 Valory AG # Copyright 2018-2021 Fetch.AI Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,13 @@ # # ------------------------------------------------------------------------------ """Implementation of the configuration validation.""" + import inspect import json import os from collections import OrderedDict from copy import deepcopy +from functools import reduce from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple @@ -36,7 +38,7 @@ from aea.configurations.constants import AGENT from aea.configurations.data_types import ComponentId, ComponentType, PublicId from aea.exceptions import AEAValidationError -from aea.helpers.base import dict_to_path_value +from aea.helpers.base import dict_to_path_value, update_nested_dict from aea.helpers.env_vars import is_env_variable from aea.helpers.io import open_file @@ -295,12 +297,26 @@ def validate_data_with_pattern( excludes_: List[Tuple[str]] = [] else: excludes_ = excludes - pattern_path_value = { + original_config = { tuple(path): value for path, value in dict_to_path_value(pattern) } - data_path_value = {tuple(path): value for path, value in dict_to_path_value(data)} + overrides = {tuple(path): value for path, value in dict_to_path_value(data)} errors = [] + # this is a workaround to fix the type of numeric keys as they can only be represented as strs in the json overrides + for path in original_config: + path_as_str = tuple(map(str, path)) + if path_as_str in overrides and path not in overrides: + value = overrides[path_as_str] + del overrides[path_as_str] + up_to_last_key = data + for key in path_as_str[:-1]: + up_to_last_key = up_to_last_key[key] + del up_to_last_key[path_as_str[-1]] + overrides[path] = value + vals = reduce(lambda d, key: {key: d}, reversed(path), value) + update_nested_dict(data, vals) + def check_excludes(path: Tuple[str, ...]) -> bool: for exclude in excludes_: if len(exclude) > len(path): # pragma: nocover @@ -315,17 +331,17 @@ def is_a_dict_override(path: Tuple[str, ...]) -> bool: flag = False while len(path) > 0: path = path[:-1] - if path in pattern_path_value: - pattern_value = pattern_path_value[path] + if path in original_config: + pattern_value = original_config[path] flag = isinstance(pattern_value, OrderedDict) break return flag - for path, new_value in data_path_value.items(): + for path, new_value in overrides.items(): if check_excludes(path): continue - if path not in pattern_path_value: + if path not in original_config: if not is_a_dict_override(path=(*path,)): errors.append( f"Attribute `{'.'.join(path)}` is not allowed to be updated!" @@ -333,7 +349,7 @@ def is_a_dict_override(path: Tuple[str, ...]) -> bool: continue - pattern_value = pattern_path_value[path] + pattern_value = original_config[path] if pattern_value is None: # not possible to determine data type for optional value not set diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 0594494d7a..785989bfd8 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022-2023 Valory AG +# Copyright 2022-2024 Valory AG # Copyright 2018-2021 Fetch.AI Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -711,10 +711,14 @@ def dict_to_path_value( """Convert dict to sequence of terminal path build of keys and value.""" path = path or [] for key, value in data.items(): + # terminal value if isinstance(value, Mapping) and value: - # terminal value + # yielding here allows for higher level dict overriding + yield path + [key], value + # recursing to the next level of the dict for p, v in dict_to_path_value(value, path + [key]): yield p, v + # non-terminal value else: yield path + [key], value @@ -1087,3 +1091,13 @@ def prepend_if_not_absolute(path: PathLike, prefix: PathLike) -> PathLike: :return: the same path if absolute, else the prepended path. """ return path if Path(path).is_absolute() else Path(prefix) / path + + +def update_nested_dict(dict_: dict, nested_update: dict) -> dict: + """Update a nested dictionary.""" + for key, value in nested_update.items(): + if isinstance(value, dict): + dict_[key] = update_nested_dict(dict_.get(key, {}), value) + else: + dict_[key] = value + return dict_ diff --git a/docs/api/helpers/base.md b/docs/api/helpers/base.md index 64f5247146..64fc16e130 100644 --- a/docs/api/helpers/base.md +++ b/docs/api/helpers/base.md @@ -838,3 +838,13 @@ Prepend a path with a prefix, but only if not absolute the same path if absolute, else the prepended path. + + +#### update`_`nested`_`dict + +```python +def update_nested_dict(dict_: dict, nested_update: dict) -> dict +``` + +Update a nested dictionary. + diff --git a/tests/test_cli/test_config.py b/tests/test_cli/test_config.py index e896557c4d..c1aa7971bb 100644 --- a/tests/test_cli/test_config.py +++ b/tests/test_cli/test_config.py @@ -462,7 +462,7 @@ def test_set_nested_attribute_not_allowed(self): assert result.exit_code == 1 assert ( result.exception.message - == "Attribute `behaviours.dummy.config.behaviour_arg_1` is not allowed to be updated!" + == "Attribute `behaviours.dummy.config` is not allowed to be updated!" ) def test_no_recognized_root(self): @@ -542,10 +542,11 @@ def test_attribute_not_found(self): catch_exceptions=False, ) - def test_set_fails_when_setting_non_primitive_type(self): + def test_incorrect_data_type(self): """Test that setting the 'dummy' skill behaviours fails because not a primitive type.""" with pytest.raises( - ClickException, match="Attribute `behaviours` is not allowed to be updated!" + ClickException, + match="For attribute `behaviours` `dict` data type is expected, but `str` was provided!", ): self.runner.invoke( cli, @@ -558,7 +559,7 @@ def test_get_fails_when_setting_nested_object(self): """Test that setting a nested object in 'dummy' skill fails because path is not valid.""" with pytest.raises( ClickException, - match=r"Attribute `non_existing_attribute.dummy` is not allowed to be updated!", + match=r"Attribute `non_existing_attribute` is not allowed to be updated!", ): self.runner.invoke( cli, diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index c2e3336956..27bab57e8c 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022-2023 Valory AG +# Copyright 2022-2024 Valory AG # Copyright 2018-2021 Fetch.AI Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -298,7 +298,7 @@ def test_update_method_raises_error_if_skill_component_not_allowed(self): with pytest.raises( ValueError, - match="Attribute `behaviours.new_behaviour.args` is not allowed to be updated!", + match="Attribute `behaviours.new_behaviour` is not allowed to be updated!", ): skill_config.update(new_configurations) diff --git a/tests/test_configurations/test_validation.py b/tests/test_configurations/test_validation.py index 1e19a71fab..682be8eba4 100644 --- a/tests/test_configurations/test_validation.py +++ b/tests/test_configurations/test_validation.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022 Valory AG +# Copyright 2022-2024 Valory AG # Copyright 2018-2021 Fetch.AI Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,14 +46,15 @@ def test_compare_data_pattern(): assert not validate_data_with_pattern( {"a": "${var}"}, {"a": "string"}, skip_env_vars=True ) + assert not validate_data_with_pattern({"a": {}}, {"a": {"b": 12}}) errors = validate_data_with_pattern({"a": 12}, {"b": 12}) assert errors assert errors[0] == "Attribute `a` is not allowed to be updated!" - errors = validate_data_with_pattern({"a": {}}, {"a": {"b": 12}}) + errors = validate_data_with_pattern({"a": {"b": 12}}, {"a": {}}) assert errors - assert errors[0] == "Attribute `a` is not allowed to be updated!" + assert errors[0] == "Attribute `a.b` is not allowed to be updated!" def test_filter_data():