Merge pull request #750 from valory-xyz/feat/override-dicts

Add support for dictionary overrides

.spelling

@@ -3,6 +3,7 @@
 # global dictionary is at the start, file overrides afterwards
 # one word per line, to define a file override use ' - filename'
 # where filename is relative to this configuration file
+fraxtal
 Flashbots
 liveness
 pre-shared

aea/configurations/validation.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,11 +18,13 @@
 #
 # ------------------------------------------------------------------------------
 """Implementation of the configuration validation."""
+
 import inspect
 import json
 import os
 from collections import OrderedDict
 from copy import deepcopy
+from functools import reduce
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
@@ -36,7 +38,7 @@
 from aea.configurations.constants import AGENT
 from aea.configurations.data_types import ComponentId, ComponentType, PublicId
 from aea.exceptions import AEAValidationError
-from aea.helpers.base import dict_to_path_value
+from aea.helpers.base import dict_to_path_value, update_nested_dict
 from aea.helpers.env_vars import is_env_variable
 from aea.helpers.io import open_file
 
@@ -295,12 +297,26 @@ def validate_data_with_pattern(
         excludes_: List[Tuple[str]] = []
     else:
         excludes_ = excludes
-    pattern_path_value = {
+    original_config = {
         tuple(path): value for path, value in dict_to_path_value(pattern)
     }
-    data_path_value = {tuple(path): value for path, value in dict_to_path_value(data)}
+    overrides = {tuple(path): value for path, value in dict_to_path_value(data)}
     errors = []
 
+    # this is a workaround to fix the type of numeric keys as they can only be represented as strs in the json overrides
+    for path in original_config:
+        path_as_str = tuple(map(str, path))
+        if path_as_str in overrides and path not in overrides:
+            value = overrides[path_as_str]
+            del overrides[path_as_str]
+            up_to_last_key = data
+            for key in path_as_str[:-1]:
+                up_to_last_key = up_to_last_key[key]
+            del up_to_last_key[path_as_str[-1]]
+            overrides[path] = value
+            vals = reduce(lambda d, key: {key: d}, reversed(path), value)
+            update_nested_dict(data, vals)
+
     def check_excludes(path: Tuple[str, ...]) -> bool:
         for exclude in excludes_:
             if len(exclude) > len(path):  # pragma: nocover
@@ -315,25 +331,25 @@ def is_a_dict_override(path: Tuple[str, ...]) -> bool:
         flag = False
         while len(path) > 0:
             path = path[:-1]
-            if path in pattern_path_value:
-                pattern_value = pattern_path_value[path]
+            if path in original_config:
+                pattern_value = original_config[path]
                 flag = isinstance(pattern_value, OrderedDict)
                 break
         return flag
 
-    for path, new_value in data_path_value.items():
+    for path, new_value in overrides.items():
         if check_excludes(path):
             continue
 
-        if path not in pattern_path_value:
+        if path not in original_config:
             if not is_a_dict_override(path=(*path,)):
                 errors.append(
                     f"Attribute `{'.'.join(path)}` is not allowed to be updated!"
                 )
 
             continue
 
-        pattern_value = pattern_path_value[path]
+        pattern_value = original_config[path]
 
         if pattern_value is None:
             # not possible to determine data type for optional value not set

aea/helpers/base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -711,10 +711,14 @@ def dict_to_path_value(
     """Convert dict to sequence of terminal path build of  keys and value."""
     path = path or []
     for key, value in data.items():
+        # terminal value
         if isinstance(value, Mapping) and value:
-            # terminal value
+            # yielding here allows for higher level dict overriding
+            yield path + [key], value
+            # recursing to the next level of the dict
             for p, v in dict_to_path_value(value, path + [key]):
                 yield p, v
+        # non-terminal value
         else:
             yield path + [key], value
 
@@ -1087,3 +1091,13 @@ def prepend_if_not_absolute(path: PathLike, prefix: PathLike) -> PathLike:
     :return: the same path if absolute, else the prepended path.
     """
     return path if Path(path).is_absolute() else Path(prefix) / path
+
+
+def update_nested_dict(dict_: dict, nested_update: dict) -> dict:
+    """Update a nested dictionary."""
+    for key, value in nested_update.items():
+        if isinstance(value, dict):
+            dict_[key] = update_nested_dict(dict_.get(key, {}), value)
+        else:
+            dict_[key] = value
+    return dict_

docs/api/helpers/base.md

@@ -838,3 +838,13 @@ Prepend a path with a prefix, but only if not absolute
 
 the same path if absolute, else the prepended path.
 
+<a id="aea.helpers.base.update_nested_dict"></a>
+
+#### update`_`nested`_`dict
+
+```python
+def update_nested_dict(dict_: dict, nested_update: dict) -> dict
+```
+
+Update a nested dictionary.
+

tests/test_cli/test_config.py

@@ -462,7 +462,7 @@ def test_set_nested_attribute_not_allowed(self):
         assert result.exit_code == 1
         assert (
             result.exception.message
-            == "Attribute `behaviours.dummy.config.behaviour_arg_1` is not allowed to be updated!"
+            == "Attribute `behaviours.dummy.config` is not allowed to be updated!"
         )
 
     def test_no_recognized_root(self):
@@ -542,10 +542,11 @@ def test_attribute_not_found(self):
                 catch_exceptions=False,
             )
 
-    def test_set_fails_when_setting_non_primitive_type(self):
+    def test_incorrect_data_type(self):
         """Test that setting the 'dummy' skill behaviours fails because not a primitive type."""
         with pytest.raises(
-            ClickException, match="Attribute `behaviours` is not allowed to be updated!"
+            ClickException,
+            match="For attribute `behaviours` `dict` data type is expected, but `str` was provided!",
         ):
             self.runner.invoke(
                 cli,
@@ -558,7 +559,7 @@ def test_get_fails_when_setting_nested_object(self):
         """Test that setting a nested object in 'dummy' skill fails because path is not valid."""
         with pytest.raises(
             ClickException,
-            match=r"Attribute `non_existing_attribute.dummy` is not allowed to be updated!",
+            match=r"Attribute `non_existing_attribute` is not allowed to be updated!",
         ):
             self.runner.invoke(
                 cli,

tests/test_configurations/test_base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -298,7 +298,7 @@ def test_update_method_raises_error_if_skill_component_not_allowed(self):
 
         with pytest.raises(
             ValueError,
-            match="Attribute `behaviours.new_behaviour.args` is not allowed to be updated!",
+            match="Attribute `behaviours.new_behaviour` is not allowed to be updated!",
         ):
             skill_config.update(new_configurations)
 

tests/test_configurations/test_validation.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -46,14 +46,15 @@ def test_compare_data_pattern():
     assert not validate_data_with_pattern(
         {"a": "${var}"}, {"a": "string"}, skip_env_vars=True
     )
+    assert not validate_data_with_pattern({"a": {}}, {"a": {"b": 12}})
 
     errors = validate_data_with_pattern({"a": 12}, {"b": 12})
     assert errors
     assert errors[0] == "Attribute `a` is not allowed to be updated!"
 
-    errors = validate_data_with_pattern({"a": {}}, {"a": {"b": 12}})
+    errors = validate_data_with_pattern({"a": {"b": 12}}, {"a": {}})
     assert errors
-    assert errors[0] == "Attribute `a` is not allowed to be updated!"
+    assert errors[0] == "Attribute `a.b` is not allowed to be updated!"
 
 
 def test_filter_data():