Add support for dictionary overrides

aea/configurations/validation.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,11 +18,13 @@
 #
 # ------------------------------------------------------------------------------
 """Implementation of the configuration validation."""
+
 import inspect
 import json
 import os
 from collections import OrderedDict
 from copy import deepcopy
+from functools import reduce
 from pathlib import Path
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
@@ -36,7 +38,7 @@
 from aea.configurations.constants import AGENT
 from aea.configurations.data_types import ComponentId, ComponentType, PublicId
 from aea.exceptions import AEAValidationError
-from aea.helpers.base import dict_to_path_value
+from aea.helpers.base import dict_to_path_value, update_nested_dict
 from aea.helpers.env_vars import is_env_variable
 from aea.helpers.io import open_file
 
@@ -295,12 +297,26 @@ def validate_data_with_pattern(
         excludes_: List[Tuple[str]] = []
     else:
         excludes_ = excludes
-    pattern_path_value = {
+    original_config = {
         tuple(path): value for path, value in dict_to_path_value(pattern)
     }
-    data_path_value = {tuple(path): value for path, value in dict_to_path_value(data)}
+    overrides = {tuple(path): value for path, value in dict_to_path_value(data)}
     errors = []
 
+    # this is a workaround to fix the type of numeric keys as they can only be represented as strs in the json overrides
+    for path in original_config:
+        path_as_str = tuple(map(str, path))
+        if path_as_str in overrides and path not in overrides:
+            value = overrides[path_as_str]
+            del overrides[path_as_str]
+            up_to_last_key = data
+            for key in path_as_str[:-1]:
+                up_to_last_key = up_to_last_key[key]
+            del up_to_last_key[path_as_str[-1]]
+            overrides[path] = value
+            vals = reduce(lambda d, key: {key: d}, reversed(path), value)
+            update_nested_dict(data, vals)
+
     def check_excludes(path: Tuple[str, ...]) -> bool:
         for exclude in excludes_:
             if len(exclude) > len(path):  # pragma: nocover
@@ -315,25 +331,25 @@ def is_a_dict_override(path: Tuple[str, ...]) -> bool:
         flag = False
         while len(path) > 0:
             path = path[:-1]
-            if path in pattern_path_value:
-                pattern_value = pattern_path_value[path]
+            if path in original_config:
+                pattern_value = original_config[path]
                 flag = isinstance(pattern_value, OrderedDict)
                 break
         return flag
 
-    for path, new_value in data_path_value.items():
+    for path, new_value in overrides.items():
         if check_excludes(path):
             continue
 
-        if path not in pattern_path_value:
+        if path not in original_config:
             if not is_a_dict_override(path=(*path,)):
                 errors.append(
                     f"Attribute `{'.'.join(path)}` is not allowed to be updated!"
                 )
 
             continue
 
-        pattern_value = pattern_path_value[path]
+        pattern_value = original_config[path]
 
         if pattern_value is None:
             # not possible to determine data type for optional value not set

aea/helpers/base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -711,10 +711,14 @@ def dict_to_path_value(
     """Convert dict to sequence of terminal path build of  keys and value."""
     path = path or []
     for key, value in data.items():
+        # terminal value
         if isinstance(value, Mapping) and value:
-            # terminal value
+            # yielding here allows for higher level dict overriding
+            yield path + [key], value
+            # recursing to the next level of the dict
             for p, v in dict_to_path_value(value, path + [key]):
                 yield p, v
+        # non-terminal value
         else:
             yield path + [key], value
 
@@ -1087,3 +1091,13 @@ def prepend_if_not_absolute(path: PathLike, prefix: PathLike) -> PathLike:
     :return: the same path if absolute, else the prepended path.
     """
     return path if Path(path).is_absolute() else Path(prefix) / path
+
+
+def update_nested_dict(dict_: dict, nested_update: dict) -> dict:
+    """Update a nested dictionary."""
+    for key, value in nested_update.items():
+        if isinstance(value, dict):
+            dict_[key] = update_nested_dict(dict_.get(key, {}), value)
+        else:
+            dict_[key] = value
+    return dict_

docs/api/helpers/base.md

@@ -838,3 +838,13 @@ Prepend a path with a prefix, but only if not absolute
 
 the same path if absolute, else the prepended path.
 
+<a id="aea.helpers.base.update_nested_dict"></a>
+
+#### update`_`nested`_`dict
+
+```python
+def update_nested_dict(dict_: dict, nested_update: dict) -> dict
+```
+
+Update a nested dictionary.
+