[v0.17.0]: Update the policy's logic

packages/packages.json

@@ -16,14 +16,14 @@
         "contract/valory/staking_token/0.1.0": "bafybeig4fl35dn7d5gnprux2nwsqbirm7zkiujz3xvrwcjuktz6hkq4as4",
         "contract/valory/relayer/0.1.0": "bafybeihzgjyvhtorugjw3yldznqsbwo3aqpxowm7k2nrvj6qtwpsc7jl7u",
         "skill/valory/market_manager_abci/0.1.0": "bafybeig44vtxcq7aarqkg2mskly3zwwnyfc45rtdvorwz6ywspwg3obqn4",
-        "skill/valory/decision_maker_abci/0.1.0": "bafybeiejbfkl4im6srigta53ysan3tu5hlbb44cbd2mgryp2hsanyxpdge",
-        "skill/valory/trader_abci/0.1.0": "bafybeigfccxw5v76pectrmh2a3ibd4gt2hhtskkqay6an3g7lgdyotuz34",
-        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeiaiek6b3p7dro45qjoecmgseznhixhrgqgxfiap7lqw4rbi5fyhvy",
+        "skill/valory/decision_maker_abci/0.1.0": "bafybeigwldz6glu2twnltkpsivxpgqauxyeozqijt2zlbzppafolbw7sn4",
+        "skill/valory/trader_abci/0.1.0": "bafybeifrudxseikqjcxmqv2vpwj52fsgbq2umqbd4f3hupxxqvihrq7nri",
+        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeih5wylffcb3cyy5hpzn53yrzemm3hmyhmvpz6jfcipb4x7aagxa64",
         "skill/valory/staking_abci/0.1.0": "bafybeidubhfvlf6x627t4wetc6ran5sb4pttjphyhv6vefp2a4kcuca2be",
         "skill/valory/check_stop_trading_abci/0.1.0": "bafybeiboctfgetji6xlzw2buts5zsejnlrrhqkppme2p6fwzu26zznb324",
-        "agent/valory/trader/0.1.0": "bafybeiaxm4ts4odvhmbltyaa7rngm6jio7b4fdwvwszahw2ndfjdpd547a",
-        "service/valory/trader/0.1.0": "bafybeig3dk2wuzspnu7aeop2wbxd7sfe5yjlsj3yreen6v6vrtqwj4htyq",
-        "service/valory/trader_pearl/0.1.0": "bafybeiannjjvfegdigiwd7fsjrpsemg3b6uamp6ucm26pcnvalkqebfnqe"
+        "agent/valory/trader/0.1.0": "bafybeicj254rppwbjb3wgynpe5epftfd2ldqg5uqkgbjzokckfjd3r2ebu",
+        "service/valory/trader/0.1.0": "bafybeiczmy65654ouobytevqifgcnrq2wnm63dguni6n6otxvx3ujrv33a",
+        "service/valory/trader_pearl/0.1.0": "bafybeihfxb47yrpxhd4cpnzqvvncu6x5li657sxloycn6pfefdxefbxs3q"
     },
     "third_party": {
         "protocol/open_aea/signing/1.0.0": "bafybeihv62fim3wl2bayavfcg3u5e5cxu3b7brtu4cn5xoxd6lqwachasi",

packages/valory/agents/trader/aea-config.yaml

@@ -47,10 +47,10 @@ skills:
 - valory/reset_pause_abci:0.1.0:bafybeiameewywqigpupy3u2iwnkfczeiiucue74x2l5lbge74rmw6bgaie
 - valory/termination_abci:0.1.0:bafybeif2zim2de356eo3sipkmoev5emwadpqqzk3huwqarywh4tmqt3vzq
 - valory/transaction_settlement_abci:0.1.0:bafybeic3tccdjypuge2lewtlgprwkbb53lhgsgn7oiwzyrcrrptrbeyote
-- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeiaiek6b3p7dro45qjoecmgseznhixhrgqgxfiap7lqw4rbi5fyhvy
+- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeih5wylffcb3cyy5hpzn53yrzemm3hmyhmvpz6jfcipb4x7aagxa64
 - valory/market_manager_abci:0.1.0:bafybeig44vtxcq7aarqkg2mskly3zwwnyfc45rtdvorwz6ywspwg3obqn4
-- valory/decision_maker_abci:0.1.0:bafybeiejbfkl4im6srigta53ysan3tu5hlbb44cbd2mgryp2hsanyxpdge
-- valory/trader_abci:0.1.0:bafybeigfccxw5v76pectrmh2a3ibd4gt2hhtskkqay6an3g7lgdyotuz34
+- valory/decision_maker_abci:0.1.0:bafybeigwldz6glu2twnltkpsivxpgqauxyeozqijt2zlbzppafolbw7sn4
+- valory/trader_abci:0.1.0:bafybeifrudxseikqjcxmqv2vpwj52fsgbq2umqbd4f3hupxxqvihrq7nri
 - valory/staking_abci:0.1.0:bafybeidubhfvlf6x627t4wetc6ran5sb4pttjphyhv6vefp2a4kcuca2be
 - valory/check_stop_trading_abci:0.1.0:bafybeiboctfgetji6xlzw2buts5zsejnlrrhqkppme2p6fwzu26zznb324
 - valory/mech_interact_abci:0.1.0:bafybeih2cck5xu6yaibomwtm5zbcp6llghr3ighdnk56fzwu3ihu5xx35e

packages/valory/services/trader/service.yaml

@@ -7,7 +7,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeigtuothskwyvrhfosps2bu6suauycolj67dpuxqvnicdrdu7yhtvq
 fingerprint_ignore_patterns: []
-agent: valory/trader:0.1.0:bafybeiaxm4ts4odvhmbltyaa7rngm6jio7b4fdwvwszahw2ndfjdpd547a
+agent: valory/trader:0.1.0:bafybeicj254rppwbjb3wgynpe5epftfd2ldqg5uqkgbjzokckfjd3r2ebu
 number_of_agents: 4
 deployment:
   agent:

packages/valory/services/trader_pearl/service.yaml

@@ -8,7 +8,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeibg7bdqpioh4lmvknw3ygnllfku32oca4eq5pqtvdrdsgw6buko7e
 fingerprint_ignore_patterns: []
-agent: valory/trader:0.1.0:bafybeiaxm4ts4odvhmbltyaa7rngm6jio7b4fdwvwszahw2ndfjdpd547a
+agent: valory/trader:0.1.0:bafybeicj254rppwbjb3wgynpe5epftfd2ldqg5uqkgbjzokckfjd3r2ebu
 number_of_agents: 1
 deployment:
   agent:

packages/valory/skills/decision_maker_abci/behaviours/decision_receive.py

@@ -111,6 +111,10 @@ def _next_dataset_row(self) -> Optional[Dict[str, str]]:
                 self._rows_exceeded = True
                 return None
 
+            next_row: Optional[Dict[str, str]] = next(reader, {})
+            if not next_row:
+                self.shared_state.last_benchmarking_has_run = True
+
         msg = f"Processing question in row with index {next_mock_data_row}: {row_with_headers}"
         self.context.logger.info(msg)
         return row_with_headers

packages/valory/skills/decision_maker_abci/behaviours/reedem.py

@@ -114,10 +114,10 @@ def _set_block_number(self, trade: Trade) -> Generator:
             f"Chose block number {self.earliest_block_number!r} as closest to timestamp {timestamp!r}"
         )
 
-    def _try_update_policy(self, tool: str) -> None:
+    def _try_update_policy(self, tool: str, winning: bool) -> None:
         """Try to update the policy."""
         try:
-            self.policy.update_accuracy_store(tool)
+            self.policy.update_accuracy_store(tool, winning)
         except KeyError:
             self.context.logger.warning(
                 f"The stored utilized tools seem to be outdated as no {tool=} was found. "
@@ -134,9 +134,7 @@ def _update_policy(self, update: Trade) -> None:
 
         # we try to avoid an ever-increasing dictionary of utilized tools by removing a tool when not needed anymore
         del self.utilized_tools[update.transactionHash]
-        if not update.is_winning:
-            return
-        self._try_update_policy(tool)
+        self._try_update_policy(tool, update.is_winning)
 
     def update_redeem_info(self, chunk: list) -> Generator:
         """Update the redeeming information using the given chunk."""
@@ -932,9 +930,9 @@ def _build_payload(self, redeem_tx_hex: Optional[str] = None) -> RedeemPayload:
 
     def _benchmarking_act(self) -> RedeemPayload:
         """The act of the agent while running in benchmarking mode."""
-        if self.mock_data.is_winning:
-            tool = self.synchronized_data.mech_tool
-            self._try_update_policy(tool)
+        tool = self.synchronized_data.mech_tool
+        winning = self.mock_data.is_winning
+        self._try_update_policy(tool, winning)
         return self._build_payload()
 
     def _normal_act(self) -> Generator[None, None, Optional[RedeemPayload]]:

packages/valory/skills/decision_maker_abci/behaviours/tool_selection.py

@@ -59,7 +59,11 @@ def async_act(self) -> Generator:
             if selected_tool is not None:
                 # the period will increment when the benchmarking finishes
                 benchmarking_running = self.synchronized_data.period_count == 0
-                if self.benchmarking_mode.enabled and benchmarking_running:
+                if (
+                    self.benchmarking_mode.enabled
+                    and benchmarking_running
+                    and not self.shared_state.last_benchmarking_has_run
+                ):
                     self.policy.tool_used(selected_tool)
                 mech_tools = json.dumps(self.mech_tools)
                 policy = self.policy.serialize()

packages/valory/skills/decision_maker_abci/models.py

@@ -209,6 +209,8 @@ def __init__(self, *args: Any, skill_context: SkillContext, **kwargs: Any) -> No
         # latest liquidity information (only relevant to the benchmarking mode)
         self.liquidity_amounts: Dict[str, List[int]] = {}
         self.liquidity_prices: Dict[str, List[float]] = {}
+        # whether this is the last run of the benchmarking mode
+        self.last_benchmarking_has_run: bool = False
 
     @property
     def mock_question_id(self) -> Any:

packages/valory/skills/decision_maker_abci/policy.py

@@ -24,9 +24,15 @@
 from dataclasses import asdict, dataclass, field, is_dataclass
 from typing import Any, Dict, List, Optional, Union
 
+from packages.valory.skills.decision_maker_abci.utils.scaling import scale_value
+
 
 RandomnessType = Union[int, float, str, bytes, bytearray, None]
 
+VOLUME_FACTOR_REGULARIZATION = 0.5
+UNSCALED_WEIGHTED_ACCURACY_INTERVAL = (0.5, 100.5)
+SCALED_WEIGHTED_ACCURACY_INTERVAL = (0, 1)
+
 
 class DataclassEncoder(json.JSONEncoder):
     """A custom JSON encoder for dataclasses."""
@@ -109,7 +115,10 @@ def n_tools(self) -> int:
     @property
     def n_requests(self) -> int:
         """Get the total number of requests."""
-        return sum(acc_info.requests for acc_info in self.accuracy_store.values())
+        return sum(
+            acc_info.requests + acc_info.pending
+            for acc_info in self.accuracy_store.values()
+        )
 
     @property
     def has_updated(self) -> bool:
@@ -131,9 +140,15 @@ def best_tool(self) -> str:
     def update_weighted_accuracy(self) -> None:
         """Update the weighted accuracy for each tool."""
         self.weighted_accuracy = {
-            tool: (acc_info.accuracy / 100)
-            * (acc_info.requests - acc_info.pending)
-            / self.n_requests
+            tool: scale_value(
+                (
+                    acc_info.accuracy
+                    + ((acc_info.requests - acc_info.pending) / self.n_requests)
+                    * VOLUME_FACTOR_REGULARIZATION
+                ),
+                UNSCALED_WEIGHTED_ACCURACY_INTERVAL,
+                SCALED_WEIGHTED_ACCURACY_INTERVAL,
+            )
             for tool, acc_info in self.accuracy_store.items()
         }
 
@@ -155,10 +170,16 @@ def tool_used(self, tool: str) -> None:
         self.accuracy_store[tool].pending += 1
         self.update_weighted_accuracy()
 
-    def update_accuracy_store(self, tool: str) -> None:
+    def update_accuracy_store(self, tool: str, winning: bool) -> None:
         """Update the accuracy store for the given tool."""
-        self.accuracy_store[tool].requests += 1
-        self.accuracy_store[tool].pending -= 1
+        acc_info = self.accuracy_store[tool]
+        total_correct_answers = acc_info.accuracy * acc_info.requests
+        if winning:
+            total_correct_answers += 1
+
+        acc_info.requests += 1
+        acc_info.pending -= 1
+        acc_info.accuracy = total_correct_answers / acc_info.requests
         self.update_weighted_accuracy()
 
     def serialize(self) -> str:

packages/valory/skills/decision_maker_abci/skill.yaml

@@ -17,24 +17,24 @@ fingerprint:
   behaviours/blacklisting.py: bafybeibofq3emoxxcndkozupsyo2xrigewy6ojvzfdvzwruvsvgyilokbu
   behaviours/check_benchmarking.py: bafybeiao2lyj7apezkqrpgsyzb3dwvrdgsrgtprf6iuhsmlsufvxfl5bci
   behaviours/claim_subscription.py: bafybeihv5dg74deifzk46ppdwcvz6lgamgl6m7qr6sgqv2zie35j2576ca
-  behaviours/decision_receive.py: bafybeib465wlp74vq2jdaed53irnuqq22vdjj6punkjg4kyh6dovxf764u
+  behaviours/decision_receive.py: bafybeiay7cbzzngvtatswrgxczt5257jgwc6ppfi43fwc6rso36kipxmr4
   behaviours/decision_request.py: bafybeiabjzzcwcfbfmtoftjhewmkgbhxfnigbc5cwmmxl6cob5gv64jwwa
   behaviours/handle_failed_tx.py: bafybeidxpc6u575ymct5tdwutvzov6zqfdoio5irgldn3fw7q3lg36mmxm
   behaviours/order_subscription.py: bafybeibl5ktq5lcwzr3komg77o67yebiokqyii7r3fsshatpww5tr5r3ke
   behaviours/randomness.py: bafybeidmr33teizrs4uxlo5tdz766ds6os4pe5lttstm7jpmhgmjz5ti3q
-  behaviours/reedem.py: bafybeigddys4zqpztjma3rakgcmkrk55hiskgivhq2cgxsqnhl5wikmrym
+  behaviours/reedem.py: bafybeiaszvuwfamdq5m7zaxf3tvbyp6wocsksezfajqv5xpkh43r62cj6u
   behaviours/round_behaviour.py: bafybeibvhobpvzzd37ecleuyp2jrbed6nontcw7urtsilbbzvqsmmupx64
   behaviours/sampling.py: bafybeibtkli72qsvotkrsepkgpiumtr5sershtkpb427oygnszs3dpgxry
   behaviours/storage_manager.py: bafybeifelwjwstg5dwrnzeav5mumz47kqrlmv4crfn27a3vliomychd2x4
-  behaviours/tool_selection.py: bafybeiat3mqtki55dnjmrrzpahjv3n26y5b45iehqx6tycqra76wtp7gxq
+  behaviours/tool_selection.py: bafybeienlxcgjs3ogyofli3d7q3p5rst3mcxxcnwqf7qolqjeefjtixeke
   dialogues.py: bafybeigpwuzku3we7axmxeamg7vn656maww6emuztau5pg3ebsoquyfdqm
   fsm_specification.yaml: bafybeicvrvzcjt4hg5jyr2eleoajpbb7q2fbq2bopmbnwy33fsjdblzygu
   handlers.py: bafybeigod6gbjrxy4mbmulbzsbokeaoycoqys64vqtxnumishfukzf73za
   io_/__init__.py: bafybeifxgmmwjqzezzn3e6keh2bfo4cyo7y5dq2ept3stfmgglbrzfl5rq
   io_/loader.py: bafybeih3sdsx5dhe4kzhtoafexjgkutsujwqy3zcdrlrkhtdks45bc7exa
-  models.py: bafybeif76bwktyx6i4h7gqunn33q2s4fjlstm4xb5hhsj3zmq4jsofmq4i
+  models.py: bafybeiafif3kzzyvvme2t6eck4sbpz6saam26jzjg46as2bqm5r5gvd4fq
   payloads.py: bafybeia32ds4ks5g3rhd43w6siy74vi5inaheuxhswiy5ndykxtgncoeu4
-  policy.py: bafybeihp4lzv4ejyu6k5reukbpa6kamglicpymxfx6obm5pgugik4vc63y
+  policy.py: bafybeifwmav2bfp5caypvirvwd5tbf7gijavno4i7km5dizpsgcgpdfqk4
   redeem_info.py: bafybeifiiix4gihfo4avraxt34sfw35v6dqq45do2drrssei2shbps63mm
   rounds.py: bafybeieffmnu3sivnagmhrhcufnh5zun6pxligc6wp26me43u6cjy2y3xi
   states/__init__.py: bafybeid23llnyp6j257dluxmrnztugo5llsrog7kua53hllyktz4dqhqoy
@@ -61,6 +61,7 @@ fingerprint:
   tests/conftest.py: bafybeidy5hw56kw5mxudnfbhvogofn6k4rqb4ux2bd45baedrrhmgyrude
   utils/__init__.py: bafybeiazrfg3kwfdl5q45azwz6b6mobqxngxpf4hazmrnkhinpk4qhbbf4
   utils/nevermined.py: bafybeigallaqxhqopznhjhefr6bukh4ojkz5vdtqyzod5dksshrf24fjgi
+  utils/scaling.py: bafybeialr3z4zogp4k3l2bzcjfi4igvxzjexmlpgze2bai2ufc3plaow4y
 fingerprint_ignore_patterns: []
 connections:
 - valory/http_server:0.22.0:bafybeihpgu56ovmq4npazdbh6y6ru5i7zuv6wvdglpxavsckyih56smu7m

packages/valory/skills/decision_maker_abci/utils/scaling.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# ------------------------------------------------------------------------------
+#
+#   Copyright 2024 Valory AG
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# ------------------------------------------------------------------------------
+
+
+"""This package contains helpers for scaling operations."""
+
+
+from typing import List, Tuple
+
+
+def min_max(li: List[float]) -> Tuple[float, float]:
+    """Get the min and max of a list."""
+    if not li:
+        raise ValueError("The list is empty.")
+
+    min_ = max_ = li[0]
+
+    for num in li[1:]:
+        if num < min_:
+            min_ = num
+        elif num > max_:
+            max_ = num
+
+    return min_, max_
+
+
+def scale_value(
+    value: float,
+    min_max_bounds: Tuple[float, float],
+    scale_bounds: Tuple[float, float] = (0, 1),
+) -> float:
+    """Perform min-max scaling on a value."""
+    min_, max_ = min_max_bounds
+    current_range = max_ - min_
+    # normalize between 0-1
+    std = (value - min_) / current_range
+    # scale between min_bound and max_bound
+    min_bound, max_bound = scale_bounds
+    target_range = max_bound - min_bound
+    return std * target_range + min_bound
+
+
+def min_max_scale(
+    li: List[float],
+    scale_bounds: Tuple[float, float] = (0, 1),
+) -> List[float]:
+    """Perform min-max scaling on a list of values."""
+    min_max_ = min_max(li)
+    return [scale_value(value, min_max_, scale_bounds) for value in li]

packages/valory/skills/trader_abci/skill.yaml

@@ -25,8 +25,8 @@ skills:
 - valory/transaction_settlement_abci:0.1.0:bafybeic3tccdjypuge2lewtlgprwkbb53lhgsgn7oiwzyrcrrptrbeyote
 - valory/termination_abci:0.1.0:bafybeif2zim2de356eo3sipkmoev5emwadpqqzk3huwqarywh4tmqt3vzq
 - valory/market_manager_abci:0.1.0:bafybeig44vtxcq7aarqkg2mskly3zwwnyfc45rtdvorwz6ywspwg3obqn4
-- valory/decision_maker_abci:0.1.0:bafybeiejbfkl4im6srigta53ysan3tu5hlbb44cbd2mgryp2hsanyxpdge
-- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeiaiek6b3p7dro45qjoecmgseznhixhrgqgxfiap7lqw4rbi5fyhvy
+- valory/decision_maker_abci:0.1.0:bafybeigwldz6glu2twnltkpsivxpgqauxyeozqijt2zlbzppafolbw7sn4
+- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeih5wylffcb3cyy5hpzn53yrzemm3hmyhmvpz6jfcipb4x7aagxa64
 - valory/staking_abci:0.1.0:bafybeidubhfvlf6x627t4wetc6ran5sb4pttjphyhv6vefp2a4kcuca2be
 - valory/check_stop_trading_abci:0.1.0:bafybeiboctfgetji6xlzw2buts5zsejnlrrhqkppme2p6fwzu26zznb324
 - valory/mech_interact_abci:0.1.0:bafybeih2cck5xu6yaibomwtm5zbcp6llghr3ighdnk56fzwu3ihu5xx35e

packages/valory/skills/tx_settlement_multiplexer_abci/skill.yaml

@@ -21,7 +21,7 @@ protocols:
 - valory/ledger_api:1.0.0:bafybeihdk6psr4guxmbcrc26jr2cbgzpd5aljkqvpwo64bvaz7tdti2oni
 skills:
 - valory/abstract_round_abci:0.1.0:bafybeiar2yhzxacfe3qqamqhaihtlcimquwedffctw55sowx6rac3cm3ui
-- valory/decision_maker_abci:0.1.0:bafybeiejbfkl4im6srigta53ysan3tu5hlbb44cbd2mgryp2hsanyxpdge
+- valory/decision_maker_abci:0.1.0:bafybeigwldz6glu2twnltkpsivxpgqauxyeozqijt2zlbzppafolbw7sn4
 - valory/staking_abci:0.1.0:bafybeidubhfvlf6x627t4wetc6ran5sb4pttjphyhv6vefp2a4kcuca2be
 - valory/mech_interact_abci:0.1.0:bafybeih2cck5xu6yaibomwtm5zbcp6llghr3ighdnk56fzwu3ihu5xx35e
 behaviours: