Merge pull request #109 from valory-xyz/fix/policy

Update the policy when a tool causes a blacklisting

packages/packages.json

@@ -1,13 +1,13 @@
 {
     "dev": {
         "skill/valory/market_manager_abci/0.1.0": "bafybeigmy3g2klkzdhttti2ouxf575ua5y7ajmh4a3yfpukeukqxjky47m",
-        "skill/valory/decision_maker_abci/0.1.0": "bafybeihtydxuzyd35cq23xoxbytpxebpbn3rsyqf4bt6z4sleflqck6riy",
-        "skill/valory/trader_abci/0.1.0": "bafybeignhbqiajnhec7xipsibm3vpjovxboaak3n3kd2ozc27sjaw7733m",
+        "skill/valory/decision_maker_abci/0.1.0": "bafybeifnprgtifiuztmvics57fljowhtj7j5ffph5cwdc2sxh5milcjmze",
+        "skill/valory/trader_abci/0.1.0": "bafybeigihuarm5766mojeeyipw7exkskotag2lsqba24z5jmihiqckri54",
         "contract/valory/market_maker/0.1.0": "bafybeif4mm2s3gxtvp227yypkcnna5ftec7vajcftvtbdmqddh7nprah5m",
-        "agent/valory/trader/0.1.0": "bafybeibpafgfqxva32x4y22zsw6pjur43xcbirjizlornqwvl5vykopdgy",
-        "service/valory/trader/0.1.0": "bafybeiadxvczqasdeukfmt2c2h66z4dxtgeyzinio7cm4alm2meboh65ye",
+        "agent/valory/trader/0.1.0": "bafybeia65a6el4zwoblik53k6nqxby6mbuylccdfwegta5jpphcnzy2exi",
+        "service/valory/trader/0.1.0": "bafybeif77evw6h4ex7vkneqmnafdv4v7l43bl7yrtgo5smq5grrl7hloiu",
         "contract/valory/erc20/0.1.0": "bafybeibysifjwhlcn7nntsjkowdmja53ed6urfzypi6sxfc7chtheecvpi",
-        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeigfkdkpkrktw4wippon6al7vreg3qh7xax33xafomepy6xrv5paa4",
+        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeib5zblszm5h5gcdxjl55wumda2diztbiti6d67vhy46wduhyvs34e",
         "contract/valory/mech/0.1.0": "bafybeifaavuqr2bz7nypapjwkufc5c6vwxm5ifucmtd3w7sh42tne35nga",
         "contract/valory/realitio/0.1.0": "bafybeic3wihpzz4deguslpzlnsgrlrabclq3njwuagyaiiv6rzqujfrs6e",
         "contract/valory/realitio_proxy/0.1.0": "bafybeidx37xzjjmapwacedgzhum6grfzhp5vhouz4zu3pvpgdy5pgb2fr4",

packages/valory/agents/trader/aea-config.yaml

@@ -41,10 +41,10 @@ skills:
 - valory/reset_pause_abci:0.1.0:bafybeicpxn2khtaesuf4cq6ypwdmdmonlqroj2q2i6cxvpizc2y4cw66pe
 - valory/termination_abci:0.1.0:bafybeieqm46zuccaagnko3qlw6p3nvoohdrfgvpmw467r5lyil2dqrzjsy
 - valory/transaction_settlement_abci:0.1.0:bafybeia6cdxdlqrcwk2maw25fo7dafzd2p3rs7syropvufophk2pitzbwy
-- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeigfkdkpkrktw4wippon6al7vreg3qh7xax33xafomepy6xrv5paa4
+- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeib5zblszm5h5gcdxjl55wumda2diztbiti6d67vhy46wduhyvs34e
 - valory/market_manager_abci:0.1.0:bafybeigmy3g2klkzdhttti2ouxf575ua5y7ajmh4a3yfpukeukqxjky47m
-- valory/decision_maker_abci:0.1.0:bafybeihtydxuzyd35cq23xoxbytpxebpbn3rsyqf4bt6z4sleflqck6riy
-- valory/trader_abci:0.1.0:bafybeignhbqiajnhec7xipsibm3vpjovxboaak3n3kd2ozc27sjaw7733m
+- valory/decision_maker_abci:0.1.0:bafybeifnprgtifiuztmvics57fljowhtj7j5ffph5cwdc2sxh5milcjmze
+- valory/trader_abci:0.1.0:bafybeigihuarm5766mojeeyipw7exkskotag2lsqba24z5jmihiqckri54
 default_ledger: ethereum
 required_ledgers:
 - ethereum

packages/valory/services/trader/service.yaml

@@ -7,7 +7,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeigtuothskwyvrhfosps2bu6suauycolj67dpuxqvnicdrdu7yhtvq
 fingerprint_ignore_patterns: []
-agent: valory/trader:0.1.0:bafybeibpafgfqxva32x4y22zsw6pjur43xcbirjizlornqwvl5vykopdgy
+agent: valory/trader:0.1.0:bafybeia65a6el4zwoblik53k6nqxby6mbuylccdfwegta5jpphcnzy2exi
 number_of_agents: 4
 deployment: {}
 ---

packages/valory/skills/decision_maker_abci/behaviours/blacklisting.py

@@ -24,11 +24,11 @@
 from packages.valory.skills.decision_maker_abci.behaviours.base import (
     DecisionMakerBaseBehaviour,
 )
+from packages.valory.skills.decision_maker_abci.payloads import BlacklistingPayload
 from packages.valory.skills.decision_maker_abci.states.blacklisting import (
     BlacklistingRound,
 )
 from packages.valory.skills.market_manager_abci.bets import BetStatus, serialize_bets
-from packages.valory.skills.market_manager_abci.payloads import UpdateBetsPayload
 
 
 class BlacklistingBehaviour(DecisionMakerBaseBehaviour):
@@ -50,13 +50,21 @@ def _blacklist(self) -> Optional[str]:
         sampled_bet.status = BetStatus.BLACKLISTED
         blacklist_expiration = self.synced_time + self.params.blacklisting_duration
         sampled_bet.blacklist_expiration = blacklist_expiration
+        # add a zero reward to the tool that has lead to the blacklisting of the market
+        self.policy.add_reward(self.synchronized_data.mech_tool_idx)
 
         return serialize_bets(bets)
 
+    def setup(self) -> None:
+        """Setup the behaviour"""
+        self._policy = self.synchronized_data.policy
+
     def async_act(self) -> Generator:
         """Do the action."""
 
         with self.context.benchmark_tool.measure(self.behaviour_id).local():
-            payload = UpdateBetsPayload(self.context.agent_address, self._blacklist())
+            bets = self._blacklist()
+            policy = self.policy.serialize()
+            payload = BlacklistingPayload(self.context.agent_address, bets, policy)
 
         yield from self.finish_behaviour(payload)

packages/valory/skills/decision_maker_abci/payloads.py

@@ -72,6 +72,13 @@ class VotingPayload(BaseTxPayload):
     vote: bool
 
 
+@dataclass(frozen=True)
+class BlacklistingPayload(UpdateBetsPayload):
+    """Represents a transaction payload for blacklisting."""
+
+    policy: str
+
+
 @dataclass(frozen=True)
 class ToolSelectionPayload(BaseTxPayload):
     """Represents a transaction payload for selecting a mech tool."""

packages/valory/skills/decision_maker_abci/policy.py

@@ -127,7 +127,7 @@ def select_tool(self) -> Optional[int]:
 
         return self.best_tool
 
-    def add_reward(self, index: int, reward: float) -> None:
+    def add_reward(self, index: int, reward: float = 0) -> None:
         """Add a reward for the tool corresponding to the given index."""
         self.counts[index] += 1
         self.rewards[index] += reward

packages/valory/skills/decision_maker_abci/skill.yaml

@@ -14,7 +14,7 @@ fingerprint:
   behaviours/__init__.py: bafybeih6ddz2ocvm6x6ytvlbcz6oi4snb5ee5xh5h65nq4w2qf7fd7zfky
   behaviours/base.py: bafybeidny4y4w4jbjbp5d5qivur6jm37fnuik4mqzchzshiuvpozxe7vwy
   behaviours/bet_placement.py: bafybeiauzdizqawjgzc3ugx33vuh726e25juhacqdr7c6uztpn6tra5dva
-  behaviours/blacklisting.py: bafybeicvespraci44y2dtddy4wi7cdhjuyk6crjs7ztnssm2rcrovha3hm
+  behaviours/blacklisting.py: bafybeie4aawdikv7pj5rorqy55jy2fkqzhitz3qmf3wyycarsj4dmwz7wa
   behaviours/decision_receive.py: bafybeibfnucc444uqcythgsn4cawluubbi4u4q7ab4aqc4ts3lu4qnvhdq
   behaviours/decision_request.py: bafybeiajbv3t6w4vz3u3bbqzaufjfi2znwsly4cvf55farwvp7y7c2jefm
   behaviours/handle_failed_tx.py: bafybeidxpc6u575ymct5tdwutvzov6zqfdoio5irgldn3fw7q3lg36mmxm
@@ -26,14 +26,14 @@ fingerprint:
   fsm_specification.yaml: bafybeifnob3ceim2mj7lqagtnpwqjqqxs5eg3oiwc73gwm6x5i2dvvlcya
   handlers.py: bafybeihj33szgrcxnpd73s4nvluyxwwsvhjum2cuq3ilhhe6vfola3k7vy
   models.py: bafybeidlis33rkpvtb2w6qrtmiqzb4tbqfolkbbs6hk3jcx73lfuwqif3e
-  payloads.py: bafybeic4iz6uxinqpaxnefkadnzaox23y7k57zawnnkeovplih36v5cchy
-  policy.py: bafybeiftviqwq6lapfxj6ykxyh64z72vauqrgmsoho7zo2lvsdvu32kw3m
+  payloads.py: bafybeiawx6ix2ljvf545fe3wocdoh7frds6gjvzgv62b3w6huaolgvywqi
+  policy.py: bafybeidkqqrkgxah5q73mlpr64swxczadhuyax3ab2ijg6obemnvy2sxnu
   redeem_info.py: bafybeibddfxwp3577c3dl2utaowwltquu5fg6crezpumoebw563wxpbfrm
   rounds.py: bafybeihopxmhzrpipc6r5x2vqf2tcixzuf5xdpqfsr7gr3n756n3ssft5q
   states/__init__.py: bafybeid23llnyp6j257dluxmrnztugo5llsrog7kua53hllyktz4dqhqoy
   states/base.py: bafybeibx3kn2b3rovyiowjzpwhspjdbzyfroequg7zsjqsbdh5cyowu72a
   states/bet_placement.py: bafybeibalhxhp2c4oljmiwqi6ds3g36fgtabmf42mb5sgq6z22znrcbhda
-  states/blacklisting.py: bafybeiao747i4z7owk6dmwfuzdijag55m3ryj3nowfoggvczpgk3koza44
+  states/blacklisting.py: bafybeifruvxwwltndwazkzzbpc5nnnad3z5t5ofy6d3i7nssljbxl6gvxu
   states/decision_receive.py: bafybeifm3oyq2aji7f5yag6wpe4vr3ivi74pybdsk2jvmziiidx5nt7t4a
   states/decision_request.py: bafybeic7otc3hjb753svbmur3yyk6szahc25yii3x4w4vcnpfz6jwvacuu
   states/final_states.py: bafybeidiwhuyd5zm2cq7vhv2owcrxdpm7fnvn3db6p6tql4jz5hgpalflu

packages/valory/skills/decision_maker_abci/states/blacklisting.py

@@ -19,15 +19,26 @@
 
 """This module contains the blacklisting state of the decision-making abci app."""
 
-from packages.valory.skills.decision_maker_abci.states.base import Event
-from packages.valory.skills.market_manager_abci.rounds import (
-    UpdateBetsRound as BaseUpdateBetsRound,
+from typing import Any, Type
+
+from packages.valory.skills.abstract_round_abci.base import get_name
+from packages.valory.skills.decision_maker_abci.payloads import BlacklistingPayload
+from packages.valory.skills.decision_maker_abci.states.base import (
+    Event,
+    SynchronizedData,
 )
+from packages.valory.skills.market_manager_abci.payloads import UpdateBetsPayload
+from packages.valory.skills.market_manager_abci.rounds import UpdateBetsRound
 
 
-class BlacklistingRound(BaseUpdateBetsRound):
+class BlacklistingRound(UpdateBetsRound):
     """A round for updating the bets after blacklisting the sampled one."""
 
+    payload_class: Type[UpdateBetsPayload] = BlacklistingPayload
     done_event = Event.DONE
     none_event = Event.NONE
     no_majority_event = Event.NO_MAJORITY
+    selection_key: Any = (
+        UpdateBetsRound.selection_key,
+        get_name(SynchronizedData.policy),
+    )

packages/valory/skills/trader_abci/skill.yaml

@@ -25,8 +25,8 @@ skills:
 - valory/transaction_settlement_abci:0.1.0:bafybeia6cdxdlqrcwk2maw25fo7dafzd2p3rs7syropvufophk2pitzbwy
 - valory/termination_abci:0.1.0:bafybeieqm46zuccaagnko3qlw6p3nvoohdrfgvpmw467r5lyil2dqrzjsy
 - valory/market_manager_abci:0.1.0:bafybeigmy3g2klkzdhttti2ouxf575ua5y7ajmh4a3yfpukeukqxjky47m
-- valory/decision_maker_abci:0.1.0:bafybeihtydxuzyd35cq23xoxbytpxebpbn3rsyqf4bt6z4sleflqck6riy
-- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeigfkdkpkrktw4wippon6al7vreg3qh7xax33xafomepy6xrv5paa4
+- valory/decision_maker_abci:0.1.0:bafybeifnprgtifiuztmvics57fljowhtj7j5ffph5cwdc2sxh5milcjmze
+- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeib5zblszm5h5gcdxjl55wumda2diztbiti6d67vhy46wduhyvs34e
 behaviours:
   main:
     args: {}

packages/valory/skills/tx_settlement_multiplexer_abci/skill.yaml

@@ -20,7 +20,7 @@ contracts: []
 protocols: []
 skills:
 - valory/abstract_round_abci:0.1.0:bafybeid7neqpxxe4ualp23gxgrtpf6r5u3myr2mmc4uvwameyqshfdoydq
-- valory/decision_maker_abci:0.1.0:bafybeihtydxuzyd35cq23xoxbytpxebpbn3rsyqf4bt6z4sleflqck6riy
+- valory/decision_maker_abci:0.1.0:bafybeifnprgtifiuztmvics57fljowhtj7j5ffph5cwdc2sxh5milcjmze
 behaviours:
   main:
     args: {}