From 6e1a5c477a4a673a61f7bc12e51d6df0f4067767 Mon Sep 17 00:00:00 2001 From: Ardian Date: Fri, 29 Sep 2023 10:51:35 +0200 Subject: [PATCH 1/2] chore: retrieve events in smaller chunks --- packages/packages.json | 14 +++++++------- packages/valory/agents/trader/aea-config.yaml | 10 +++++----- .../contracts/conditional_tokens/contract.py | 2 +- .../contracts/conditional_tokens/contract.yaml | 2 +- packages/valory/contracts/realitio/contract.py | 15 +++++++++------ packages/valory/contracts/realitio/contract.yaml | 2 +- packages/valory/services/trader/service.yaml | 2 +- .../valory/skills/decision_maker_abci/skill.yaml | 4 ++-- packages/valory/skills/trader_abci/skill.yaml | 4 ++-- .../tx_settlement_multiplexer_abci/skill.yaml | 2 +- 10 files changed, 30 insertions(+), 27 deletions(-) diff --git a/packages/packages.json b/packages/packages.json index 621786ba6..67a8499e8 100644 --- a/packages/packages.json +++ b/packages/packages.json @@ -1,17 +1,17 @@ { "dev": { "skill/valory/market_manager_abci/0.1.0": "bafybeidnqerwkljbjgog73qaa5duu5ymsfjs6jsszeupseshx7go3n6afq", - "skill/valory/decision_maker_abci/0.1.0": "bafybeiersmwlzih3yhv5nhy7ay6lqe377wn757pugsl457ypoik57aotom", - "skill/valory/trader_abci/0.1.0": "bafybeiajxg7a47vubwyhqj7brrqheoq3ohwqlq3t4uck75r5p45klt43be", + "skill/valory/decision_maker_abci/0.1.0": "bafybeienzc3c5xygb4z3l57zivtc3zexpoochjzmiil7pv646jhxrgetqm", + "skill/valory/trader_abci/0.1.0": "bafybeiho45hvspxswav2zial53i3aeeq6dpb6nrpwpzltor2qphbzld26y", "contract/valory/market_maker/0.1.0": "bafybeidaz4dol7qsrwitw5jwasjtvd2vtlxhxizwkzen5kurblwacz4biu", - "agent/valory/trader/0.1.0": "bafybeihyksse4ohe3x4tshyrlsral6dgptvb6uqvoteu7prlsdggqjk7tm", - "service/valory/trader/0.1.0": "bafybeiemvxyaai7lcfjx5zujfixxfew6vw2xfdpdqzwyyiwnsg6jxu55ny", + "agent/valory/trader/0.1.0": "bafybeigh5kh6qdyrflwbv2s7hmoypp6qtlbwiavb7apms3z4cnpqksjxce", + "service/valory/trader/0.1.0": "bafybeifkb5pjds3yyda6fubolr36zjfypy7nbq7z3v5etiodr35vhuznza", "contract/valory/erc20/0.1.0": "bafybeid6p64a6tnovatbwybc6ocdd4u7bqkxsb4ks52rvf7ozcxxl6iaf4", - "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeicwz2jiwwkcufavqc3ttxkaowjcguq7cuaczzlsvonbv2fk373ezu", + "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeidj6a3arhm6wpswofka3liwdaofv5dh5x5dv2sbvgqrrnb6djdr5q", "contract/valory/mech/0.1.0": "bafybeiddc6pgurpyja2k64wzsb3jgcvl254s7rplgt5iooftsyfalqlbfq", - "contract/valory/realitio/0.1.0": "bafybeicdgm2a7evjw6szcpo3uaam5mzd6axtevtzwvejr6uaeymbg437da", + "contract/valory/realitio/0.1.0": "bafybeiewpazpi2wgixmiu64kauwn7qes3xzphqid4pxvznm2himv4tciee", "contract/valory/realitio_proxy/0.1.0": "bafybeibvndq6756qck7forgeavhdbn6ykgqs2ufyg7n5g6qdfpveatxuwy", - "contract/valory/conditional_tokens/0.1.0": "bafybeig26ktayat3m32to3riescg7plsahdylmpf3ubsguiosqixdwjm6m", + "contract/valory/conditional_tokens/0.1.0": "bafybeigryh3d4iuyj4nruwekvurnnbzyj77flmn4vtgatwdpvbn2erpffa", "contract/valory/agent_registry/0.1.0": "bafybeigpd6wvnbb7fbf6yd77rugepv5hrbb6l3qfodstwgemagtcgne3bm" }, "third_party": { diff --git a/packages/valory/agents/trader/aea-config.yaml b/packages/valory/agents/trader/aea-config.yaml index 7f8694200..db52e1b58 100644 --- a/packages/valory/agents/trader/aea-config.yaml +++ b/packages/valory/agents/trader/aea-config.yaml @@ -22,8 +22,8 @@ contracts: - valory/erc20:0.1.0:bafybeid6p64a6tnovatbwybc6ocdd4u7bqkxsb4ks52rvf7ozcxxl6iaf4 - valory/multisend:0.1.0:bafybeieg4tywd5lww2vygvpkilg3hcepa4rmhehjuamyvdf6vazt554v6u - valory/mech:0.1.0:bafybeiddc6pgurpyja2k64wzsb3jgcvl254s7rplgt5iooftsyfalqlbfq -- valory/conditional_tokens:0.1.0:bafybeig26ktayat3m32to3riescg7plsahdylmpf3ubsguiosqixdwjm6m -- valory/realitio:0.1.0:bafybeicdgm2a7evjw6szcpo3uaam5mzd6axtevtzwvejr6uaeymbg437da +- valory/conditional_tokens:0.1.0:bafybeigryh3d4iuyj4nruwekvurnnbzyj77flmn4vtgatwdpvbn2erpffa +- valory/realitio:0.1.0:bafybeiewpazpi2wgixmiu64kauwn7qes3xzphqid4pxvznm2himv4tciee - valory/realitio_proxy:0.1.0:bafybeibvndq6756qck7forgeavhdbn6ykgqs2ufyg7n5g6qdfpveatxuwy protocols: - open_aea/signing:1.0.0:bafybeifuxs7gdg2okbn7uofymenjlmnih2wxwkym44lsgwmklgwuckxm2m @@ -41,10 +41,10 @@ skills: - valory/reset_pause_abci:0.1.0:bafybeigeoc363gv3wp2rrmk6p2fdxney33nxd3owtpfugzapgruwe4klyu - valory/termination_abci:0.1.0:bafybeigqpij2sgrpnilqjljfciixop4fldq5qceixc7534q6af4potdmdm - valory/transaction_settlement_abci:0.1.0:bafybeia7rzsbea3ch4gcafyp3z6uvqh4npws2xpdwbkkdbrqqpjops7nui -- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeicwz2jiwwkcufavqc3ttxkaowjcguq7cuaczzlsvonbv2fk373ezu +- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeidj6a3arhm6wpswofka3liwdaofv5dh5x5dv2sbvgqrrnb6djdr5q - valory/market_manager_abci:0.1.0:bafybeidnqerwkljbjgog73qaa5duu5ymsfjs6jsszeupseshx7go3n6afq -- valory/decision_maker_abci:0.1.0:bafybeiersmwlzih3yhv5nhy7ay6lqe377wn757pugsl457ypoik57aotom -- valory/trader_abci:0.1.0:bafybeiajxg7a47vubwyhqj7brrqheoq3ohwqlq3t4uck75r5p45klt43be +- valory/decision_maker_abci:0.1.0:bafybeienzc3c5xygb4z3l57zivtc3zexpoochjzmiil7pv646jhxrgetqm +- valory/trader_abci:0.1.0:bafybeiho45hvspxswav2zial53i3aeeq6dpb6nrpwpzltor2qphbzld26y default_ledger: ethereum required_ledgers: - ethereum diff --git a/packages/valory/contracts/conditional_tokens/contract.py b/packages/valory/contracts/conditional_tokens/contract.py index b1ded5aa7..b3f1ceebe 100644 --- a/packages/valory/contracts/conditional_tokens/contract.py +++ b/packages/valory/contracts/conditional_tokens/contract.py @@ -51,7 +51,7 @@ def check_redeemed( condition_ids: List[HexBytes], index_sets: List[List[int]], from_block_numbers: Dict[HexBytes, BlockIdentifier], - chunk_size: int = 50_000, + chunk_size: int = 10_000, ) -> JSONLike: """Filter to find out whether a position has already been redeemed.""" earliest_block = DEFAULT_FROM_BLOCK diff --git a/packages/valory/contracts/conditional_tokens/contract.yaml b/packages/valory/contracts/conditional_tokens/contract.yaml index b754d54f6..33427cced 100644 --- a/packages/valory/contracts/conditional_tokens/contract.yaml +++ b/packages/valory/contracts/conditional_tokens/contract.yaml @@ -8,7 +8,7 @@ aea_version: '>=1.0.0, <2.0.0' fingerprint: __init__.py: bafybeidhdxio3oq5gqdnxmngumvt3fcd6zyiyrpk5f2k4dwhflbg4e5iky build/ConditionalTokens.json: bafybeia2ahis7zx2yhhf23kpkcxu56hto6fwg6ptjg5ld46lp4dgz7cz3e - contract.py: bafybeibex3xrphf7ssmblx3bo6cy3qzv2wk3nxmnv3k6w56yr6cwnkejlq + contract.py: bafybeigjmnpqkauiy34dlzj4iajrgrquqsw3oucpk5pbfedby3bmgoo3ge fingerprint_ignore_patterns: [] class_name: ConditionalTokensContract contract_interface_paths: diff --git a/packages/valory/contracts/realitio/contract.py b/packages/valory/contracts/realitio/contract.py index f7282a03c..9c12303dc 100644 --- a/packages/valory/contracts/realitio/contract.py +++ b/packages/valory/contracts/realitio/contract.py @@ -59,17 +59,20 @@ def _get_claim_params( contract_address: str, from_block: BlockIdentifier, question_id: bytes, + chunk_size: int = 10_000, ) -> Union[str, Tuple[bytes, List[bytes], List[ChecksumAddress], List[int], List[bytes]]]: """Filters the `LogNewAnswer` event by question id to calculate the history hashes.""" contract_instance = cls.get_instance(ledger_api, contract_address) - - answer_filter = contract_instance.events.LogNewAnswer.build_filter() - answer_filter.fromBlock = from_block - answer_filter.toBlock = "latest" - answer_filter.args.question_id.match_single(question_id) + to_block = ledger_api.api.eth.block_number try: - answered = list(answer_filter.deploy(ledger_api.api).get_all_entries()) + answered = [] + for chunk in range(from_block, to_block, chunk_size): + answer_filter = contract_instance.events.LogNewAnswer.build_filter() + answer_filter.fromBlock = chunk + answer_filter.toBlock = min(chunk + chunk_size, to_block) + answer_filter.args.question_id.match_single(question_id) + answered.extend(list(answer_filter.deploy(ledger_api.api).get_all_entries())) except (Urllib3ReadTimeoutError, RequestsReadTimeoutError): msg = ( "The RPC timed out! This usually happens if the filtering is too wide. " diff --git a/packages/valory/contracts/realitio/contract.yaml b/packages/valory/contracts/realitio/contract.yaml index 8a1cf23fb..7385a8d1f 100644 --- a/packages/valory/contracts/realitio/contract.yaml +++ b/packages/valory/contracts/realitio/contract.yaml @@ -8,7 +8,7 @@ aea_version: '>=1.0.0, <2.0.0' fingerprint: __init__.py: bafybeictahkgfmlqv5kksvj6klmxtmjdpeq4sp3x7dp2yr5x4kmzbcihse build/Realitio.json: bafybeiagi7zoeoy5s7duhg4oeuekj2s6z5mad2z6g2pn3n5elsvze25qiu - contract.py: bafybeicvb4v6iw7dkjhnlgaiewkrdmkozxxxu24zqnmya5aiahnttnjhwa + contract.py: bafybeibjkaeoqthdg6wzp2lauvb4laxifm66iszuawljl4vnqw4d6sbh3i fingerprint_ignore_patterns: [] class_name: RealitioContract contract_interface_paths: diff --git a/packages/valory/services/trader/service.yaml b/packages/valory/services/trader/service.yaml index d8fbd0945..7da6a2a21 100644 --- a/packages/valory/services/trader/service.yaml +++ b/packages/valory/services/trader/service.yaml @@ -7,7 +7,7 @@ license: Apache-2.0 fingerprint: README.md: bafybeigtuothskwyvrhfosps2bu6suauycolj67dpuxqvnicdrdu7yhtvq fingerprint_ignore_patterns: [] -agent: valory/trader:0.1.0:bafybeihyksse4ohe3x4tshyrlsral6dgptvb6uqvoteu7prlsdggqjk7tm +agent: valory/trader:0.1.0:bafybeigh5kh6qdyrflwbv2s7hmoypp6qtlbwiavb7apms3z4cnpqksjxce number_of_agents: 4 deployment: {} --- diff --git a/packages/valory/skills/decision_maker_abci/skill.yaml b/packages/valory/skills/decision_maker_abci/skill.yaml index e1550e3ef..9e3ea76a1 100644 --- a/packages/valory/skills/decision_maker_abci/skill.yaml +++ b/packages/valory/skills/decision_maker_abci/skill.yaml @@ -49,8 +49,8 @@ contracts: - valory/erc20:0.1.0:bafybeid6p64a6tnovatbwybc6ocdd4u7bqkxsb4ks52rvf7ozcxxl6iaf4 - valory/multisend:0.1.0:bafybeieg4tywd5lww2vygvpkilg3hcepa4rmhehjuamyvdf6vazt554v6u - valory/mech:0.1.0:bafybeiddc6pgurpyja2k64wzsb3jgcvl254s7rplgt5iooftsyfalqlbfq -- valory/conditional_tokens:0.1.0:bafybeig26ktayat3m32to3riescg7plsahdylmpf3ubsguiosqixdwjm6m -- valory/realitio:0.1.0:bafybeicdgm2a7evjw6szcpo3uaam5mzd6axtevtzwvejr6uaeymbg437da +- valory/conditional_tokens:0.1.0:bafybeigryh3d4iuyj4nruwekvurnnbzyj77flmn4vtgatwdpvbn2erpffa +- valory/realitio:0.1.0:bafybeiewpazpi2wgixmiu64kauwn7qes3xzphqid4pxvznm2himv4tciee - valory/realitio_proxy:0.1.0:bafybeibvndq6756qck7forgeavhdbn6ykgqs2ufyg7n5g6qdfpveatxuwy - valory/agent_registry:0.1.0:bafybeigpd6wvnbb7fbf6yd77rugepv5hrbb6l3qfodstwgemagtcgne3bm protocols: diff --git a/packages/valory/skills/trader_abci/skill.yaml b/packages/valory/skills/trader_abci/skill.yaml index b81041fff..0fee5ea84 100644 --- a/packages/valory/skills/trader_abci/skill.yaml +++ b/packages/valory/skills/trader_abci/skill.yaml @@ -25,8 +25,8 @@ skills: - valory/transaction_settlement_abci:0.1.0:bafybeia7rzsbea3ch4gcafyp3z6uvqh4npws2xpdwbkkdbrqqpjops7nui - valory/termination_abci:0.1.0:bafybeigqpij2sgrpnilqjljfciixop4fldq5qceixc7534q6af4potdmdm - valory/market_manager_abci:0.1.0:bafybeidnqerwkljbjgog73qaa5duu5ymsfjs6jsszeupseshx7go3n6afq -- valory/decision_maker_abci:0.1.0:bafybeiersmwlzih3yhv5nhy7ay6lqe377wn757pugsl457ypoik57aotom -- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeicwz2jiwwkcufavqc3ttxkaowjcguq7cuaczzlsvonbv2fk373ezu +- valory/decision_maker_abci:0.1.0:bafybeienzc3c5xygb4z3l57zivtc3zexpoochjzmiil7pv646jhxrgetqm +- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeidj6a3arhm6wpswofka3liwdaofv5dh5x5dv2sbvgqrrnb6djdr5q behaviours: main: args: {} diff --git a/packages/valory/skills/tx_settlement_multiplexer_abci/skill.yaml b/packages/valory/skills/tx_settlement_multiplexer_abci/skill.yaml index 8e4cb454f..37bd49539 100644 --- a/packages/valory/skills/tx_settlement_multiplexer_abci/skill.yaml +++ b/packages/valory/skills/tx_settlement_multiplexer_abci/skill.yaml @@ -20,7 +20,7 @@ contracts: [] protocols: [] skills: - valory/abstract_round_abci:0.1.0:bafybeih2fyfb6kkf7r45pvdk7pyyebr5xloia4xiqxtb3qsrasnstqmepq -- valory/decision_maker_abci:0.1.0:bafybeiersmwlzih3yhv5nhy7ay6lqe377wn757pugsl457ypoik57aotom +- valory/decision_maker_abci:0.1.0:bafybeienzc3c5xygb4z3l57zivtc3zexpoochjzmiil7pv646jhxrgetqm behaviours: main: args: {} From 879625ea398e302bafc7a905907617a0a06fec9f Mon Sep 17 00:00:00 2001 From: Ardian Date: Fri, 29 Sep 2023 13:51:25 +0200 Subject: [PATCH 2/2] chore: gitleaks --- .gitleaksignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitleaksignore b/.gitleaksignore index f98f86881..09699f347 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -7,3 +7,4 @@ cb7b8233efe65445b5835dd2ea6c93eac3364d39:packages/valory/skills/tx_settlement_mu e601530b1b2503589126e99070459e3e46a789d5:packages/valory/skills/decision_maker_abci/skill.yaml:generic-api-key:166 e601530b1b2503589126e99070459e3e46a789d5:packages/valory/skills/trader_abci/skill.yaml:generic-api-key:159 9150ab78d5e30f576226cf38fb4cc62d73b09752:packages/valory/contracts/conditional_tokens/contract.yaml:generic-api-key:10 +cd8a3159e660a5eb6838657db4d7537912e657bf:packages/valory/skills/market_manager_abci/skill.yaml:generic-api-key:17