chunked multicalls, more logs, memory snapshots, nuke explorer apis

README.md

@@ -58,8 +58,8 @@ Options:
   --max-orders <max-orders>                            Maximum number of orders to place in single drip call (default: 250)
   --buy-amount-slippage-bps <buy-amount-slippage-bps>  Tolerance to add to the quoted buyAmount (default: 100)
   --module <module>                                    COWFeeModule address
-  --token-list-strategy <strategy>                     Strategy to use to get the list of tokens to swap on (choices: "explorer", "chain", default: "explorer")
   --lookback-range <n>                                 Last <n> number of blocks to check the `Trade` events for (default: 1000)
+  --multicall-size <n>                                 max number of calls in a multicall (default: 100)
   -h, --help                                           display help for command
 ```
 
@@ -73,8 +73,8 @@ yarn ts-node index.ts \
   --rpc-url https://eth.llamarpc.com \
   --buy-amount-slippage-bps 100 \
   --module <module-address> \
-  --token-list-strategy explorer \
-  --lookback-range 1000
+  --lookback-range 1000 \
+  --multicall-size 100
 ```
 
 #### Docker
@@ -85,15 +85,16 @@ docker build -t cow-fee .
 # run the container
 docker run --rm \
   -e PRIVATE_KEY=$PRIVATE_KEY \
+  -e LOG_LEVEL=debug \
   cow-fee \
   --network mainnet \
   --max-orders 250 \
   --min-out 0.1 \
   --rpc-url https://eth.llamarpc.com \
   --buy-amount-slippage-bps 100 \
   --module <module-address> \
-  --token-list-strategy explorer \
-  --lookback-range 1000
+  --lookback-range 1000 \
+  --multicall-size 100
 ```
 
 ### Module operations with cast

index.ts

@@ -1,7 +1,11 @@
 import { ethers } from 'ethers';
 import { formatUnits } from 'ethers/lib/utils';
 import { getTokensToSwap, swapTokens } from './ts/cowfee';
-import { IConfig, networkSpecificConfigs } from './ts/common';
+import {
+  IConfig,
+  getLogger,
+  networkSpecificConfigs,
+} from './ts/common';
 import { Command, Option } from '@commander-js/extra-typings';
 import { erc20Abi, moduleAbi, settlementAbi } from './ts/abi';
 
@@ -43,18 +47,23 @@ const readConfig = async (): Promise<
     .addOption(new Option('--module <module>', 'COWFeeModule address'))
     .addOption(
       new Option(
-        '--token-list-strategy <strategy>',
-        'Strategy to use to get the list of tokens to swap on'
+        '--lookback-range <n>',
+        'Last <n> number of blocks to check the `Trade` events for'
       )
-        .choices(['explorer', 'chain'] as const)
-        .default('explorer' as 'explorer' | 'chain')
+        .default(1000)
+        .argParser((x) => +x)
+    )
+    .addOption(
+      new Option('--multicall-size <n>', 'max number of calls in a multicall')
+        .default(100)
+        .argParser((x) => +x)
     )
     .addOption(
       new Option(
-        '--lookback-range <n>',
-        'Last <n> number of blocks to check the `Trade` events for'
+        '--query-logs-size <n>',
+        'max block range to use for eth_getLogs'
       )
-        .default(1000)
+        .default(50000)
         .argParser((x) => +x)
     );
   program.parse();
@@ -67,7 +76,8 @@ const readConfig = async (): Promise<
     buyAmountSlippageBps,
     module: selectedModule,
     lookbackRange,
-    tokenListStrategy,
+    multicallSize,
+    queryLogsSize,
   } = options;
   const network = selectedNetwork || 'mainnet';
 
@@ -120,32 +130,35 @@ const readConfig = async (): Promise<
       buyAmountSlippageBps,
       keeper,
       appData,
-      tokenListStrategy,
       lookbackRange,
       targetSafe,
+      multicallSize,
+      queryLogsSize,
     },
     provider,
   ];
 };
 
+const logger = getLogger('index');
+
 export const dripItAll = async () => {
   // await getAppData().then(console.log);
 
   const [config, provider] = await readConfig();
-  console.log(config.options);
-  // return;
+  logger.info(config.options, 'config options');
   const signer = new ethers.Wallet(config.privateKey, provider);
 
+  process.on('warning', (e) => console.warn(e.stack));
   const tokensToSwap = await getTokensToSwap(config, provider);
-  console.log(
-    'tokensToSwap',
+  logger.info(
     tokensToSwap.map((token) => [
       token.symbol,
       token.address,
       formatUnits(token.balance, token.decimals),
       token.buyAmount,
       token.needsApproval,
-    ])
+    ]),
+    'tokensToSwap'
   );
 
   for (let i = 0; i < tokensToSwap.length; i += config.maxOrders) {

package.json

@@ -5,7 +5,9 @@
     "@cowprotocol/cow-sdk": "^5.2.0",
     "axios": "^1.6.8",
     "commander": "^12.0.0",
-    "ethers": "v5"
+    "ethers": "v5",
+    "pino": "^9.0.0",
+    "pino-pretty": "^11.0.0"
   },
   "devDependencies": {
     "@types/minimist": "^1.2.5",

ts/common.ts

@@ -1,6 +1,7 @@
 import { OrderBookApi, SupportedChainId } from '@cowprotocol/cow-sdk';
 import { ethers } from 'ethers';
 import { multicall3Abi } from './abi';
+import pino from 'pino';
 
 export const networkSpecificConfigs = {
   mainnet: {
@@ -31,8 +32,9 @@ export interface IConfig {
   buyAmountSlippageBps: number;
   keeper: string;
   appData: string;
-  tokenListStrategy: 'explorer' | 'chain';
   lookbackRange: number;
+  multicallSize: number;
+  queryLogsSize: number;
 }
 
 const toChainId = (network: keyof typeof networkSpecificConfigs) => {
@@ -71,3 +73,43 @@ export const getMulticall3 = (provider: ethers.providers.JsonRpcProvider) => {
     provider
   );
 };
+
+export const chunkedMulticall = async (
+  provider: ethers.providers.JsonRpcProvider,
+  calls: { target: string; callData: string }[],
+  chunkSize: number
+) => {
+  const mcall = getMulticall3(provider);
+  const nChunks = Math.ceil(calls.length / chunkSize);
+  const ret = [];
+  for (let i = 0; i < nChunks; i++) {
+    const chunk = calls.slice(i * chunkSize, (i + 1) * chunkSize);
+    ret.push(...(await mcall.tryAggregate(false, chunk)));
+  }
+  return ret;
+};
+
+export const chunkedQueryFilter = async (
+  contract: ethers.Contract,
+  filter: ethers.EventFilter,
+  fromBlock: number,
+  toBlock: number,
+  chunkSize: number
+) => {
+  const ret = [];
+  for (let i = fromBlock; i <= toBlock; i += chunkSize) {
+    ret.push(
+      ...(await contract.queryFilter(
+        filter,
+        i,
+        Math.min(i + chunkSize - 1, toBlock)
+      ))
+    );
+  }
+  return ret;
+};
+
+const logger = pino();
+export const getLogger = (name: string) => {
+  return logger.child({ name, level: process.env.LOG_LEVEL || 'info' });
+};

ts/cowfee.ts

@@ -1,11 +1,13 @@
 import { BigNumber, ContractTransaction, ethers } from 'ethers';
 import {
   IConfig,
+  chunkedMulticall,
+  getLogger,
   getMulticall3,
   getOrderbookApi,
   networkSpecificConfigs,
 } from './common';
-import { getTokenBalances } from './explorer-apis';
+import { getTokenBalances } from './token-fetcher';
 import { multicall3Abi, erc20Abi, moduleAbi } from './abi';
 import { formatUnits, parseEther, parseUnits } from 'ethers/lib/utils';
 import {
@@ -20,12 +22,14 @@ import {
 } from '@cowprotocol/cow-sdk';
 import { MetadataApi } from '@cowprotocol/app-data';
 
+const logger = getLogger('cowfee');
 const ABI_CODER = new ethers.utils.AbiCoder();
 
 const getBalances = async (
   provider: ethers.providers.JsonRpcProvider,
   address: string,
-  tokens: string[]
+  tokens: string[],
+  multicallSize: number
 ): Promise<BigNumber[]> => {
   const Multicall3 = getMulticall3(provider);
   const balanceOfCalldata = new ethers.utils.Interface(
@@ -35,7 +39,7 @@ const getBalances = async (
     target: token,
     callData: balanceOfCalldata,
   }));
-  const balancesRet = await Multicall3.tryAggregate(false, cds);
+  const balancesRet = await chunkedMulticall(provider, cds, multicallSize);
   const balances = balancesRet.map((r: any) =>
     r.success ? ABI_CODER.decode(['uint'], r.returnData)[0] : BigNumber.from(0)
   ) as BigNumber[];
@@ -46,18 +50,20 @@ const getAllowances = async (
   provider: ethers.providers.JsonRpcProvider,
   owner: string,
   spender: string,
-  tokens: string[]
+  tokens: string[],
+  multicallSize: number
 ): Promise<BigNumber[]> => {
   const Multicall3 = getMulticall3(provider);
   const allowanceCalldata = new ethers.utils.Interface(
     erc20Abi
   ).encodeFunctionData('allowance', [owner, spender]);
-  const allowancesRet = await Multicall3.tryAggregate(
-    false,
+  const allowancesRet = await chunkedMulticall(
+    provider,
     tokens.map((token) => ({
       target: token,
       callData: allowanceCalldata,
-    }))
+    })),
+    multicallSize
   );
   const allowances = allowancesRet.map((r: any) =>
     r.success ? ABI_CODER.decode(['uint'], r.returnData)[0] : BigNumber.from(0)
@@ -69,24 +75,30 @@ export const getTokensToSwap = async (
   config: IConfig,
   provider: ethers.providers.JsonRpcProvider
 ) => {
-  const unfiltered = await getTokenBalances(
-    config.gpv2Settlement,
-    config.network,
-    config.tokenListStrategy,
-    config
-  );
+  const unfiltered = await getTokenBalances(config);
 
   // populate the balances and allowances
   const tokenAddresses = unfiltered.map((token) => token.address);
+  logger.info(
+    `Getting token balances and allowances for ${tokenAddresses.length} tokens`
+  );
   const [balances, allowances] = await Promise.all([
-    getBalances(provider, config.gpv2Settlement, tokenAddresses),
+    getBalances(
+      provider,
+      config.gpv2Settlement,
+      tokenAddresses,
+      config.multicallSize
+    ),
     getAllowances(
       provider,
       config.gpv2Settlement,
       config.vaultRelayer,
-      tokenAddresses
+      tokenAddresses,
+      config.multicallSize
     ),
   ]);
+  logger.info('got balances and allowances');
+
   // minValue filter again with _real_ balance
   const unfilteredWithBalanceAndAllowance = unfiltered.map((token, idx) => ({
     ...token,
@@ -95,6 +107,10 @@ export const getTokensToSwap = async (
     needsApproval: allowances[idx].lt(balances[idx]),
   }));
 
+
+  logger.info(
+    `Getting quotes for ${unfilteredWithBalanceAndAllowance.length} tokens`
+  );
   // filter shitcoins with no liquidity by using the quotes api
   const orderBookApi = getOrderbookApi(config);
   const quotes = await Promise.allSettled(
@@ -119,7 +135,7 @@ export const getTokensToSwap = async (
         (quotes[i].status === 'fulfilled' &&
           (quotes[i] as PromiseFulfilledResult<OrderQuoteResponse>).value.quote
             .buyAmount) ||
-          0
+        0
       )
         .mul(10000 - config.buyAmountSlippageBps)
         .div(10000),
@@ -196,17 +212,17 @@ export const swapTokens = async (
       })
     )
   );
-  console.log(
-    'failed',
+  logger.info(
     orders
       .filter((x) => x.status === 'rejected')
-      .map((x) => (x as PromiseRejectedResult).reason)
+      .map((x) => (x as PromiseRejectedResult).reason),
+    'failed orders'
   );
-  console.log(
-    'orderIds',
+  logger.info(
     orders
       .filter((x) => x.status === 'fulfilled')
-      .map((x) => (x as PromiseFulfilledResult<string>).value)
+      .map((x) => (x as PromiseFulfilledResult<string>).value),
+    'orderIds'
   );
   // only execute drip for successfully created orders
   const toActuallySwap = toSwap.filter(
@@ -232,7 +248,7 @@ export const swapTokens = async (
     toApprove,
     toDrip
   );
-  console.log('dripTx', dripTx.hash);
+  logger.info(dripTx.hash, 'dripTx');
   const dripTxReceipt = await dripTx.wait();
   if (dripTxReceipt.status === 0)
     throw new Error(`drip failed: ${dripTxReceipt.transactionHash}`);