improve(inventory-manager): Check ERC20 balance before submitting cross chain bridge

src/clients/InventoryClient.ts

@@ -12,6 +12,7 @@ import {
   DefaultLogLevels,
   TransactionResponse,
   AnyObject,
+  ERC20,
 } from "../utils";
 import { HubPoolClient, TokenClient, BundleDataClient } from ".";
 import { AdapterManager, CrossChainTransferClient } from "./bridges";
@@ -317,9 +318,34 @@ export class InventoryClient {
         // the rebalance to this particular chain. Note that if the sum of all rebalances required exceeds the l1
         // balance then this logic ensures that we only fill the first n number of chains where we can.
         if (amount.lt(balance)) {
-          possibleRebalances.push(rebalance);
-          // Decrement token balance in client for this chain and increment cross chain counter.
-          this.trackCrossChainTransfer(l1Token, amount, chainId);
+          // As a precautionary step before proceeding, check that the token balance for the token we're about to send
+          // hasn't changed on L1. It's possible its changed since we updated the inventory due to one or more of the
+          // RPC's returning slowly, leading to concurrent/overlapping instances of the bot running.
+          const expectedBalance = this.tokenClient.getBalance(1, l1Token);
+          const tokenContract = new Contract(l1Token, ERC20.abi, this.hubPoolClient.hubPool.signer);
+          const currentBalance = await tokenContract.balanceOf(this.relayer);
+          if (!expectedBalance.eq(currentBalance)) {
+            this.logger.warn({
+              at: "InventoryClient",
+              message: "🚧 Token balance on Ethereum changed before sending transaction, skipping rebalance",
+              l1Token,
+              l2ChainId: chainId,
+              expectedBalance,
+              currentBalance,
+            });
+            continue;
+          } else {
+            this.logger.debug({
+              at: "InventoryClient",
+              message: "Token balance in relayer on Ethereum is as expected, sending cross chain transfer",
+              l1Token,
+              l2ChainId: chainId,
+              expectedBalance,
+            });
+            possibleRebalances.push(rebalance);
+            // Decrement token balance in client for this chain and increment cross chain counter.
+            this.trackCrossChainTransfer(l1Token, amount, chainId);
+          }
         } else {
           // Extract unexecutable rebalances for logging.
           unexecutedRebalances.push(rebalance);

src/relayer/RelayerConfig.ts

@@ -10,6 +10,7 @@ export class RelayerConfig extends CommonConfig {
   readonly debugProfitability: boolean;
   // Whether token price fetch failures will be ignored when computing relay profitability.
   // If this is false, the relayer will throw an error when fetching prices fails.
+  readonly skipRelays: boolean;
   readonly sendingRelaysEnabled: boolean;
   readonly sendingSlowRelaysEnabled: boolean;
   readonly sendingRefundRequestsEnabled: boolean;
@@ -44,6 +45,7 @@ export class RelayerConfig extends CommonConfig {
       RELAYER_INVENTORY_CONFIG,
       RELAYER_TOKENS,
       SEND_RELAYS,
+      SKIP_RELAYS,
       SEND_SLOW_RELAYS,
       SEND_REFUND_REQUESTS,
       MIN_RELAYER_FEE_PCT,
@@ -132,6 +134,7 @@ export class RelayerConfig extends CommonConfig {
     this.debugProfitability = DEBUG_PROFITABILITY === "true";
     this.relayerGasMultiplier = toBNWei(RELAYER_GAS_MULTIPLIER || Constants.DEFAULT_RELAYER_GAS_MULTIPLIER);
     this.sendingRelaysEnabled = SEND_RELAYS === "true";
+    this.skipRelays = SKIP_RELAYS === "true";
     this.sendingRefundRequestsEnabled = SEND_REFUND_REQUESTS !== "false";
     this.sendingSlowRelaysEnabled = SEND_SLOW_RELAYS === "true";
     this.acceptInvalidFills = ACCEPT_INVALID_FILLS === "true";

src/relayer/index.ts

@@ -24,16 +24,18 @@ export async function runRelayer(_logger: winston.Logger, baseSigner: Wallet): P
     for (;;) {
       await updateRelayerClients(relayerClients, config);
 
-      // @note: For fills with a different repaymentChainId, refunds are requested on the _subsequent_ relayer run.
-      // Refunds requests are enqueued before new fills, so fillRelay simulation occurs closest to txn submission.
-      const version = configStoreClient.getConfigStoreVersionForTimestamp();
-      if (sdkUtils.isUBA(version) && version <= configStoreClient.configStoreVersion) {
-        await relayer.requestRefunds(config.sendingSlowRelaysEnabled);
-      }
+      if (!config.skipRelays) {
+        // @note: For fills with a different repaymentChainId, refunds are requested on the _subsequent_ relayer run.
+        // Refunds requests are enqueued before new fills, so fillRelay simulation occurs closest to txn submission.
+        const version = configStoreClient.getConfigStoreVersionForTimestamp();
+        if (sdkUtils.isUBA(version) && version <= configStoreClient.configStoreVersion) {
+          await relayer.requestRefunds(config.sendingSlowRelaysEnabled);
+        }
 
-      await relayer.checkForUnfilledDepositsAndFill(config.sendingSlowRelaysEnabled);
+        await relayer.checkForUnfilledDepositsAndFill(config.sendingSlowRelaysEnabled);
 
-      await relayerClients.multiCallerClient.executeTransactionQueue(!config.sendingRelaysEnabled);
+        await relayerClients.multiCallerClient.executeTransactionQueue(!config.sendingRelaysEnabled);
+      }
 
       // Unwrap WETH after filling deposits so we don't mess up slow fill logic, but before rebalancing
       // any tokens so rebalancing can take into account unwrapped WETH balances.

test/InventoryClient.InventoryRebalance.ts

@@ -1,5 +1,6 @@
 import {
   BigNumber,
+  FakeContract,
   SignerWithAddress,
   createSpyLogger,
   deployConfigStore,
@@ -9,6 +10,7 @@ import {
   lastSpyLogIncludes,
   randomAddress,
   sinon,
+  smock,
   spyLogIncludes,
   toBN,
   toWei,
@@ -19,6 +21,7 @@ import { ConfigStoreClient, InventoryClient } from "../src/clients"; // Tested
 import { CrossChainTransferClient } from "../src/clients/bridges";
 import { InventoryConfig } from "../src/interfaces";
 import { MockAdapterManager, MockBundleDataClient, MockHubPoolClient, MockTokenClient } from "./mocks/";
+import { ERC20 } from "../src/utils";
 
 const toMegaWei = (num: string | number | BigNumber) => ethers.utils.parseUnits(num.toString(), 6);
 
@@ -33,6 +36,9 @@ const enabledChainIds = [1, 10, 137, 42161];
 const mainnetWeth = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2";
 const mainnetUsdc = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48";
 
+let mainnetWethContract: FakeContract;
+let mainnetUsdcContract: FakeContract;
+
 // construct two mappings of chainId to token address. Set the l1 token address to the "real" token address.
 const l2TokensForWeth = { 1: mainnetWeth };
 const l2TokensForUsdc = { 1: mainnetUsdc };
@@ -103,6 +109,12 @@ describe("InventoryClient: Rebalancing inventory", async function () {
       crossChainTransferClient
     );
 
+    mainnetWethContract = await smock.fake(ERC20.abi, { address: mainnetWeth });
+    mainnetUsdcContract = await smock.fake(ERC20.abi, { address: mainnetUsdc });
+
+    mainnetWethContract.balanceOf.whenCalledWith(owner.address).returns(initialAllocation[1][mainnetWeth]);
+    mainnetUsdcContract.balanceOf.whenCalledWith(owner.address).returns(initialAllocation[1][mainnetUsdc]);
+
     seedMocks(initialAllocation);
   });
 
@@ -257,6 +269,36 @@ describe("InventoryClient: Rebalancing inventory", async function () {
     // expect(spyLogIncludes(spy, -2, `"42161":{"actualBalanceOnChain":"945.00"`)).to.be.true;
     // expect(spyLogIncludes(spy, -2, `"proRataShare":"7.00%"`)).to.be.true;
   });
+
+  it("Refuses to send rebalance when ERC20 balance changes", async function () {
+    await inventoryClient.update();
+    await inventoryClient.rebalanceInventoryIfNeeded();
+
+    // Now, simulate the re-allocation of funds. Say that the USDC on arbitrum is half used up. This will leave arbitrum
+    // with 500 USDC, giving a percentage of 500/14000 = 0.035. This is below the threshold of 0.5 so we should see
+    // a re-balance executed in size of the target allocation + overshoot percentage.
+    const initialBalance = initialAllocation[42161][mainnetUsdc];
+    expect(tokenClient.getBalance(42161, l2TokensForUsdc[42161])).to.equal(initialBalance);
+    const withdrawAmount = toMegaWei(500);
+    tokenClient.decrementLocalBalance(42161, l2TokensForUsdc[42161], withdrawAmount);
+    expect(tokenClient.getBalance(42161, l2TokensForUsdc[42161])).to.equal(withdrawAmount);
+
+    // The allocation of this should now be below the threshold of 5% so the inventory client should instruct a rebalance.
+    const expectedAlloc = withdrawAmount.mul(toWei(1)).div(initialUsdcTotal.sub(withdrawAmount));
+    expect(inventoryClient.getCurrentAllocationPct(mainnetUsdc, 42161)).to.equal(expectedAlloc);
+
+    // Set USDC balance to be lower than expected.
+    mainnetUsdcContract.balanceOf
+      .whenCalledWith(owner.address)
+      .returns(initialAllocation[1][mainnetUsdc].sub(toMegaWei(1)));
+    await inventoryClient.rebalanceInventoryIfNeeded();
+    expect(spyLogIncludes(spy, -2, "Token balance on Ethereum changed")).to.be.true;
+
+    // Reset and check again.
+    mainnetUsdcContract.balanceOf.whenCalledWith(owner.address).returns(initialAllocation[1][mainnetUsdc]);
+    await inventoryClient.rebalanceInventoryIfNeeded();
+    expect(lastSpyLogIncludes(spy, "Executed Inventory rebalances")).to.be.true;
+  });
 });
 
 function seedMocks(seedBalances: { [chainId: string]: { [token: string]: BigNumber } }) {