parallel evm chain funnel sync + move multiple network configurations…

… to a file (#300) * move network related configs from .env to yaml the goal is to allow specifying multiple evm networks for example, without having to do weird things with env variables. * tmp: disable deps that cause a build error in config/loading.ts * add evm funnel for parallel evm chain * unhardcode some network strings * re-add: parametrize the network in the config again * remove some env validations * save lastBlock in the db and fix internal events composition * fix some eslint issues * fix some block continuity bugs and add comments * uncomment commented code in runtime presync loop * find the maxTimestamp by picking the last element * fix hardcoded network name in carp funnel * improve comments and remove some code duplication * fix web3 instantiation per extension * fix lastBlock computation * fix non root imports * minor cleanup/improvements * evm funnel: fix presync stopping one block after * add presyncStepSize setting per network in order to be able to control rate limitations separatedly * revert wm-core/state to use the ENV file * remove old TODO in evm funnel * improve presync round log in runtime loops * add chain name to the binary search log * remove deployment setting * avoid network presync after finished * improve config error logs + required fields * remove unused type * add backwards compatibility by fallbacking to env file configs * fix typo: even -> event * refactor network tag type into variable * de-hardcode cardano network name * pollingRate and blockTime are not really being used right now * rename evm funnel to parallel evm funnel
PaimaStudios · Mar 2, 2024 · 0308626 · 0308626
1 parent b9841fd
commit 0308626
Show file tree

Hide file tree

Showing 30 changed files with 1,419 additions and 250 deletions.
diff --git a/packages/batcher/utils/src/config-validation.ts b/packages/batcher/utils/src/config-validation.ts
@@ -5,7 +5,6 @@ export function getInvalidEnvVars(): string[] {
 
   const MANDATORY_TRUTHY_VARS: ReadonlyArray<keyof typeof ENV> = [
     // Blockchain
-    'CHAIN_URI',
     'CONTRACT_ADDRESS',
     'DEFAULT_FEE',
     'BATCHER_PRIVATE_KEY',

diff --git a/packages/batcher/webserver/src/index.ts b/packages/batcher/webserver/src/index.ts
@@ -19,6 +19,7 @@ import type { BatchedSubunit } from '@paima/concise';
 import { createMessageForBatcher } from '@paima/concise';
 import { AddressType, getWriteNamespace } from '@paima/utils';
 import { hashBatchSubunit } from '@paima/concise';
+import { GlobalConfig } from '@paima/utils';
 
 const port = ENV.BATCHER_PORT;
 
@@ -105,7 +106,8 @@ async function initializeServer(
   errorCodeToMessage: ErrorMessageFxn,
   truffleProvider: TruffleEvmProvider
 ): Promise<void> {
-  const addressValidator = new AddressValidator(ENV.CHAIN_URI, pool);
+  const [chainName, config] = await GlobalConfig.mainEvmConfig();
+  const addressValidator = new AddressValidator(config.chainUri, pool);
   await addressValidator.initialize();
 
   server.get(

diff --git a/packages/build-utils/paima-build-utils/src/middleware-esbuildconfig.template.cts b/packages/build-utils/paima-build-utils/src/middleware-esbuildconfig.template.cts
@@ -17,13 +17,7 @@ if (process.env.SECURITY_NAMESPACE) {
 }
 
 // Verify env file is filled out
-if (
-  !process.env.CONTRACT_ADDRESS ||
-  !process.env.CHAIN_URI ||
-  !process.env.CHAIN_ID ||
-  !process.env.BACKEND_URI
-)
-  throw new Error('Please ensure you have filled out your .env file');
+if (!process.env.BACKEND_URI) throw new Error('Please ensure you have filled out your .env file');
 
 export const config: esbuild.BuildOptions = {
   // JS output from previous compilation step used here instead of index.ts to have more control over the TS build process

diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts
@@ -12,6 +12,7 @@ export type CacheMapType = {
   [QueuedBlockCacheEntry.SYMBOL]?: QueuedBlockCacheEntry;
   [RpcCacheEntry.SYMBOL]?: RpcCacheEntry;
   [CarpFunnelCacheEntry.SYMBOL]?: CarpFunnelCacheEntry;
+  [EvmFunnelCacheEntry.SYMBOL]?: EvmFunnelCacheEntry;
 };
 export class FunnelCacheManager {
   public cacheEntries: CacheMapType = {};
@@ -114,3 +115,46 @@ export class CarpFunnelCacheEntry implements FunnelCacheEntry {
     this.state = null;
   };
 }
+
+export type EvmFunnelCacheEntryState = {
+  bufferedChainData: ChainData[];
+  timestampToBlockNumber: [number, number][];
+  lastBlock: number | undefined;
+  startBlockHeight: number;
+  lastMaxTimestamp: number;
+};
+
+export class EvmFunnelCacheEntry implements FunnelCacheEntry {
+  private cachedData: Record<number, RpcRequestResult<EvmFunnelCacheEntryState>> = {};
+  public static readonly SYMBOL = Symbol('EvmFunnelCacheEntry');
+
+  public updateState = (
+    chainId: number,
+    bufferedChainData: ChainData[],
+    timestampToBlockNumber: [number, number][],
+    startBlockHeight: number
+  ): void => {
+    this.cachedData[chainId] = {
+      state: RpcRequestState.HasResult,
+      result: {
+        bufferedChainData,
+        timestampToBlockNumber,
+        startBlockHeight,
+        lastBlock: undefined,
+        lastMaxTimestamp: 0,
+      },
+    };
+  };
+
+  public getState(chainId: number): Readonly<RpcRequestResult<EvmFunnelCacheEntryState>> {
+    return (
+      this.cachedData[chainId] ?? {
+        state: RpcRequestState.NotRequested,
+      }
+    );
+  }
+
+  clear: FunnelCacheEntry['clear'] = () => {
+    this.cachedData = {};
+  };
+}
diff --git a/packages/engine/paima-funnel/src/funnels/block/funnel.ts b/packages/engine/paima-funnel/src/funnels/block/funnel.ts
@@ -1,4 +1,4 @@
-import { ENV, Network, doLog, timeout } from '@paima/utils';
+import { ENV, EvmConfig, GlobalConfig, doLog, timeout } from '@paima/utils';
 import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime';
 import type { ChainData, PresyncChainData } from '@paima/sm';
 import { getBaseChainDataMulti, getBaseChainDataSingle } from '../../reading.js';
@@ -9,16 +9,27 @@ import type { FunnelSharedData } from '../BaseFunnel.js';
 import { RpcCacheEntry, RpcRequestState } from '../FunnelCache.js';
 import type { PoolClient } from 'pg';
 import { FUNNEL_PRESYNC_FINISHED } from '@paima/utils';
+import { ConfigNetworkType } from '@paima/utils';
 
 const GET_BLOCK_NUMBER_TIMEOUT = 5000;
 
 export class BlockFunnel extends BaseFunnel implements ChainFunnel {
-  protected constructor(sharedData: FunnelSharedData, dbTx: PoolClient) {
+  config: EvmConfig;
+  chainName: string;
+
+  protected constructor(
+    sharedData: FunnelSharedData,
+    dbTx: PoolClient,
+    config: EvmConfig,
+    chainName: string
+  ) {
     super(sharedData, dbTx);
     // TODO: replace once TS5 decorators are better supported
     this.readData.bind(this);
     this.readPresyncData.bind(this);
     this.getDbTx.bind(this);
+    this.config = config;
+    this.chainName = chainName;
   }
 
   public override async readData(blockHeight: number): Promise<ChainData[]> {
@@ -32,10 +43,10 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
     }
 
     if (toBlock === fromBlock) {
-      doLog(`Block funnel ${ENV.CHAIN_ID}: #${toBlock}`);
+      doLog(`Block funnel ${this.config.chainId}: #${toBlock}`);
       return await this.internalReadDataSingle(fromBlock);
     } else {
-      doLog(`Block funnel ${ENV.CHAIN_ID}: #${fromBlock}-${toBlock}`);
+      doLog(`Block funnel ${this.config.chainId}: #${fromBlock}-${toBlock}`);
       return await this.internalReadDataMulti(fromBlock, toBlock);
     }
   }
@@ -53,7 +64,7 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
 
     const latestBlockQueryState = this.sharedData.cacheManager.cacheEntries[
       RpcCacheEntry.SYMBOL
-    ]?.getState(ENV.CHAIN_ID);
+    ]?.getState(this.config.chainId);
     if (latestBlockQueryState?.state !== RpcRequestState.HasResult) {
       throw new Error(`[funnel] latest block cache entry not found`);
     }
@@ -78,11 +89,12 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
           this.sharedData.web3,
           this.sharedData.paimaL2Contract,
           blockNumber,
-          this.dbTx
+          this.dbTx,
+          this.chainName
         ),
         getUngroupedCdeData(
           this.sharedData.web3,
-          this.sharedData.extensions,
+          this.sharedData.extensions.filter(extension => extension.network === this.chainName),
           blockNumber,
           blockNumber
         ),
@@ -114,11 +126,23 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
           this.sharedData.paimaL2Contract,
           fromBlock,
           toBlock,
-          this.dbTx
+          this.dbTx,
+          this.chainName
+        ),
+        getUngroupedCdeData(
+          this.sharedData.web3,
+          this.sharedData.extensions.filter(extension => extension.network === this.chainName),
+          fromBlock,
+          toBlock
         ),
-        getUngroupedCdeData(this.sharedData.web3, this.sharedData.extensions, fromBlock, toBlock),
       ]);
-      const cdeData = groupCdeData(Network.CARDANO, fromBlock, toBlock, ungroupedCdeData);
+      const cdeData = groupCdeData(
+        this.chainName,
+        ConfigNetworkType.EVM,
+        fromBlock,
+        toBlock,
+        ungroupedCdeData
+      );
       return composeChainData(baseChainData, cdeData);
     } catch (err) {
       doLog(`[funnel] at ${fromBlock}-${toBlock} caught ${err}`);
@@ -128,34 +152,42 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
 
   public override async readPresyncData(
     args: ReadPresyncDataFrom
-  ): Promise<{ [network: number]: PresyncChainData[] | typeof FUNNEL_PRESYNC_FINISHED }> {
-    let arg = args.find(arg => arg.network == Network.EVM);
+  ): Promise<{ [network: string]: PresyncChainData[] | typeof FUNNEL_PRESYNC_FINISHED }> {
+    let arg = args.find(arg => arg.network == this.chainName);
 
     if (!arg) {
-      return [];
+      return {};
     }
 
     let fromBlock = arg.from;
     let toBlock = arg.to;
 
     if (fromBlock >= ENV.START_BLOCKHEIGHT) {
-      return { [Network.EVM]: FUNNEL_PRESYNC_FINISHED };
+      return { [this.chainName]: FUNNEL_PRESYNC_FINISHED };
     }
 
     try {
       toBlock = Math.min(toBlock, ENV.START_BLOCKHEIGHT);
       fromBlock = Math.max(fromBlock, 0);
       if (fromBlock > toBlock) {
-        return [];
+        return {};
       }
 
       const ungroupedCdeData = await getUngroupedCdeData(
         this.sharedData.web3,
-        this.sharedData.extensions,
+        this.sharedData.extensions.filter(extension => extension.network === this.chainName),
         fromBlock,
         toBlock
       );
-      return { [Network.EVM]: groupCdeData(Network.EVM, fromBlock, toBlock, ungroupedCdeData) };
+      return {
+        [this.chainName]: groupCdeData(
+          this.chainName,
+          ConfigNetworkType.EVM,
+          fromBlock,
+          toBlock,
+          ungroupedCdeData
+        ),
+      };
     } catch (err) {
       doLog(`[paima-funnel::readPresyncData] Exception occurred while reading blocks: ${err}`);
       throw err;
@@ -182,8 +214,10 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
       return newEntry;
     })();
 
-    cacheEntry.updateState(ENV.CHAIN_ID, latestBlock);
+    const [chainName, config] = await GlobalConfig.mainEvmConfig();
+
+    cacheEntry.updateState(config.chainId, latestBlock);
 
-    return new BlockFunnel(sharedData, dbTx);
+    return new BlockFunnel(sharedData, dbTx, config, chainName);
   }
 }