handle presync in carp funnel

packages/engine/paima-funnel/src/cde/reading.ts

@@ -21,13 +21,15 @@ export async function getUngroupedCdeData(
     return extensions.map(_ => []);
   }
   const allData = await Promise.all(
-    extensions.map(extension => getSpecificCdeData(extension, fromBlock, toBlock))
+    extensions.map(extension =>
+      'startBlockHeight' in extension ? getSpecificCdeData(extension, fromBlock, toBlock) : []
+    )
   );
   return allData;
 }
 
 async function getSpecificCdeData(
-  extension: ChainDataExtension,
+  extension: ChainDataExtension & { startBlockHeight: number },
   fromBlock: number,
   toBlock: number
 ): Promise<ChainDataExtensionDatum[]> {

packages/engine/paima-funnel/src/funnels/BaseFunnel.ts

@@ -1,4 +1,10 @@
-import type { ChainData, ChainDataExtension, ChainFunnel, PresyncChainData } from '@paima/runtime';
+import type {
+  ChainData,
+  ChainDataExtension,
+  ChainFunnel,
+  PresyncChainData,
+  ReadPresyncDataFrom,
+} from '@paima/runtime';
 import type { PaimaL2Contract, Web3 } from '@paima/utils';
 import type { FunnelCacheManager } from './FunnelCache';
 import type { PoolClient } from 'pg';
@@ -29,8 +35,10 @@ export class BaseFunnel implements ChainFunnel {
     return [];
   }
 
-  public async readPresyncData(_fromBlock: number, _toBlock: number): Promise<PresyncChainData[]> {
-    return [];
+  public async readPresyncData(
+    _args: ReadPresyncDataFrom
+  ): Promise<{ [network: number]: PresyncChainData[] | 'finished' }> {
+    return {};
   }
 
   public getDbTx(): PoolClient {

packages/engine/paima-funnel/src/funnels/block/funnel.ts

@@ -1,5 +1,5 @@
-import { ENV, doLog, timeout } from '@paima/utils';
-import type { ChainData, ChainFunnel, PresyncChainData } from '@paima/runtime';
+import { ENV, Network, doLog, timeout } from '@paima/utils';
+import type { ChainData, ChainFunnel, PresyncChainData, ReadPresyncDataFrom } from '@paima/runtime';
 import { getBaseChainDataMulti, getBaseChainDataSingle } from '../../reading';
 import { getUngroupedCdeData } from '../../cde/reading';
 import { composeChainData, groupCdeData } from '../../utils';
@@ -110,7 +110,7 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
         ),
         getUngroupedCdeData(this.sharedData.web3, this.sharedData.extensions, fromBlock, toBlock),
       ]);
-      const cdeData = groupCdeData(fromBlock, toBlock, ungroupedCdeData);
+      const cdeData = groupCdeData(Network.CARDANO, fromBlock, toBlock, ungroupedCdeData);
       return composeChainData(baseChainData, cdeData);
     } catch (err) {
       doLog(`[funnel] at ${fromBlock}-${toBlock} caught ${err}`);
@@ -119,9 +119,21 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
   };
 
   public override async readPresyncData(
-    fromBlock: number,
-    toBlock: number
-  ): Promise<PresyncChainData[]> {
+    args: ReadPresyncDataFrom
+  ): Promise<{ [network: number]: PresyncChainData[] | 'finished' }> {
+    let arg = args.find(arg => arg.network == Network.EVM);
+
+    if (!arg) {
+      return [];
+    }
+
+    let fromBlock = arg.from;
+    let toBlock = arg.to;
+
+    if (fromBlock >= ENV.START_BLOCKHEIGHT) {
+      return { [Network.EVM]: 'finished' };
+    }
+
     try {
       toBlock = Math.min(toBlock, ENV.START_BLOCKHEIGHT);
       fromBlock = Math.max(fromBlock, 0);
@@ -135,7 +147,7 @@ export class BlockFunnel extends BaseFunnel implements ChainFunnel {
         fromBlock,
         toBlock
       );
-      return groupCdeData(fromBlock, toBlock, ungroupedCdeData);
+      return { [Network.EVM]: groupCdeData(Network.EVM, fromBlock, toBlock, ungroupedCdeData) };
     } catch (err) {
       doLog(`[paima-funnel::readPresyncData] Exception occurred while reading blocks: ${err}`);
       throw err;

packages/engine/paima-funnel/src/funnels/carp/funnel.ts

@@ -1,23 +1,27 @@
 import {
   ChainDataExtensionType,
   DEFAULT_FUNNEL_TIMEOUT,
+  ENV,
+  delay,
   doLog,
   logError,
   timeout,
 } from '@paima/utils';
-import {
-  type ChainData,
-  type ChainDataExtension,
-  type ChainDataExtensionCardanoDelegation,
-  type ChainFunnel,
-  type PresyncChainData,
+import type {
+  ReadPresyncDataFrom,
+  ChainData,
+  ChainDataExtension,
+  ChainDataExtensionCardanoDelegation,
+  ChainFunnel,
+  PresyncChainData,
 } from '@paima/runtime';
 import { composeChainData, groupCdeData } from '../../utils';
 import { BaseFunnel } from '../BaseFunnel';
 import type { FunnelSharedData } from '../BaseFunnel';
 import type { PoolClient } from 'pg';
 import getCdePoolData from '../../cde/cardanoPool';
 import axios from 'axios';
+import { Network } from '@paima/utils/src/constants';
 
 type BlockInfo = {
   block: {
@@ -29,25 +33,36 @@ type BlockInfo = {
   };
 };
 
-// hardcoded preview time
-const knownTime = 1666656000;
-
 const confirmationDepth = '10';
 
+function knownTime(): number {
+  switch (ENV.CARDANO_NETWORK) {
+    case 'preview':
+      return 1666656000;
+    case 'preprod':
+      return 1666656000;
+    case 'mainnet':
+      return 1666656000;
+    default:
+      throw new Error('unknown cardano network');
+  }
+}
+
 function timestampToAbsoluteSlot(timestamp: number): number {
   const firstSlot = 0;
   // map timestamps with a delta, since we are waiting for blocks.
-  const confirmationTimeDelta = 20 * 10;
+  const confirmationTimeDelta = 20 * Number(confirmationDepth);
 
-  return timestamp - confirmationTimeDelta - knownTime + firstSlot;
+  return timestamp - confirmationTimeDelta - knownTime() + firstSlot;
 }
 
 export class CarpFunnel extends BaseFunnel implements ChainFunnel {
   protected constructor(
     sharedData: FunnelSharedData,
     dbTx: PoolClient,
     private readonly baseFunnel: ChainFunnel,
-    private readonly carpUrl: string
+    private readonly carpUrl: string,
+    private readonly startSlot: number
   ) {
     super(sharedData, dbTx);
     // TODO: replace once TS5 decorators are better supported
@@ -95,19 +110,54 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel {
   }
 
   public override async readPresyncData(
-    fromBlock: number,
-    toBlock: number
-  ): Promise<PresyncChainData[]> {
-    return await this.baseFunnel.readPresyncData(fromBlock, toBlock);
+    args: ReadPresyncDataFrom
+  ): Promise<{ [network: number]: PresyncChainData[] | 'finished' }> {
+    const arg = args.find(arg => arg.network == Network.CARDANO);
+
+    let data = await this.baseFunnel.readPresyncData(args);
+
+    if (arg && arg.from >= 0 && arg.from < this.startSlot) {
+      const poolEvents = await Promise.all(
+        this.sharedData.extensions
+          .filter(extension => extension.cdeType === ChainDataExtensionType.CardanoPool)
+          .map(extension => {
+            const data = getCdePoolData(
+              `${this.carpUrl}/delegation/pool`,
+              extension as ChainDataExtensionCardanoDelegation,
+              arg.from,
+              Math.min(arg.to, this.startSlot - 1),
+              slot => {
+                return slot;
+              }
+            );
+            return data;
+          })
+      );
+
+      let grouped = groupCdeData(Network.CARDANO, arg.from, arg.to, poolEvents);
+
+      if (grouped.length > 0) {
+        data[Network.CARDANO] = grouped;
+      }
+    } else if (arg) {
+      data[Network.CARDANO] = 'finished';
+    }
+
+    return data;
   }
 
   public static async recoverState(
     sharedData: FunnelSharedData,
     dbTx: PoolClient,
     baseFunnel: ChainFunnel,
-    carpUrl: string
+    carpUrl: string,
+    startingBlockHeight: number
   ): Promise<CarpFunnel> {
-    return new CarpFunnel(sharedData, dbTx, baseFunnel, carpUrl);
+    const startingSlot = timestampToAbsoluteSlot(
+      (await sharedData.web3.eth.getBlock(startingBlockHeight)).timestamp as number
+    );
+
+    return new CarpFunnel(sharedData, dbTx, baseFunnel, carpUrl, startingSlot);
   }
 }
 
@@ -122,8 +172,6 @@ async function readDataInternal(
   // the upper range is inclusive
   const max = timestampToAbsoluteSlot(Math.max(...data.map(data => data.timestamp)));
 
-  const sleep = (ms: number): Promise<number> => new Promise(resolve => setTimeout(resolve, ms));
-
   while (true) {
     // TODO: replace with carp client
     const stableBlock = await timeout(
@@ -137,8 +185,7 @@ async function readDataInternal(
       break;
     }
 
-    // TODO: is there a more js-like way of doing this?
-    await sleep(1000);
+    await delay(1000);
   }
 
   const blockNumbers = data.reduce(
@@ -172,7 +219,12 @@ async function readDataInternal(
       })
   );
 
-  let grouped = groupCdeData(data[0].blockNumber, data[data.length - 1].blockNumber, poolEvents);
+  let grouped = groupCdeData(
+    Network.EVM,
+    data[0].blockNumber,
+    data[data.length - 1].blockNumber,
+    poolEvents
+  );
 
   return grouped;
 }
@@ -181,14 +233,21 @@ export async function wrapToCarpFunnel(
   chainFunnel: ChainFunnel,
   sharedData: FunnelSharedData,
   dbTx: PoolClient,
-  carpUrl: string | undefined
+  carpUrl: string | undefined,
+  startingBlockHeight: number
 ): Promise<ChainFunnel> {
   if (!carpUrl) {
     return chainFunnel;
   }
 
   try {
-    const ebp = await CarpFunnel.recoverState(sharedData, dbTx, chainFunnel, carpUrl);
+    const ebp = await CarpFunnel.recoverState(
+      sharedData,
+      dbTx,
+      chainFunnel,
+      carpUrl,
+      startingBlockHeight
+    );
     return ebp;
   } catch (err) {
     doLog('[paima-funnel] Unable to initialize carp events processor:');

packages/engine/paima-funnel/src/funnels/emulated/funnel.ts

@@ -1,7 +1,7 @@
 import type { PoolClient } from 'pg';
 
 import Prando from '@paima/prando';
-import type { ChainData, ChainFunnel, PresyncChainData } from '@paima/runtime';
+import type { ChainData, ChainFunnel, PresyncChainData, ReadPresyncDataFrom } from '@paima/runtime';
 import { ENV, doLog } from '@paima/utils';
 import {
   emulatedSelectLatestPrior,
@@ -17,6 +17,7 @@ import { BaseFunnel } from '../BaseFunnel';
 import type { FunnelSharedData } from '../BaseFunnel';
 
 import { QueuedBlockCacheEntry, RpcCacheEntry, RpcRequestState } from '../FunnelCache';
+import { Network } from '@paima/utils/src/constants';
 
 /**
  * For hash calculation of empty blocks to work,
@@ -157,12 +158,16 @@ export class EmulatedBlocksFunnel extends BaseFunnel {
   };
 
   public override async readPresyncData(
-    fromBlock: number,
-    toBlock: number
-  ): Promise<PresyncChainData[]> {
+    args: ReadPresyncDataFrom
+  ): Promise<{ [network: number]: PresyncChainData[] | 'finished' }> {
     // map base funnel data to the right timestamp range
-    const baseData = await this.ctorData.baseFunnel.readPresyncData(fromBlock, toBlock);
-    return baseData.map(data => {
+    const baseData = await this.ctorData.baseFunnel.readPresyncData(args);
+
+    if (!baseData[Network.EVM] || baseData[Network.EVM] === 'finished') {
+      return baseData;
+    }
+
+    baseData[Network.EVM] = baseData[Network.EVM].map(data => {
       const timestamp = calculateBoundaryTimestamp(
         this.ctorData.startTimestamp,
         ENV.BLOCK_TIME,
@@ -177,6 +182,8 @@ export class EmulatedBlocksFunnel extends BaseFunnel {
         ),
       };
     });
+
+    return baseData;
   }
 
   /**

packages/engine/paima-funnel/src/index.ts

@@ -47,7 +47,13 @@ export class FunnelFactory implements IFunnelFactory {
     // and wrap it with dynamic decorators as needed
 
     let chainFunnel: ChainFunnel = await BlockFunnel.recoverState(this.sharedData, dbTx);
-    chainFunnel = await wrapToCarpFunnel(chainFunnel, this.sharedData, dbTx, ENV.CARP_URL);
+    chainFunnel = await wrapToCarpFunnel(
+      chainFunnel,
+      this.sharedData,
+      dbTx,
+      ENV.CARP_URL,
+      ENV.START_BLOCKHEIGHT
+    );
     chainFunnel = await wrapToEmulatedBlocksFunnel(
       chainFunnel,
       this.sharedData,

packages/engine/paima-funnel/src/reading.ts

@@ -1,7 +1,7 @@
 import type Web3 from 'web3';
 import type { BlockTransactionString } from 'web3-eth';
 
-import { timeout, cutAfterFirstRejected, DEFAULT_FUNNEL_TIMEOUT, doLog, delay } from '@paima/utils';
+import { timeout, cutAfterFirstRejected, DEFAULT_FUNNEL_TIMEOUT, doLog } from '@paima/utils';
 import type { PaimaL2Contract } from '@paima/utils';
 import { TimeoutError, type ChainData } from '@paima/runtime';
 import type { PaimaGameInteraction } from '@paima/utils/src/contract-types/PaimaL2Contract';

packages/engine/paima-funnel/src/utils.ts

@@ -1,6 +1,8 @@
 import type { ChainData, ChainDataExtensionDatum, PresyncChainData } from '@paima/runtime';
+import type { Network } from '@paima/utils';
 
 export function groupCdeData(
+  network: Network,
   fromBlock: number,
   toBlock: number,
   data: ChainDataExtensionDatum[][]
@@ -19,6 +21,7 @@ export function groupCdeData(
     result.push({
       blockNumber,
       extensionDatums,
+      network,
     });
   }
   return result;