From 72d896d437413b1ecc883ac54b546bb0c198e124 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Mon, 4 Dec 2023 18:05:22 -0300 Subject: [PATCH] cache the startingSlot and the previous range upper end just to reduce the number of rpc calls --- .../paima-funnel/src/funnels/FunnelCache.ts | 32 +++++++++ .../paima-funnel/src/funnels/carp/funnel.ts | 66 ++++++++++++++----- 2 files changed, 81 insertions(+), 17 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts index 215a56981..f2c483ec8 100644 --- a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts +++ b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts @@ -11,6 +11,7 @@ export interface FunnelCacheEntry { export type CacheMapType = { [QueuedBlockCacheEntry.SYMBOL]?: QueuedBlockCacheEntry; [RpcCacheEntry.SYMBOL]?: RpcCacheEntry; + [CarpFunnelCacheEntry.SYMBOL]?: CarpFunnelCacheEntry; }; export class FunnelCacheManager { public cacheEntries: CacheMapType = {}; @@ -71,3 +72,34 @@ export class RpcCacheEntry implements FunnelCacheEntry { this.rpcResult = {}; }; } + +export type CarpFunnelCacheEntryState = { + startingSlot: number; + lastPoint: { blockHeight: number; timestamp: number } | undefined; +}; + +export class CarpFunnelCacheEntry implements FunnelCacheEntry { + private state: CarpFunnelCacheEntryState | null = null; + public static readonly SYMBOL = Symbol('CarpFunnelStartingSlot'); + + public updateStartingSlot(startingSlot: number): void { + this.state = { startingSlot, lastPoint: this.state?.lastPoint }; + } + + public updateLastPoint(blockHeight: number, timestamp: number): void { + if (this.state) { + this.state.lastPoint = { blockHeight, timestamp }; + } + } + + public getState(): Readonly { + if (!this.state) { + throw new Error('[carp-funnel] Uninitialized cache entry'); + } + return this.state; + } + + clear: FunnelCacheEntry['clear'] = () => { + this.state = null; + }; +} diff --git a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts index 207347589..5463a79eb 100644 --- a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts @@ -24,8 +24,10 @@ import getCdePoolData from '../../cde/cardanoPool.js'; import { query } from '@dcspark/carp-client/client/src/index'; import { Routes } from '@dcspark/carp-client/shared/routes'; import { FUNNEL_PRESYNC_FINISHED } from '@paima/utils/src/constants'; +import { CarpFunnelCacheEntry } from '../FunnelCache.js'; const confirmationDepth = 10; +const delayForWaitingForFinalityLoop = 1000; function knownShelleyTime(): number { switch (ENV.CARDANO_NETWORK) { @@ -54,7 +56,7 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { dbTx: PoolClient, private readonly baseFunnel: ChainFunnel, private readonly carpUrl: string, - private readonly startSlot: number + private cache: CarpFunnelCacheEntry ) { super(sharedData, dbTx); // TODO: replace once TS5 decorators are better supported @@ -77,21 +79,30 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { this.bufferedData = data; } + const cachedState = this.cache.getState().lastPoint; + + let lastTimestamp; + // there are most likely some slots between the last end of range and the // first block in the current range, so we need to start from the previous point. + if (cachedState && cachedState.blockHeight == blockHeight - 1) { + // this is the last timestamp that was queried as the max in the previous pull + lastTimestamp = cachedState.timestamp; + } else { + lastTimestamp = await timeout( + this.sharedData.web3.eth.getBlock(blockHeight - 1), + DEFAULT_FUNNEL_TIMEOUT + ); - // TODO: cache this? but it's not in the db afaik, so it can't be done on - // recoverState - const lastTimestamp = await timeout( - this.sharedData.web3.eth.getBlock(blockHeight - 1), - DEFAULT_FUNNEL_TIMEOUT - ); + lastTimestamp = lastTimestamp.timestamp as number; + } let grouped = await readDataInternal( this.bufferedData, this.carpUrl, this.sharedData.extensions, - lastTimestamp.timestamp as number + lastTimestamp, + this.cache ); const composed = composeChainData(this.bufferedData, grouped); @@ -108,7 +119,7 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { let basePromise = this.baseFunnel.readPresyncData(args); - if (arg && arg.from >= 0 && arg.from < this.startSlot) { + if (arg && arg.from >= 0 && arg.from < this.cache.getState().startingSlot) { const [poolEvents, data] = await Promise.all([ Promise.all( this.sharedData.extensions @@ -118,7 +129,7 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { this.carpUrl, extension as ChainDataExtensionCardanoDelegation, arg.from, - Math.min(arg.to, this.startSlot - 1), + Math.min(arg.to, this.cache.getState().startingSlot - 1), slot => { return slot; } @@ -154,11 +165,23 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { carpUrl: string, startingBlockHeight: number ): Promise { - const startingSlot = timestampToAbsoluteSlot( - (await sharedData.web3.eth.getBlock(startingBlockHeight)).timestamp as number - ); + const cacheEntry = (async (): Promise => { + const entry = sharedData.cacheManager.cacheEntries[CarpFunnelCacheEntry.SYMBOL]; + if (entry != null) return entry; + + const newEntry = new CarpFunnelCacheEntry(); + sharedData.cacheManager.cacheEntries[CarpFunnelCacheEntry.SYMBOL] = newEntry; + + newEntry.updateStartingSlot( + timestampToAbsoluteSlot( + (await sharedData.web3.eth.getBlock(startingBlockHeight)).timestamp as number + ) + ); - return new CarpFunnel(sharedData, dbTx, baseFunnel, carpUrl, startingSlot); + return newEntry; + })(); + + return new CarpFunnel(sharedData, dbTx, baseFunnel, carpUrl, await cacheEntry); } } @@ -166,13 +189,22 @@ async function readDataInternal( data: ChainData[], carpUrl: string, extensions: ChainDataExtension[], - lastTimestamp: number + lastTimestamp: number, + cache: CarpFunnelCacheEntry ): Promise { // the lower range is exclusive const min = timestampToAbsoluteSlot(lastTimestamp); // the upper range is inclusive - const max = timestampToAbsoluteSlot(Math.max(...data.map(data => data.timestamp))); + const maxElement = data[data.length - 1]; + + const max = timestampToAbsoluteSlot(maxElement.timestamp); + + cache.updateLastPoint(maxElement.blockNumber, maxElement.timestamp); + // Block finality depends on depth, and not on time, so it's possible that a + // block at a non confirmed depth falls in the slot range that we are querying + // here. This waits until the upper end of the range falls in the confirmed + // zone. while (true) { const stableBlock = await timeout( query(carpUrl, Routes.blockLatest, { @@ -185,7 +217,7 @@ async function readDataInternal( break; } - await delay(1000); + await delay(delayForWaitingForFinalityLoop); } const blockNumbers = data.reduce(