From d6cf58168f119b6df46a7622587a9e659b2c77bd Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Thu, 21 Mar 2024 00:06:45 -0300 Subject: [PATCH 01/31] mina funnel: wip --- .../paima-funnel/src/cde/minaGeneric.ts | 68 +++ .../engine/paima-funnel/src/cde/reading.ts | 4 + .../paima-funnel/src/funnels/FunnelCache.ts | 57 +++ .../paima-funnel/src/funnels/carp/funnel.ts | 14 +- .../paima-funnel/src/funnels/mina/funnel.ts | 399 ++++++++++++++++++ packages/engine/paima-funnel/src/index.ts | 11 + .../paima-runtime/src/cde-config/loading.ts | 16 + .../engine/paima-runtime/src/runtime-loops.ts | 5 +- packages/engine/paima-sm/src/cde-generic.ts | 4 +- .../engine/paima-sm/src/cde-processing.ts | 2 + packages/engine/paima-sm/src/index.ts | 45 ++ packages/engine/paima-sm/src/types.ts | 36 +- .../paima-utils/src/config/loading.ts | 38 +- .../paima-utils/src/config/singleton.ts | 16 +- .../paima-sdk/paima-utils/src/constants.ts | 2 + 15 files changed, 706 insertions(+), 11 deletions(-) create mode 100644 packages/engine/paima-funnel/src/cde/minaGeneric.ts create mode 100644 packages/engine/paima-funnel/src/funnels/mina/funnel.ts diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts new file mode 100644 index 000000000..30abcad55 --- /dev/null +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -0,0 +1,68 @@ +import { + CdeMinaGenericDatum, + ChainDataExtensionDatum, + ChainDataExtensionMinaGeneric, +} from '@paima/sm'; +import { ChainDataExtensionDatumType } from '@paima/utils'; + +export default async function getCdeData( + minaArchive: string, + extension: ChainDataExtensionMinaGeneric, + fromTimestamp: number, + toTimestamp: number, + getBlockNumber: (minaTimestamp: number) => number, + network: string +): Promise { + console.log('from-to', fromTimestamp, toTimestamp); + const data = (await fetch(minaArchive, { + method: 'POST', + + headers: { + 'Content-Type': 'application/json', + }, + + body: JSON.stringify({ + query: ` + { + events( + input: { + address: "${extension.address}", + fromTimestamp: "${fromTimestamp}", + toTimestamp: "${toTimestamp}" + } + ) { + blockInfo { + height + timestamp + } + eventData { + data + } + } + } + `, + }), + }) + .then(res => res.json()) + .then(res => { + return res; + }) + .then(json => json['data']['events'])) as { + blockInfo: { height: number; timestamp: string }; + eventData: { data: string[] }[]; + }[]; + + return data.flatMap(singleBlockData => + singleBlockData.eventData.flatMap(perTx => + perTx.data.map(txEvent => ({ + cdeId: extension.cdeId, + cdeDatumType: ChainDataExtensionDatumType.MinaGeneric, + blockNumber: getBlockNumber(Number.parseInt(singleBlockData.blockInfo.timestamp, 10)), + payload: txEvent, + network, + scheduledPrefix: extension.scheduledPrefix, + paginationCursor: { cursor: singleBlockData.blockInfo.timestamp, finished: false }, + })) + ) + ); +} diff --git a/packages/engine/paima-funnel/src/cde/reading.ts b/packages/engine/paima-funnel/src/cde/reading.ts index 0f53e4b1a..f114a7094 100644 --- a/packages/engine/paima-funnel/src/cde/reading.ts +++ b/packages/engine/paima-funnel/src/cde/reading.ts @@ -66,6 +66,10 @@ async function getSpecificCdeData( // this is used by the block funnel, which can't get information for this // extension return []; + case ChainDataExtensionType.MinaGeneric: + // this is used by the block funnel, which can't get information for this + // extension + return []; default: assertNever(extension); } diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts index 161ebdf22..78a423832 100644 --- a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts +++ b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts @@ -13,6 +13,7 @@ export type CacheMapType = { [RpcCacheEntry.SYMBOL]?: RpcCacheEntry; [CarpFunnelCacheEntry.SYMBOL]?: CarpFunnelCacheEntry; [EvmFunnelCacheEntry.SYMBOL]?: EvmFunnelCacheEntry; + [MinaFunnelCacheEntry.SYMBOL]?: MinaFunnelCacheEntry; }; export class FunnelCacheManager { public cacheEntries: CacheMapType = {}; @@ -178,3 +179,59 @@ export class EvmFunnelCacheEntry implements FunnelCacheEntry { this.cachedData = {}; }; } + +export type MinaFunnelCacheEntryState = { + startingSlotTimestamp: number; + lastPoint: { timestamp: number } | undefined; + genesisTime: number; + cursors: + | { + [cdeId: number]: { cursor: string; finished: boolean }; + } + | undefined; +}; + +export class MinaFunnelCacheEntry implements FunnelCacheEntry { + private state: MinaFunnelCacheEntryState | null = null; + public static readonly SYMBOL = Symbol('MinaFunnelStartingSlot'); + + public updateStartingSlot(startingSlotTimestamp: number, genesisTime: number): void { + this.state = { + startingSlotTimestamp, + genesisTime, + lastPoint: this.state?.lastPoint, + cursors: this.state?.cursors, + }; + } + + public updateLastPoint(timestamp: number): void { + if (this.state) { + this.state.lastPoint = { timestamp }; + } + } + + public updateCursor(cdeId: number, presyncCursor: { cursor: string; finished: boolean }): void { + if (this.state) { + if (!this.state.cursors) { + this.state.cursors = {}; + } + + this.state.cursors[cdeId] = presyncCursor; + } + } + + public initialized(): boolean { + return !!this.state; + } + + public getState(): Readonly { + if (!this.state) { + throw new Error('[mina-funnel] Uninitialized cache entry'); + } + return this.state; + } + + clear: FunnelCacheEntry['clear'] = () => { + this.state = null; + }; +} diff --git a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts index 1605a7b50..74239cb93 100644 --- a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts @@ -458,11 +458,17 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { const cursors = await getCarpCursors.run(undefined, dbTx); + const extensions = sharedData.extensions + .filter(extensions => (extensions.network = config.network)) + .map(extension => extension.cdeId); + for (const cursor of cursors) { - newEntry.updateCursor(cursor.cde_id, { - cursor: cursor.cursor, - finished: cursor.finished, - }); + // TODO: performance concern? but not likely + if (extensions.find(cdeId => cdeId === cursor.cde_id)) + newEntry.updateCursor(cursor.cde_id, { + cursor: cursor.cursor, + finished: cursor.finished, + }); } return newEntry; diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts new file mode 100644 index 000000000..0d6e8d32a --- /dev/null +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -0,0 +1,399 @@ +import type { EvmConfig, Web3 } from '@paima/utils'; +import { + doLog, + initWeb3, + logError, + timeout, + delay, + InternalEventType, + ChainDataExtensionType, +} from '@paima/utils'; +import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime'; +import type { + ChainData, + ChainDataExtension, + ChainDataExtensionDatum, + MinaPresyncChainData, + PresyncChainData, +} from '@paima/sm'; +import { getUngroupedCdeData } from '../../cde/reading.js'; +import { composeChainData, groupCdeData } from '../../utils.js'; +import { BaseFunnel } from '../BaseFunnel.js'; +import type { FunnelSharedData } from '../BaseFunnel.js'; +import type { PoolClient } from 'pg'; +import { FUNNEL_PRESYNC_FINISHED, ConfigNetworkType } from '@paima/utils'; +import { getCarpCursors, getLatestProcessedCdeBlockheight } from '@paima/db'; +import getMinaGenericCdeData from '../../cde/minaGeneric.js'; +import { MinaConfig } from '@paima/utils/src/config/loading.js'; +import { MinaFunnelCacheEntry } from '../FunnelCache.js'; +import { timeStamp } from 'console'; + +async function getGenesisTime(graphql: string): Promise { + const genesisTime = (await fetch(graphql, { + method: 'POST', + + headers: { + 'Content-Type': 'application/json', + }, + + body: JSON.stringify({ + query: ` + { + genesisBlock { + protocolState { + blockchainState { + utcDate + } + } + } + } + `, + }), + }) + .then(res => res.json()) + .then( + res => res['data']['genesisBlock']['protocolState']['blockchainState']['utcDate'] + )) as string; + + return Number.parseInt(genesisTime, 10); +} + +const SLOT_DURATION = 3 * 60000; + +function slotToTimestamp(slot: number, genesisTime: number): number { + console.log('slot', slot, genesisTime); + return slot * SLOT_DURATION + genesisTime; +} + +function timestampToSlot(ts: number, genesisTime: number): number { + return Math.max(Math.floor((ts - genesisTime) / SLOT_DURATION), 0); +} + +async function findMinaConfirmedSlot(graphql: string, confirmationDepth: number): Promise { + const body = JSON.stringify({ + query: ` + { + bestChain(maxLength: ${confirmationDepth}) { + stateHash + protocolState { + consensusState { + blockHeight + slotSinceGenesis + } + previousStateHash + } + } + } + `, + }); + + const confirmedSlot = await fetch(graphql, { + method: 'POST', + + headers: { + 'Content-Type': 'application/json', + }, + + body, + }) + .then(res => res.json()) + .then( + res => res['data']['bestChain'][0]['protocolState']['consensusState']['slotSinceGenesis'] + ); + + return Number.parseInt(confirmedSlot, 10); +} + +export class MinaFunnel extends BaseFunnel implements ChainFunnel { + config: MinaConfig; + chainName: string; + + protected constructor( + sharedData: FunnelSharedData, + dbTx: PoolClient, + config: MinaConfig, + chainName: string, + private readonly baseFunnel: ChainFunnel, + private cache: MinaFunnelCacheEntry + ) { + super(sharedData, dbTx); + this.readData.bind(this); + this.readPresyncData.bind(this); + this.getDbTx.bind(this); + this.config = config; + this.chainName = chainName; + } + + public override async readData(blockHeight: number): Promise { + const baseData = await this.baseFunnel.readData(blockHeight); + + if (baseData.length === 0) { + return baseData; + } + + let cachedState = this.cache.getState(); + + const confirmedSlot = await findMinaConfirmedSlot( + this.config.graphql, + this.config.confirmationDepth + ); + console.log('confirmedSlot', confirmedSlot); + const confirmedTimestamp = slotToTimestamp(confirmedSlot, cachedState.genesisTime); + + const fromTimestamp = + this.cache.getState().lastPoint?.timestamp || cachedState.startingSlotTimestamp; + + console.log('baseData', baseData); + + const toTimestamp = Math.max( + confirmedTimestamp, + baseData[baseData.length - 1].timestamp - 3 * 60000 * this.config.confirmationDepth + ); + + const fromSlot = timestampToSlot(fromTimestamp, cachedState.genesisTime); + const toSlot = timestampToSlot(toTimestamp, cachedState.genesisTime); + + console.log('from slot to slot', fromSlot, toSlot); + + const mapSlotsToEvmNumbers: { [slot: number]: number } = {}; + + let curr = 0; + + for (let slot = fromSlot; slot <= toSlot; slot++) { + while ( + curr < baseData.length && + timestampToSlot(baseData[curr].timestamp * 1000, cachedState.genesisTime) < slot + ) + curr++; + + if (curr < baseData.length) { + mapSlotsToEvmNumbers[slot] = baseData[curr].blockNumber; + } + } + + console.log('made mapping'); + + const ungroupedCdeData = await Promise.all( + this.sharedData.extensions.reduce( + (promises, extension) => { + if (extension.cdeType === ChainDataExtensionType.MinaGeneric) { + const promise = getMinaGenericCdeData( + this.config.archive, + extension, + fromTimestamp, + toTimestamp, + ts => mapSlotsToEvmNumbers[timestampToSlot(ts, cachedState.genesisTime)], + this.chainName + ); + + promises.push(promise); + } + + return promises; + }, + [] as Promise[] + ) + ); + + console.log('grouping cde data'); + + const grouped = groupCdeData( + this.chainName, + baseData[0].blockNumber, + baseData[baseData.length - 1].blockNumber, + ungroupedCdeData.filter(data => data.length > 0) + ); + + console.log('composing cde data'); + + const composed = composeChainData(baseData, grouped); + + return composed; + } + + public override async readPresyncData( + args: ReadPresyncDataFrom + ): Promise<{ [network: string]: PresyncChainData[] | typeof FUNNEL_PRESYNC_FINISHED }> { + const basePromise = this.baseFunnel.readPresyncData(args); + + const cache = this.cache.getState(); + + const cursors = cache.cursors; + + console.log('cursors', cursors); + + if (cursors && Object.values(cursors).every(x => x.finished)) { + const data = await basePromise; + data[this.chainName] = FUNNEL_PRESYNC_FINISHED; + + return data; + } + + const startingSlotTimestamp = this.cache.getState().startingSlotTimestamp; + + try { + // doLog(`Mina funnel presync ${this.chainName}: #${fromTimestamp}-${toTimestamp}`); + + const [baseData, ungroupedCdeData] = await Promise.all([ + basePromise, + Promise.all( + this.sharedData.extensions + .filter(extension => { + if (extension.cdeType !== ChainDataExtensionType.MinaGeneric) { + return false; + } + + if (cursors) { + const cursor = cursors[extension.cdeId]; + + if (!cursor) { + return true; + } + + return !cursor.finished; + } else { + return true; + } + }) + .map(extension => { + switch (extension.cdeType) { + case ChainDataExtensionType.MinaGeneric: + console.log('startSlot', extension.startSlot); + let cursor = + (cursors && Number.parseInt(cursors[extension.cdeId].cursor, 10)) || + slotToTimestamp(extension.startSlot, cache.genesisTime); + + console.log('cursors', cursor); + + let to = cursor + 10 * 60000; + + const data = getMinaGenericCdeData( + this.config.archive, + extension, + cursor, + Math.min(to - 1, startingSlotTimestamp - 1), + x => timestampToSlot(x, cache.genesisTime), + this.chainName + ).then(data => { + this.cache.updateCursor(extension.cdeId, { + cursor: to.toString(), + finished: to >= startingSlotTimestamp, + }); + + // if (data.length > 0) { + // data[data.length - 1].paginationCursor.finished = finished; + // } + + return data; + }); + + return data.then(data => ({ + cdeId: extension.cdeId, + cdeType: extension.cdeType, + data, + })); + default: + throw new Error(`[mina funnel] unhandled extension: ${extension.cdeType}`); + } + }) + ), + ]); + + const list: MinaPresyncChainData[] = []; + + for (const events of ungroupedCdeData) { + for (const event of events.data || []) { + list.push({ + extensionDatums: [event], + network: this.chainName, + networkType: ConfigNetworkType.MINA, + minaCursor: { + cdeId: event.cdeId, + cursor: event.paginationCursor.cursor, + finished: event.paginationCursor.finished, + }, + }); + } + } + + baseData[this.chainName] = list; + + return baseData; + } catch (err) { + doLog(`[paima-funnel::readPresyncData] Exception occurred while reading blocks: ${err}`); + + process.exit(1); + throw err; + } + } + + public static async recoverState( + sharedData: FunnelSharedData, + dbTx: PoolClient, + baseFunnel: ChainFunnel, + chainName: string, + config: MinaConfig, + startingBlockHeight: number + ): Promise { + const cacheEntry = (async (): Promise => { + const entry = sharedData.cacheManager.cacheEntries[MinaFunnelCacheEntry.SYMBOL]; + if (entry != null && entry.initialized()) return entry; + + const newEntry = new MinaFunnelCacheEntry(); + sharedData.cacheManager.cacheEntries[MinaFunnelCacheEntry.SYMBOL] = newEntry; + + const genesisTime = await getGenesisTime(config.graphql); + const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) + .timestamp as number; + + newEntry.updateStartingSlot( + slotToTimestamp(timestampToSlot(startingBlockTimestamp * 1000, genesisTime), genesisTime), + genesisTime + ); + + const cursors = await getCarpCursors.run(undefined, dbTx); + + const extensions = sharedData.extensions + .filter(extensions => (extensions.network = chainName)) + .map(extension => extension.cdeId); + + for (const cursor of cursors) { + // TODO: performance concern? but not likely + if (extensions.find(cdeId => cdeId === cursor.cde_id)) + newEntry.updateCursor(cursor.cde_id, { + cursor: cursor.cursor, + finished: cursor.finished, + }); + } + + return newEntry; + })(); + + return new MinaFunnel(sharedData, dbTx, config, chainName, baseFunnel, await cacheEntry); + } +} + +export async function wrapToMinaFunnel( + chainFunnel: ChainFunnel, + sharedData: FunnelSharedData, + dbTx: PoolClient, + startingBlockHeight: number, + chainName: string, + config: MinaConfig +): Promise { + try { + const ebp = await MinaFunnel.recoverState( + sharedData, + dbTx, + chainFunnel, + chainName, + config, + startingBlockHeight + ); + return ebp; + } catch (err) { + doLog('[paima-funnel] Unable to initialize mina cde events processor:'); + logError(err); + throw new Error('[paima-funnel] Unable to initialize mina cde events processor'); + } +} diff --git a/packages/engine/paima-funnel/src/index.ts b/packages/engine/paima-funnel/src/index.ts index dda1527d5..347a278ee 100644 --- a/packages/engine/paima-funnel/src/index.ts +++ b/packages/engine/paima-funnel/src/index.ts @@ -18,6 +18,7 @@ import { wrapToCarpFunnel } from './funnels/carp/funnel.js'; import { wrapToParallelEvmFunnel } from './funnels/parallelEvm/funnel.js'; import { ConfigNetworkType } from '@paima/utils'; import type Web3 from 'web3'; +import { wrapToMinaFunnel } from './funnels/mina/funnel.js'; export class FunnelFactory implements IFunnelFactory { private constructor(public sharedData: FunnelSharedData) {} @@ -93,6 +94,16 @@ export class FunnelFactory implements IFunnelFactory { ); } chainFunnel = await wrapToCarpFunnel(chainFunnel, this.sharedData, dbTx, ENV.START_BLOCKHEIGHT); + for (const [chainName, config] of await GlobalConfig.minaConfig()) { + chainFunnel = await wrapToMinaFunnel( + chainFunnel, + this.sharedData, + dbTx, + ENV.START_BLOCKHEIGHT, + chainName, + config + ); + } chainFunnel = await wrapToEmulatedBlocksFunnel( chainFunnel, this.sharedData, diff --git a/packages/engine/paima-runtime/src/cde-config/loading.ts b/packages/engine/paima-runtime/src/cde-config/loading.ts index e92d2b721..4f875abdb 100644 --- a/packages/engine/paima-runtime/src/cde-config/loading.ts +++ b/packages/engine/paima-runtime/src/cde-config/loading.ts @@ -43,6 +43,7 @@ import { ChainDataExtensionErc6551RegistryConfig, ChainDataExtensionErc721Config, ChainDataExtensionGenericConfig, + ChainDataExtensionMinaGenericConfig, } from '@paima/sm'; import { loadAbi } from './utils.js'; import assertNever from 'assert-never'; @@ -162,6 +163,12 @@ export function parseCdeConfigFile(configFileData: string): Static { const cdeId = cdeDatum.cdeId; diff --git a/packages/engine/paima-sm/src/cde-processing.ts b/packages/engine/paima-sm/src/cde-processing.ts index dcce614ae..bdb814d40 100644 --- a/packages/engine/paima-sm/src/cde-processing.ts +++ b/packages/engine/paima-sm/src/cde-processing.ts @@ -48,6 +48,8 @@ export async function cdeTransitionFunction( return await processCardanoTransferDatum(cdeDatum, inPresync); case ChainDataExtensionDatumType.CardanoMintBurn: return await processCardanoMintBurnDatum(cdeDatum, inPresync); + case ChainDataExtensionDatumType.MinaGeneric: + return await processGenericDatum(cdeDatum, inPresync); default: assertNever(cdeDatum); } diff --git a/packages/engine/paima-sm/src/index.ts b/packages/engine/paima-sm/src/index.ts index 0a6ec78f7..ae50db4b1 100644 --- a/packages/engine/paima-sm/src/index.ts +++ b/packages/engine/paima-sm/src/index.ts @@ -130,6 +130,19 @@ const SM: GameStateMachineInitializer = { `[${latestCdeData.network}] Processed ${cdeDataLength} CDE events in ${latestCdeData.carpCursor.cursor}` ); } + } else if (latestCdeData.networkType === ConfigNetworkType.MINA) { + // not really correct, but should work for now. Probably that function should be renamed + const cdeDataLength = await processCardanoCdeData( + latestCdeData.minaCursor, + latestCdeData.extensionDatums, + dbTx, + true + ); + if (cdeDataLength > 0) { + doLog( + `[${latestCdeData.network}] Processed ${cdeDataLength} CDE events in ${latestCdeData.minaCursor.cursor}` + ); + } } }, markPresyncMilestone: async ( @@ -338,6 +351,38 @@ async function processCardanoCdeData( }); } +async function processMinaCdeData( + paginationCursor: + | { cdeId: number; cursor: string; finished: boolean } + | { cdeId: number; slot: number; finished: boolean }, + cdeData: ChainDataExtensionDatum[] | undefined, + dbTx: PoolClient, + inPresync: boolean +): Promise { + return await processCdeDataBase(cdeData, dbTx, inPresync, async () => { + if ('cursor' in paginationCursor) { + await updateCardanoPaginationCursor.run( + { + cde_id: paginationCursor.cdeId, + cursor: paginationCursor.cursor, + finished: paginationCursor.finished, + }, + dbTx + ); + } else { + await updateCardanoPaginationCursor.run( + { + cde_id: paginationCursor.cdeId, + cursor: paginationCursor.slot.toString(), + finished: paginationCursor.finished, + }, + dbTx + ); + } + return; + }); +} + // Process all of the scheduled data inputs by running each of them through the game STF, // saving the results to the DB, and deleting the schedule data all together in one postgres tx. // Function returns number of scheduled inputs that were processed. diff --git a/packages/engine/paima-sm/src/types.ts b/packages/engine/paima-sm/src/types.ts index 7ba427681..cb8c40d8b 100644 --- a/packages/engine/paima-sm/src/types.ts +++ b/packages/engine/paima-sm/src/types.ts @@ -57,7 +57,15 @@ export interface CardanoPresyncChainData { extensionDatums: ChainDataExtensionDatum[]; } -export type PresyncChainData = EvmPresyncChainData | CardanoPresyncChainData; +// TODO: potentially this and Cardano can be collapsed +export interface MinaPresyncChainData { + network: string; + networkType: ConfigNetworkType.MINA; + minaCursor: { cdeId: number; cursor: string; finished: boolean }; + extensionDatums: ChainDataExtensionDatum[]; +} + +export type PresyncChainData = EvmPresyncChainData | CardanoPresyncChainData | MinaPresyncChainData; interface CdeDatumErc20TransferPayload { from: string; @@ -247,6 +255,12 @@ export interface CdeCardanoMintBurnDatum extends CdeDatumBase { paginationCursor: { cursor: string; finished: boolean }; } +export interface CdeMinaGenericDatum extends CdeDatumBase { + cdeDatumType: ChainDataExtensionDatumType.MinaGeneric; + paginationCursor: { cursor: string; finished: boolean }; + scheduledPrefix: string; +} + export type ChainDataExtensionDatum = | CdeErc20TransferDatum | CdeErc721MintDatum @@ -259,7 +273,8 @@ export type ChainDataExtensionDatum = | CdeCardanoProjectedNFTDatum | CdeCardanoAssetUtxoDatum | CdeCardanoTransferDatum - | CdeCardanoMintBurnDatum; + | CdeCardanoMintBurnDatum + | CdeMinaGenericDatum; export enum CdeEntryTypeName { Generic = 'generic', @@ -273,6 +288,7 @@ export enum CdeEntryTypeName { CardanoDelayedAsset = 'cardano-delayed-asset', CardanoTransfer = 'cardano-transfer', CardanoMintBurn = 'cardano-mint-burn', + MinaGeneric = 'mina-generic', } const EvmAddress = Type.Transform(Type.RegExp('0x[0-9a-fA-F]{40}')) @@ -462,6 +478,20 @@ export type ChainDataExtensionCardanoMintBurn = ChainDataExtensionBase & cdeType: ChainDataExtensionType.CardanoMintBurn; }; +export const ChainDataExtensionMinaGenericConfig = Type.Object({ + type: Type.Literal(CdeEntryTypeName.MinaGeneric), + // TODO: change? + address: Type.String(), + scheduledPrefix: Type.String(), + startSlot: Type.Number(), +}); + +export type TChainDataExtensionMinaGenericConfig = Static; +export type ChainDataExtensionMinaGeneric = ChainDataExtensionBase & + Static & { + cdeType: ChainDataExtensionType.MinaGeneric; + }; + export const CdeConfig = Type.Object({ extensions: Type.Array( Type.Intersect([ @@ -477,6 +507,7 @@ export const CdeConfig = Type.Object({ ChainDataExtensionCardanoDelayedAssetConfig, ChainDataExtensionCardanoTransferConfig, ChainDataExtensionCardanoMintBurnConfig, + ChainDataExtensionMinaGenericConfig, ]), Type.Partial(Type.Object({ network: Type.String() })), ]) @@ -509,6 +540,7 @@ export type ChainDataExtension = ( | ChainDataExtensionCardanoDelayedAsset | ChainDataExtensionCardanoTransfer | ChainDataExtensionCardanoMintBurn + | ChainDataExtensionMinaGeneric ) & { network: string | undefined }; export type GameStateTransitionFunctionRouter = ( diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index 34bd8cb0e..95cce721a 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -9,6 +9,7 @@ export enum ConfigNetworkType { EVM = 'evm-main', EVM_OTHER = 'evm-other', CARDANO = 'cardano', + MINA = 'mina', } export type EvmConfig = Static; @@ -60,6 +61,15 @@ export const CardanoConfigSchema = Type.Object({ export type CardanoConfig = Static; +export const MinaConfigSchema = Type.Object({ + archive: Type.String(), + graphql: Type.String(), + confirmationDepth: Type.Number(), + //presyncStepSize: Type.Number({ default: 1000 }), +}); + +export type MinaConfig = Static; + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const TaggedEvmConfig = (T: T, MAIN_CONFIG: U) => Type.Union([ @@ -88,9 +98,21 @@ export const TaggedCardanoConfig = (T: T) => Type.Object({ type: Type.Literal(ConfigNetworkType.CARDANO) }), ]); +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export const TaggedMinaConfig = (T: T) => + Type.Intersect([ + T ? MinaConfigSchema : Type.Partial(MinaConfigSchema), + Type.Object({ type: Type.Literal(ConfigNetworkType.MINA) }), + ]); + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const TaggedConfig = (T: T) => - Type.Union([TaggedEvmMainConfig(T), TaggedEvmOtherConfig(T), TaggedCardanoConfig(T)]); + Type.Union([ + TaggedEvmMainConfig(T), + TaggedEvmOtherConfig(T), + TaggedCardanoConfig(T), + TaggedMinaConfig(T), + ]); // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export const BaseConfig = (T: T) => Type.Record(Type.String(), TaggedConfig(T)); @@ -109,6 +131,10 @@ const cardanoConfigDefaults = { paginationLimits: 50, }; +const minaConfigDefaults = { + confirmationDepth: 15, +}; + // used as a placeholder name for the ENV fallback mechanism // will need to be removed afterwards export const defaultEvmMainNetworkName = 'evm'; @@ -170,6 +196,9 @@ export async function loadConfig(): Promise; @@ -58,4 +64,12 @@ export class GlobalConfig { .filter(key => instance[key].type === ConfigNetworkType.EVM_OTHER) .map(key => [key, instance[key] as EvmConfig]); } + + public static async minaConfig(): Promise<[string, MinaConfig][]> { + const instance = await GlobalConfig.getInstance(); + + return Object.keys(instance) + .filter(key => instance[key].type === ConfigNetworkType.MINA) + .map(key => [key, instance[key] as MinaConfig]); + } } diff --git a/packages/paima-sdk/paima-utils/src/constants.ts b/packages/paima-sdk/paima-utils/src/constants.ts index 523b6cf7e..d286ecc8c 100644 --- a/packages/paima-sdk/paima-utils/src/constants.ts +++ b/packages/paima-sdk/paima-utils/src/constants.ts @@ -24,6 +24,7 @@ export const enum ChainDataExtensionType { CardanoTransfer = 10, CardanoMintBurn = 11, ERC1155 = 12, + MinaGeneric = 13, } export const enum ChainDataExtensionDatumType { @@ -39,6 +40,7 @@ export const enum ChainDataExtensionDatumType { CardanoTransfer, CardanoMintBurn, Erc1155Transfer, + MinaGeneric, } export const FUNNEL_PRESYNC_FINISHED = 'finished'; From 380eccc8a2f1b4c9016b755cdb4cd8dd8e1e6c81 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Mon, 1 Apr 2024 13:50:32 -0300 Subject: [PATCH 02/31] wip --- packages/engine/paima-funnel/src/funnels/mina/funnel.ts | 2 -- packages/engine/paima-sm/src/types.ts | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 0d6e8d32a..3aa07c0f3 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -11,12 +11,10 @@ import { import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime'; import type { ChainData, - ChainDataExtension, ChainDataExtensionDatum, MinaPresyncChainData, PresyncChainData, } from '@paima/sm'; -import { getUngroupedCdeData } from '../../cde/reading.js'; import { composeChainData, groupCdeData } from '../../utils.js'; import { BaseFunnel } from '../BaseFunnel.js'; import type { FunnelSharedData } from '../BaseFunnel.js'; diff --git a/packages/engine/paima-sm/src/types.ts b/packages/engine/paima-sm/src/types.ts index cb8c40d8b..833661949 100644 --- a/packages/engine/paima-sm/src/types.ts +++ b/packages/engine/paima-sm/src/types.ts @@ -484,6 +484,7 @@ export const ChainDataExtensionMinaGenericConfig = Type.Object({ address: Type.String(), scheduledPrefix: Type.String(), startSlot: Type.Number(), + name: Type.String(), }); export type TChainDataExtensionMinaGenericConfig = Static; From c6110ee22eca3de857d689e9f3cb7e5c3ba55e26 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Wed, 3 Apr 2024 23:17:57 -0300 Subject: [PATCH 03/31] add actions fetching --- .../paima-funnel/src/cde/minaGeneric.ts | 70 +++++++++++++------ .../paima-funnel/src/funnels/mina/funnel.ts | 26 +++---- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 30abcad55..56b5ab800 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -1,8 +1,4 @@ -import { - CdeMinaGenericDatum, - ChainDataExtensionDatum, - ChainDataExtensionMinaGeneric, -} from '@paima/sm'; +import { CdeMinaGenericDatum, ChainDataExtensionMinaGeneric } from '@paima/sm'; import { ChainDataExtensionDatumType } from '@paima/utils'; export default async function getCdeData( @@ -14,7 +10,7 @@ export default async function getCdeData( network: string ): Promise { console.log('from-to', fromTimestamp, toTimestamp); - const data = (await fetch(minaArchive, { + const data = await fetch(minaArchive, { method: 'POST', headers: { @@ -39,30 +35,60 @@ export default async function getCdeData( data } } + actions( + input: { + address: "${extension.address}", + fromTimestamp: "${fromTimestamp}", + toTimestamp: "${toTimestamp}" + } + ) { + blockInfo { + height + timestamp + } + actionData { + data + } + } } `, }), }) .then(res => res.json()) - .then(res => { - return res; - }) - .then(json => json['data']['events'])) as { + .then(json => [json['data']['events'], json['data']['actions']]); + + const events = data[0] as { blockInfo: { height: number; timestamp: string }; eventData: { data: string[] }[]; }[]; - return data.flatMap(singleBlockData => - singleBlockData.eventData.flatMap(perTx => - perTx.data.map(txEvent => ({ - cdeId: extension.cdeId, - cdeDatumType: ChainDataExtensionDatumType.MinaGeneric, - blockNumber: getBlockNumber(Number.parseInt(singleBlockData.blockInfo.timestamp, 10)), - payload: txEvent, - network, - scheduledPrefix: extension.scheduledPrefix, - paginationCursor: { cursor: singleBlockData.blockInfo.timestamp, finished: false }, - })) - ) + const actions = data[1] as { + blockInfo: { height: number; timestamp: string }; + actionData: { data: string[] }[]; + }[]; + + const eventsAndActions = [ + ...events.map(ev => ({ + blockInfo: ev.blockInfo, + data: ev.eventData.map(datum => Object.assign(datum, { kind: 'event' })), + })), + ...actions.map(act => ({ + blockInfo: act.blockInfo, + data: act.actionData.map(datum => Object.assign(datum, { kind: 'action' })), + })), + ]; + + eventsAndActions.sort((a, b) => a.blockInfo.height - b.blockInfo.height); + + return eventsAndActions.flatMap(perBlock => + perBlock.data.map(txEvent => ({ + cdeId: extension.cdeId, + cdeDatumType: ChainDataExtensionDatumType.MinaGeneric, + blockNumber: getBlockNumber(Number.parseInt(perBlock.blockInfo.timestamp, 10)), + payload: txEvent, + network, + scheduledPrefix: extension.scheduledPrefix, + paginationCursor: { cursor: perBlock.blockInfo.timestamp, finished: false }, + })) ); } diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 3aa07c0f3..8a7c0a117 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -20,11 +20,10 @@ import { BaseFunnel } from '../BaseFunnel.js'; import type { FunnelSharedData } from '../BaseFunnel.js'; import type { PoolClient } from 'pg'; import { FUNNEL_PRESYNC_FINISHED, ConfigNetworkType } from '@paima/utils'; -import { getCarpCursors, getLatestProcessedCdeBlockheight } from '@paima/db'; +import { getCarpCursors } from '@paima/db'; import getMinaGenericCdeData from '../../cde/minaGeneric.js'; import { MinaConfig } from '@paima/utils/src/config/loading.js'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; -import { timeStamp } from 'console'; async function getGenesisTime(graphql: string): Promise { const genesisTime = (await fetch(graphql, { @@ -67,6 +66,9 @@ function timestampToSlot(ts: number, genesisTime: number): number { return Math.max(Math.floor((ts - genesisTime) / SLOT_DURATION), 0); } +// TODO: maybe using the node's rpc here it's not really safe? if it's out of +// sync with the archive db we could end up skipping events +// it would be better to have an endpoint for this on the archive api async function findMinaConfirmedSlot(graphql: string, confirmationDepth: number): Promise { const body = JSON.stringify({ query: ` @@ -135,14 +137,12 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { this.config.graphql, this.config.confirmationDepth ); - console.log('confirmedSlot', confirmedSlot); + const confirmedTimestamp = slotToTimestamp(confirmedSlot, cachedState.genesisTime); const fromTimestamp = this.cache.getState().lastPoint?.timestamp || cachedState.startingSlotTimestamp; - console.log('baseData', baseData); - const toTimestamp = Math.max( confirmedTimestamp, baseData[baseData.length - 1].timestamp - 3 * 60000 * this.config.confirmationDepth @@ -151,8 +151,6 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const fromSlot = timestampToSlot(fromTimestamp, cachedState.genesisTime); const toSlot = timestampToSlot(toTimestamp, cachedState.genesisTime); - console.log('from slot to slot', fromSlot, toSlot); - const mapSlotsToEvmNumbers: { [slot: number]: number } = {}; let curr = 0; @@ -169,8 +167,6 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { } } - console.log('made mapping'); - const ungroupedCdeData = await Promise.all( this.sharedData.extensions.reduce( (promises, extension) => { @@ -193,8 +189,6 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { ) ); - console.log('grouping cde data'); - const grouped = groupCdeData( this.chainName, baseData[0].blockNumber, @@ -202,8 +196,6 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { ungroupedCdeData.filter(data => data.length > 0) ); - console.log('composing cde data'); - const composed = composeChainData(baseData, grouped); return composed; @@ -218,8 +210,6 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const cursors = cache.cursors; - console.log('cursors', cursors); - if (cursors && Object.values(cursors).every(x => x.finished)) { const data = await basePromise; data[this.chainName] = FUNNEL_PRESYNC_FINISHED; @@ -256,13 +246,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { .map(extension => { switch (extension.cdeType) { case ChainDataExtensionType.MinaGeneric: - console.log('startSlot', extension.startSlot); let cursor = (cursors && Number.parseInt(cursors[extension.cdeId].cursor, 10)) || slotToTimestamp(extension.startSlot, cache.genesisTime); - console.log('cursors', cursor); - let to = cursor + 10 * 60000; const data = getMinaGenericCdeData( @@ -320,6 +307,8 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { } catch (err) { doLog(`[paima-funnel::readPresyncData] Exception occurred while reading blocks: ${err}`); + // TODO: remove later, but it's useful for debugging, since otherwise we + // just get stuck in a loop with infinite errors sometimes. process.exit(1); throw err; } @@ -341,6 +330,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { sharedData.cacheManager.cacheEntries[MinaFunnelCacheEntry.SYMBOL] = newEntry; const genesisTime = await getGenesisTime(config.graphql); + const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) .timestamp as number; From 09d5ad20c596230a278d032e8e8c1f6aa7ca24a9 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Thu, 4 Apr 2024 02:53:36 -0300 Subject: [PATCH 04/31] cleanup some lints --- .../paima-funnel/src/cde/minaGeneric.ts | 5 ++--- .../paima-funnel/src/funnels/mina/funnel.ts | 22 ++++--------------- packages/paima-sdk/paima-utils/src/index.ts | 2 ++ 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 56b5ab800..2b94549c6 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -1,4 +1,4 @@ -import { CdeMinaGenericDatum, ChainDataExtensionMinaGeneric } from '@paima/sm'; +import type { CdeMinaGenericDatum, ChainDataExtensionMinaGeneric } from '@paima/sm'; import { ChainDataExtensionDatumType } from '@paima/utils'; export default async function getCdeData( @@ -9,7 +9,6 @@ export default async function getCdeData( getBlockNumber: (minaTimestamp: number) => number, network: string ): Promise { - console.log('from-to', fromTimestamp, toTimestamp); const data = await fetch(minaArchive, { method: 'POST', @@ -55,7 +54,7 @@ export default async function getCdeData( }), }) .then(res => res.json()) - .then(json => [json['data']['events'], json['data']['actions']]); + .then(json => [json.data.events, json.data.actions]); const events = data[0] as { blockInfo: { height: number; timestamp: string }; diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 8a7c0a117..9b1fa5d0d 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -1,13 +1,4 @@ -import type { EvmConfig, Web3 } from '@paima/utils'; -import { - doLog, - initWeb3, - logError, - timeout, - delay, - InternalEventType, - ChainDataExtensionType, -} from '@paima/utils'; +import { doLog, logError, ChainDataExtensionType } from '@paima/utils'; import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime'; import type { ChainData, @@ -22,7 +13,7 @@ import type { PoolClient } from 'pg'; import { FUNNEL_PRESYNC_FINISHED, ConfigNetworkType } from '@paima/utils'; import { getCarpCursors } from '@paima/db'; import getMinaGenericCdeData from '../../cde/minaGeneric.js'; -import { MinaConfig } from '@paima/utils/src/config/loading.js'; +import type { MinaConfig } from '@paima/utils'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; async function getGenesisTime(graphql: string): Promise { @@ -48,9 +39,7 @@ async function getGenesisTime(graphql: string): Promise { }), }) .then(res => res.json()) - .then( - res => res['data']['genesisBlock']['protocolState']['blockchainState']['utcDate'] - )) as string; + .then(res => res.data.genesisBlock.protocolState.blockchainState.utcDate)) as string; return Number.parseInt(genesisTime, 10); } @@ -58,7 +47,6 @@ async function getGenesisTime(graphql: string): Promise { const SLOT_DURATION = 3 * 60000; function slotToTimestamp(slot: number, genesisTime: number): number { - console.log('slot', slot, genesisTime); return slot * SLOT_DURATION + genesisTime; } @@ -97,9 +85,7 @@ async function findMinaConfirmedSlot(graphql: string, confirmationDepth: number) body, }) .then(res => res.json()) - .then( - res => res['data']['bestChain'][0]['protocolState']['consensusState']['slotSinceGenesis'] - ); + .then(res => res.data.bestChain[0].protocolState.consensusState.slotSinceGenesis); return Number.parseInt(confirmedSlot, 10); } diff --git a/packages/paima-sdk/paima-utils/src/index.ts b/packages/paima-sdk/paima-utils/src/index.ts index 8c3d1671c..8e2b3822d 100644 --- a/packages/paima-sdk/paima-utils/src/index.ts +++ b/packages/paima-sdk/paima-utils/src/index.ts @@ -10,6 +10,7 @@ import { GlobalConfig } from './config/singleton.js'; import { EvmConfig, CardanoConfig, + MinaConfig, ConfigNetworkType, defaultEvmMainNetworkName, defaultCardanoNetworkName, @@ -34,6 +35,7 @@ export { GlobalConfig, EvmConfig, CardanoConfig, + MinaConfig, ConfigNetworkType, defaultEvmMainNetworkName, defaultCardanoNetworkName, From f6e8914e6970d8b0a3b5d3465439dde2d813d46a Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 9 Apr 2024 12:50:29 -0300 Subject: [PATCH 05/31] split actions and events into 2 cdes --- .../paima-funnel/src/cde/minaGeneric.ts | 148 ++++++++++++++--- .../engine/paima-funnel/src/cde/reading.ts | 9 +- .../paima-funnel/src/funnels/mina/funnel.ts | 157 ++++++++++++------ .../paima-runtime/src/cde-config/loading.ts | 26 ++- packages/engine/paima-sm/src/cde-generic.ts | 8 +- .../engine/paima-sm/src/cde-processing.ts | 4 +- packages/engine/paima-sm/src/types.ts | 52 ++++-- .../paima-sdk/paima-utils/src/constants.ts | 6 +- 8 files changed, 301 insertions(+), 109 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 2b94549c6..366c67c86 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -1,14 +1,30 @@ -import type { CdeMinaGenericDatum, ChainDataExtensionMinaGeneric } from '@paima/sm'; +import type { + CdeMinaActionGenericDatum, + CdeMinaEventGenericDatum, + ChainDataExtensionMinaActionGeneric, + ChainDataExtensionMinaEventGeneric, +} from '@paima/sm'; import { ChainDataExtensionDatumType } from '@paima/utils'; -export default async function getCdeData( +export async function getEventCdeData( minaArchive: string, - extension: ChainDataExtensionMinaGeneric, + extension: ChainDataExtensionMinaEventGeneric, fromTimestamp: number, toTimestamp: number, getBlockNumber: (minaTimestamp: number) => number, network: string -): Promise { +): Promise { + console.log('from,to', fromTimestamp, toTimestamp); + + const grouped = [] as { + blockInfo: { + height: number; + timestamp: string; + }; + // TODO: could each data by just a tuple? + eventData: { data: string[][]; txHash: string }[]; + }[]; + const data = await fetch(minaArchive, { method: 'POST', @@ -32,8 +48,75 @@ export default async function getCdeData( } eventData { data + transactionInfo { + hash + } } } + } + `, + }), + }) + .then(res => res.json()) + .then(res => { + console.log('res', JSON.stringify(res, undefined, '\t')); + return res; + }) + .then(json => json.data.events); + + const events = data as { + blockInfo: { height: number; timestamp: string }; + eventData: { data: string[]; transactionInfo: { hash: string } }[]; + }[]; + + for (const block of events) { + const eventData = [] as { data: string[][]; txHash: string }[]; + + for (const blockEvent of block.eventData) { + if ( + eventData[eventData.length - 1] && + blockEvent.transactionInfo.hash == eventData[eventData.length - 1].txHash + ) { + eventData[eventData.length - 1].data.push(blockEvent.data); + } else { + eventData.push({ txHash: blockEvent.transactionInfo.hash, data: [blockEvent.data] }); + } + } + + grouped.push({ blockInfo: block.blockInfo, eventData }); + } + + return grouped.flatMap(perBlock => + perBlock.eventData.map(txEvent => ({ + cdeId: extension.cdeId, + cdeDatumType: ChainDataExtensionDatumType.MinaEventGeneric, + blockNumber: getBlockNumber(Number.parseInt(perBlock.blockInfo.timestamp, 10)), + payload: txEvent, + network, + scheduledPrefix: extension.scheduledPrefix, + paginationCursor: { cursor: txEvent.txHash, finished: false }, + })) + ); +} + +export async function getActionCdeData( + minaArchive: string, + extension: ChainDataExtensionMinaActionGeneric, + fromTimestamp: number, + toTimestamp: number, + getBlockNumber: (minaTimestamp: number) => number, + network: string +): Promise { + const data = await fetch(minaArchive, { + method: 'POST', + + headers: { + 'Content-Type': 'application/json', + }, + + body: JSON.stringify({ + query: ` + { actions( input: { address: "${extension.address}", @@ -47,6 +130,9 @@ export default async function getCdeData( } actionData { data + transactionInfo { + hash + } } } } @@ -54,40 +140,48 @@ export default async function getCdeData( }), }) .then(res => res.json()) - .then(json => [json.data.events, json.data.actions]); + .then(json => json.data.actions); - const events = data[0] as { + const actions = data as { blockInfo: { height: number; timestamp: string }; - eventData: { data: string[] }[]; + actionData: { data: string[]; hash: string }[]; }[]; - const actions = data[1] as { - blockInfo: { height: number; timestamp: string }; - actionData: { data: string[] }[]; + const grouped = [] as { + blockInfo: { + height: number; + timestamp: string; + }; + // TODO: could each data by just a tuple? + actionData: { data: string[][]; txHash: string }[]; }[]; - const eventsAndActions = [ - ...events.map(ev => ({ - blockInfo: ev.blockInfo, - data: ev.eventData.map(datum => Object.assign(datum, { kind: 'event' })), - })), - ...actions.map(act => ({ - blockInfo: act.blockInfo, - data: act.actionData.map(datum => Object.assign(datum, { kind: 'action' })), - })), - ]; - - eventsAndActions.sort((a, b) => a.blockInfo.height - b.blockInfo.height); - - return eventsAndActions.flatMap(perBlock => - perBlock.data.map(txEvent => ({ + for (const block of actions) { + const actionData = [] as { data: string[][]; txHash: string }[]; + + for (const blockEvent of block.actionData) { + if ( + actionData[actionData.length - 1] && + blockEvent.hash == actionData[actionData.length - 1].txHash + ) { + actionData[actionData.length - 1].data.push(blockEvent.data); + } else { + actionData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); + } + } + + grouped.push({ blockInfo: block.blockInfo, actionData }); + } + + return grouped.flatMap(perBlock => + perBlock.actionData.map(txEvent => ({ cdeId: extension.cdeId, - cdeDatumType: ChainDataExtensionDatumType.MinaGeneric, + cdeDatumType: ChainDataExtensionDatumType.MinaActionGeneric, blockNumber: getBlockNumber(Number.parseInt(perBlock.blockInfo.timestamp, 10)), payload: txEvent, network, scheduledPrefix: extension.scheduledPrefix, - paginationCursor: { cursor: perBlock.blockInfo.timestamp, finished: false }, + paginationCursor: { cursor: txEvent.txHash, finished: false }, })) ); } diff --git a/packages/engine/paima-funnel/src/cde/reading.ts b/packages/engine/paima-funnel/src/cde/reading.ts index f114a7094..7cc43a4ab 100644 --- a/packages/engine/paima-funnel/src/cde/reading.ts +++ b/packages/engine/paima-funnel/src/cde/reading.ts @@ -58,17 +58,14 @@ async function getSpecificCdeData( return await getCdeErc1155Data(extension, fromBlock, toBlock, network); case ChainDataExtensionType.ERC6551Registry: return await getCdeErc6551RegistryData(extension, fromBlock, toBlock, network); + // these are not used by the block funnel case ChainDataExtensionType.CardanoPool: case ChainDataExtensionType.CardanoProjectedNFT: case ChainDataExtensionType.CardanoAssetUtxo: case ChainDataExtensionType.CardanoTransfer: case ChainDataExtensionType.CardanoMintBurn: - // this is used by the block funnel, which can't get information for this - // extension - return []; - case ChainDataExtensionType.MinaGeneric: - // this is used by the block funnel, which can't get information for this - // extension + case ChainDataExtensionType.MinaEventGeneric: + case ChainDataExtensionType.MinaActionGeneric: return []; default: assertNever(extension); diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 9b1fa5d0d..73a890473 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -12,7 +12,7 @@ import type { FunnelSharedData } from '../BaseFunnel.js'; import type { PoolClient } from 'pg'; import { FUNNEL_PRESYNC_FINISHED, ConfigNetworkType } from '@paima/utils'; import { getCarpCursors } from '@paima/db'; -import getMinaGenericCdeData from '../../cde/minaGeneric.js'; +import { getActionCdeData, getEventCdeData } from '../../cde/minaGeneric.js'; import type { MinaConfig } from '@paima/utils'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; @@ -46,14 +46,18 @@ async function getGenesisTime(graphql: string): Promise { const SLOT_DURATION = 3 * 60000; -function slotToTimestamp(slot: number, genesisTime: number): number { +function slotToMinaTimestamp(slot: number, genesisTime: number): number { return slot * SLOT_DURATION + genesisTime; } -function timestampToSlot(ts: number, genesisTime: number): number { +function minaTimestampToSlot(ts: number, genesisTime: number): number { return Math.max(Math.floor((ts - genesisTime) / SLOT_DURATION), 0); } +function baseChainTimestampToMina(baseChainTimestamp: number, confirmationDepth: number): number { + return Math.max(baseChainTimestamp * 1000 - SLOT_DURATION * confirmationDepth, 0); +} + // TODO: maybe using the node's rpc here it's not really safe? if it's out of // sync with the archive db we could end up skipping events // it would be better to have an endpoint for this on the archive api @@ -124,18 +128,21 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { this.config.confirmationDepth ); - const confirmedTimestamp = slotToTimestamp(confirmedSlot, cachedState.genesisTime); + const confirmedTimestamp = slotToMinaTimestamp(confirmedSlot, cachedState.genesisTime); const fromTimestamp = this.cache.getState().lastPoint?.timestamp || cachedState.startingSlotTimestamp; const toTimestamp = Math.max( confirmedTimestamp, - baseData[baseData.length - 1].timestamp - 3 * 60000 * this.config.confirmationDepth + baseChainTimestampToMina( + baseData[baseData.length - 1].timestamp, + this.config.confirmationDepth + ) ); - const fromSlot = timestampToSlot(fromTimestamp, cachedState.genesisTime); - const toSlot = timestampToSlot(toTimestamp, cachedState.genesisTime); + const fromSlot = minaTimestampToSlot(fromTimestamp, cachedState.genesisTime); + const toSlot = minaTimestampToSlot(toTimestamp, cachedState.genesisTime); const mapSlotsToEvmNumbers: { [slot: number]: number } = {}; @@ -144,7 +151,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { for (let slot = fromSlot; slot <= toSlot; slot++) { while ( curr < baseData.length && - timestampToSlot(baseData[curr].timestamp * 1000, cachedState.genesisTime) < slot + minaTimestampToSlot( + baseChainTimestampToMina(baseData[curr].timestamp, this.config.confirmationDepth), + cachedState.genesisTime + ) < slot ) curr++; @@ -156,13 +166,26 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const ungroupedCdeData = await Promise.all( this.sharedData.extensions.reduce( (promises, extension) => { - if (extension.cdeType === ChainDataExtensionType.MinaGeneric) { - const promise = getMinaGenericCdeData( + if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { + const promise = getEventCdeData( + this.config.archive, + extension, + fromTimestamp, + toTimestamp, + ts => mapSlotsToEvmNumbers[minaTimestampToSlot(ts, cachedState.genesisTime)], + this.chainName + ); + + promises.push(promise); + } + + if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { + const promise = getActionCdeData( this.config.archive, extension, fromTimestamp, toTimestamp, - ts => mapSlotsToEvmNumbers[timestampToSlot(ts, cachedState.genesisTime)], + ts => mapSlotsToEvmNumbers[minaTimestampToSlot(ts, cachedState.genesisTime)], this.chainName ); @@ -203,17 +226,30 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { return data; } + const mapCursorPaginatedData = (cdeId: number) => (datums: any) => { + const finished = datums.length === 0 || datums.length < this.config.paginationLimit; + + this.cache.updateCursor(cdeId, { + cursor: datums[datums.length - 1] ? datums[datums.length - 1].paginationCursor.cursor : '', + finished, + }); + + if (datums.length > 0) { + datums[datums.length - 1].paginationCursor.finished = finished; + } + + return datums; + }; + const startingSlotTimestamp = this.cache.getState().startingSlotTimestamp; try { - // doLog(`Mina funnel presync ${this.chainName}: #${fromTimestamp}-${toTimestamp}`); - const [baseData, ungroupedCdeData] = await Promise.all([ basePromise, Promise.all( this.sharedData.extensions .filter(extension => { - if (extension.cdeType !== ChainDataExtensionType.MinaGeneric) { + if (extension.network !== this.chainName) { return false; } @@ -230,41 +266,46 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { } }) .map(extension => { - switch (extension.cdeType) { - case ChainDataExtensionType.MinaGeneric: - let cursor = - (cursors && Number.parseInt(cursors[extension.cdeId].cursor, 10)) || - slotToTimestamp(extension.startSlot, cache.genesisTime); - - let to = cursor + 10 * 60000; - - const data = getMinaGenericCdeData( - this.config.archive, - extension, - cursor, - Math.min(to - 1, startingSlotTimestamp - 1), - x => timestampToSlot(x, cache.genesisTime), - this.chainName - ).then(data => { - this.cache.updateCursor(extension.cdeId, { - cursor: to.toString(), - finished: to >= startingSlotTimestamp, - }); - - // if (data.length > 0) { - // data[data.length - 1].paginationCursor.finished = finished; - // } - - return data; - }); - - return data.then(data => ({ - cdeId: extension.cdeId, - cdeType: extension.cdeType, - data, - })); - default: - throw new Error(`[mina funnel] unhandled extension: ${extension.cdeType}`); + if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { + let from = slotToMinaTimestamp(extension.startSlot, cache.genesisTime); + + const cursor = cursors && cursors[extension.cdeId]; + + const data = getEventCdeData( + this.config.archive, + extension, + from, + startingSlotTimestamp - 1, + x => minaTimestampToSlot(x, cache.genesisTime), + this.chainName + ).then(mapCursorPaginatedData(extension.cdeId)); + + return data.then(data => ({ + cdeId: extension.cdeId, + cdeType: extension.cdeType, + data, + })); + } else if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { + let from = slotToMinaTimestamp(extension.startSlot, cache.genesisTime); + + const cursor = cursors && cursors[extension.cdeId]; + + const data = getActionCdeData( + this.config.archive, + extension, + from, + startingSlotTimestamp - 1, + x => minaTimestampToSlot(x, cache.genesisTime), + this.chainName + ).then(mapCursorPaginatedData(extension.cdeId)); + + return data.then(data => ({ + cdeId: extension.cdeId, + cdeType: extension.cdeType, + data, + })); + } else { + throw new Error(`[mina funnel] unhandled extension: ${extension.cdeType}`); } }) ), @@ -293,9 +334,8 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { } catch (err) { doLog(`[paima-funnel::readPresyncData] Exception occurred while reading blocks: ${err}`); - // TODO: remove later, but it's useful for debugging, since otherwise we - // just get stuck in a loop with infinite errors sometimes. process.exit(1); + throw err; } } @@ -317,14 +357,25 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const genesisTime = await getGenesisTime(config.graphql); + console.log('startingBlockHeight', startingBlockHeight); const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) .timestamp as number; - newEntry.updateStartingSlot( - slotToTimestamp(timestampToSlot(startingBlockTimestamp * 1000, genesisTime), genesisTime), + console.log('startingBlockTimestamp', startingBlockTimestamp); + + const slot = minaTimestampToSlot( + baseChainTimestampToMina(startingBlockTimestamp, config.confirmationDepth), genesisTime ); + const slotAsMinaTimestamp = slotToMinaTimestamp(slot, genesisTime); + + console.log('slot', slot, slotAsMinaTimestamp); + + newEntry.updateStartingSlot(slotAsMinaTimestamp, genesisTime); + + console.log('starting slot timestamp', newEntry.getState().startingSlotTimestamp); + const cursors = await getCarpCursors.run(undefined, dbTx); const extensions = sharedData.extensions diff --git a/packages/engine/paima-runtime/src/cde-config/loading.ts b/packages/engine/paima-runtime/src/cde-config/loading.ts index 4f875abdb..4881aeb55 100644 --- a/packages/engine/paima-runtime/src/cde-config/loading.ts +++ b/packages/engine/paima-runtime/src/cde-config/loading.ts @@ -43,7 +43,8 @@ import { ChainDataExtensionErc6551RegistryConfig, ChainDataExtensionErc721Config, ChainDataExtensionGenericConfig, - ChainDataExtensionMinaGenericConfig, + ChainDataExtensionMinaEventGenericConfig, + ChainDataExtensionMinaActionGenericConfig, } from '@paima/sm'; import { loadAbi } from './utils.js'; import assertNever from 'assert-never'; @@ -163,10 +164,16 @@ export function parseCdeConfigFile(configFileData: string): Static { const cdeId = cdeDatum.cdeId; diff --git a/packages/engine/paima-sm/src/cde-processing.ts b/packages/engine/paima-sm/src/cde-processing.ts index bdb814d40..13b29dd79 100644 --- a/packages/engine/paima-sm/src/cde-processing.ts +++ b/packages/engine/paima-sm/src/cde-processing.ts @@ -48,7 +48,9 @@ export async function cdeTransitionFunction( return await processCardanoTransferDatum(cdeDatum, inPresync); case ChainDataExtensionDatumType.CardanoMintBurn: return await processCardanoMintBurnDatum(cdeDatum, inPresync); - case ChainDataExtensionDatumType.MinaGeneric: + case ChainDataExtensionDatumType.MinaEventGeneric: + return await processGenericDatum(cdeDatum, inPresync); + case ChainDataExtensionDatumType.MinaActionGeneric: return await processGenericDatum(cdeDatum, inPresync); default: assertNever(cdeDatum); diff --git a/packages/engine/paima-sm/src/types.ts b/packages/engine/paima-sm/src/types.ts index 833661949..133791ddf 100644 --- a/packages/engine/paima-sm/src/types.ts +++ b/packages/engine/paima-sm/src/types.ts @@ -255,8 +255,13 @@ export interface CdeCardanoMintBurnDatum extends CdeDatumBase { paginationCursor: { cursor: string; finished: boolean }; } -export interface CdeMinaGenericDatum extends CdeDatumBase { - cdeDatumType: ChainDataExtensionDatumType.MinaGeneric; +export interface CdeMinaEventGenericDatum extends CdeDatumBase { + cdeDatumType: ChainDataExtensionDatumType.MinaEventGeneric; + paginationCursor: { cursor: string; finished: boolean }; + scheduledPrefix: string; +} +export interface CdeMinaActionGenericDatum extends CdeDatumBase { + cdeDatumType: ChainDataExtensionDatumType.MinaActionGeneric; paginationCursor: { cursor: string; finished: boolean }; scheduledPrefix: string; } @@ -274,7 +279,8 @@ export type ChainDataExtensionDatum = | CdeCardanoAssetUtxoDatum | CdeCardanoTransferDatum | CdeCardanoMintBurnDatum - | CdeMinaGenericDatum; + | CdeMinaEventGenericDatum + | CdeMinaActionGenericDatum; export enum CdeEntryTypeName { Generic = 'generic', @@ -288,7 +294,8 @@ export enum CdeEntryTypeName { CardanoDelayedAsset = 'cardano-delayed-asset', CardanoTransfer = 'cardano-transfer', CardanoMintBurn = 'cardano-mint-burn', - MinaGeneric = 'mina-generic', + MinaEventGeneric = 'mina-event-generic', + MinaActionGeneric = 'mina-action-generic', } const EvmAddress = Type.Transform(Type.RegExp('0x[0-9a-fA-F]{40}')) @@ -478,19 +485,36 @@ export type ChainDataExtensionCardanoMintBurn = ChainDataExtensionBase & cdeType: ChainDataExtensionType.CardanoMintBurn; }; -export const ChainDataExtensionMinaGenericConfig = Type.Object({ - type: Type.Literal(CdeEntryTypeName.MinaGeneric), - // TODO: change? +export const ChainDataExtensionMinaEventGenericConfig = Type.Object({ + type: Type.Literal(CdeEntryTypeName.MinaEventGeneric), + address: Type.String(), + scheduledPrefix: Type.String(), + startSlot: Type.Number(), + name: Type.String(), +}); + +export type TChainDataExtensionMinaEventGenericConfig = Static< + typeof ChainDataExtensionGenericConfig +>; +export type ChainDataExtensionMinaEventGeneric = ChainDataExtensionBase & + Static & { + cdeType: ChainDataExtensionType.MinaEventGeneric; + }; + +export const ChainDataExtensionMinaActionGenericConfig = Type.Object({ + type: Type.Literal(CdeEntryTypeName.MinaActionGeneric), address: Type.String(), scheduledPrefix: Type.String(), startSlot: Type.Number(), name: Type.String(), }); -export type TChainDataExtensionMinaGenericConfig = Static; -export type ChainDataExtensionMinaGeneric = ChainDataExtensionBase & - Static & { - cdeType: ChainDataExtensionType.MinaGeneric; +export type TChainDataExtensionMinaActionGenericConfig = Static< + typeof ChainDataExtensionGenericConfig +>; +export type ChainDataExtensionMinaActionGeneric = ChainDataExtensionBase & + Static & { + cdeType: ChainDataExtensionType.MinaActionGeneric; }; export const CdeConfig = Type.Object({ @@ -508,7 +532,8 @@ export const CdeConfig = Type.Object({ ChainDataExtensionCardanoDelayedAssetConfig, ChainDataExtensionCardanoTransferConfig, ChainDataExtensionCardanoMintBurnConfig, - ChainDataExtensionMinaGenericConfig, + ChainDataExtensionMinaEventGenericConfig, + ChainDataExtensionMinaActionGenericConfig, ]), Type.Partial(Type.Object({ network: Type.String() })), ]) @@ -541,7 +566,8 @@ export type ChainDataExtension = ( | ChainDataExtensionCardanoDelayedAsset | ChainDataExtensionCardanoTransfer | ChainDataExtensionCardanoMintBurn - | ChainDataExtensionMinaGeneric + | ChainDataExtensionMinaEventGeneric + | ChainDataExtensionMinaActionGeneric ) & { network: string | undefined }; export type GameStateTransitionFunctionRouter = ( diff --git a/packages/paima-sdk/paima-utils/src/constants.ts b/packages/paima-sdk/paima-utils/src/constants.ts index d286ecc8c..90ac1df2c 100644 --- a/packages/paima-sdk/paima-utils/src/constants.ts +++ b/packages/paima-sdk/paima-utils/src/constants.ts @@ -24,7 +24,8 @@ export const enum ChainDataExtensionType { CardanoTransfer = 10, CardanoMintBurn = 11, ERC1155 = 12, - MinaGeneric = 13, + MinaEventGeneric = 13, + MinaActionGeneric = 14, } export const enum ChainDataExtensionDatumType { @@ -40,7 +41,8 @@ export const enum ChainDataExtensionDatumType { CardanoTransfer, CardanoMintBurn, Erc1155Transfer, - MinaGeneric, + MinaEventGeneric, + MinaActionGeneric, } export const FUNNEL_PRESYNC_FINISHED = 'finished'; From 7fc458cbca568e1ac4a06c257c3bf5c15880a743 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Wed, 10 Apr 2024 20:56:07 -0300 Subject: [PATCH 06/31] add after/cursor argument to the queries --- .../paima-funnel/src/cde/minaGeneric.ts | 26 ++++++++------- .../paima-funnel/src/funnels/mina/funnel.ts | 10 +++--- packages/engine/paima-sm/src/index.ts | 33 +------------------ 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 366c67c86..74882fc69 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -12,10 +12,9 @@ export async function getEventCdeData( fromTimestamp: number, toTimestamp: number, getBlockNumber: (minaTimestamp: number) => number, - network: string + network: string, + cursor?: string ): Promise { - console.log('from,to', fromTimestamp, toTimestamp); - const grouped = [] as { blockInfo: { height: number; @@ -25,6 +24,8 @@ export async function getEventCdeData( eventData: { data: string[][]; txHash: string }[]; }[]; + const after = cursor ? `after: "${cursor}"` : ''; + const data = await fetch(minaArchive, { method: 'POST', @@ -39,7 +40,8 @@ export async function getEventCdeData( input: { address: "${extension.address}", fromTimestamp: "${fromTimestamp}", - toTimestamp: "${toTimestamp}" + toTimestamp: "${toTimestamp}", + ${after} } ) { blockInfo { @@ -58,10 +60,6 @@ export async function getEventCdeData( }), }) .then(res => res.json()) - .then(res => { - console.log('res', JSON.stringify(res, undefined, '\t')); - return res; - }) .then(json => json.data.events); const events = data as { @@ -105,8 +103,11 @@ export async function getActionCdeData( fromTimestamp: number, toTimestamp: number, getBlockNumber: (minaTimestamp: number) => number, - network: string + network: string, + cursor?: string ): Promise { + const after = cursor ? `after: "${cursor}"` : ''; + const data = await fetch(minaArchive, { method: 'POST', @@ -122,6 +123,7 @@ export async function getActionCdeData( address: "${extension.address}", fromTimestamp: "${fromTimestamp}", toTimestamp: "${toTimestamp}" + ${after} } ) { blockInfo { @@ -144,7 +146,7 @@ export async function getActionCdeData( const actions = data as { blockInfo: { height: number; timestamp: string }; - actionData: { data: string[]; hash: string }[]; + actionData: { data: string[]; transactionInfo: { hash: string } }[]; }[]; const grouped = [] as { @@ -162,11 +164,11 @@ export async function getActionCdeData( for (const blockEvent of block.actionData) { if ( actionData[actionData.length - 1] && - blockEvent.hash == actionData[actionData.length - 1].txHash + blockEvent.transactionInfo.hash == actionData[actionData.length - 1].txHash ) { actionData[actionData.length - 1].data.push(blockEvent.data); } else { - actionData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); + actionData.push({ txHash: blockEvent.transactionInfo.hash, data: [blockEvent.data] }); } } diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 73a890473..314f007c1 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -61,6 +61,8 @@ function baseChainTimestampToMina(baseChainTimestamp: number, confirmationDepth: // TODO: maybe using the node's rpc here it's not really safe? if it's out of // sync with the archive db we could end up skipping events // it would be better to have an endpoint for this on the archive api +// either that, or the archive node api should take a block hash, and if it's +// not there it should return an error. async function findMinaConfirmedSlot(graphql: string, confirmationDepth: number): Promise { const body = JSON.stringify({ query: ` @@ -277,7 +279,8 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { from, startingSlotTimestamp - 1, x => minaTimestampToSlot(x, cache.genesisTime), - this.chainName + this.chainName, + cursor?.cursor ).then(mapCursorPaginatedData(extension.cdeId)); return data.then(data => ({ @@ -296,7 +299,8 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { from, startingSlotTimestamp - 1, x => minaTimestampToSlot(x, cache.genesisTime), - this.chainName + this.chainName, + cursor?.cursor ).then(mapCursorPaginatedData(extension.cdeId)); return data.then(data => ({ @@ -334,8 +338,6 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { } catch (err) { doLog(`[paima-funnel::readPresyncData] Exception occurred while reading blocks: ${err}`); - process.exit(1); - throw err; } } diff --git a/packages/engine/paima-sm/src/index.ts b/packages/engine/paima-sm/src/index.ts index ae50db4b1..ff1e710bc 100644 --- a/packages/engine/paima-sm/src/index.ts +++ b/packages/engine/paima-sm/src/index.ts @@ -329,38 +329,7 @@ async function processCardanoCdeData( ): Promise { return await processCdeDataBase(cdeData, dbTx, inPresync, async () => { if ('cursor' in paginationCursor) { - await updateCardanoPaginationCursor.run( - { - cde_id: paginationCursor.cdeId, - cursor: paginationCursor.cursor, - finished: paginationCursor.finished, - }, - dbTx - ); - } else { - await updateCardanoPaginationCursor.run( - { - cde_id: paginationCursor.cdeId, - cursor: paginationCursor.slot.toString(), - finished: paginationCursor.finished, - }, - dbTx - ); - } - return; - }); -} - -async function processMinaCdeData( - paginationCursor: - | { cdeId: number; cursor: string; finished: boolean } - | { cdeId: number; slot: number; finished: boolean }, - cdeData: ChainDataExtensionDatum[] | undefined, - dbTx: PoolClient, - inPresync: boolean -): Promise { - return await processCdeDataBase(cdeData, dbTx, inPresync, async () => { - if ('cursor' in paginationCursor) { + console.log('cursor', paginationCursor); await updateCardanoPaginationCursor.run( { cde_id: paginationCursor.cdeId, From cba5de1124bca039279bf3a91c40b043b3a160b7 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Wed, 10 Apr 2024 20:57:23 -0300 Subject: [PATCH 07/31] remove some debugging logs --- packages/engine/paima-funnel/src/funnels/mina/funnel.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 314f007c1..292169cf8 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -359,12 +359,9 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const genesisTime = await getGenesisTime(config.graphql); - console.log('startingBlockHeight', startingBlockHeight); const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) .timestamp as number; - console.log('startingBlockTimestamp', startingBlockTimestamp); - const slot = minaTimestampToSlot( baseChainTimestampToMina(startingBlockTimestamp, config.confirmationDepth), genesisTime @@ -372,12 +369,8 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const slotAsMinaTimestamp = slotToMinaTimestamp(slot, genesisTime); - console.log('slot', slot, slotAsMinaTimestamp); - newEntry.updateStartingSlot(slotAsMinaTimestamp, genesisTime); - console.log('starting slot timestamp', newEntry.getState().startingSlotTimestamp); - const cursors = await getCarpCursors.run(undefined, dbTx); const extensions = sharedData.extensions From ed6fa32f39f065181f6dbb84ce641673f9cd2bbc Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Wed, 10 Apr 2024 22:30:59 -0300 Subject: [PATCH 08/31] rename cardano cursors table to generic cursor based pagination this way it can be re-used also for mina or any other system, since there is no fundamental difference --- .../paima-funnel/src/funnels/carp/funnel.ts | 8 +-- .../paima-funnel/src/funnels/mina/funnel.ts | 6 +- .../paima-runtime/src/cde-config/loading.ts | 7 +- packages/engine/paima-sm/src/index.ts | 7 +- packages/node-sdk/paima-db/migrations/up.sql | 2 +- packages/node-sdk/paima-db/src/index.ts | 4 +- .../node-sdk/paima-db/src/paima-tables.ts | 12 ++-- ...cde-cardano-tracking-pagination.queries.ts | 67 ------------------- .../sql/cde-cardano-tracking-pagination.sql | 15 ----- .../cde-cursor-tracking-pagination.queries.ts | 67 +++++++++++++++++++ .../sql/cde-cursor-tracking-pagination.sql | 15 +++++ .../paima-utils/src/config/loading.ts | 1 + packages/paima-sdk/paima-utils/src/index.ts | 2 + 13 files changed, 107 insertions(+), 106 deletions(-) delete mode 100644 packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts delete mode 100644 packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql create mode 100644 packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts create mode 100644 packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql diff --git a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts index 74239cb93..5475047b3 100644 --- a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts @@ -30,7 +30,7 @@ import { query } from '@dcspark/carp-client'; import { Routes } from '@dcspark/carp-client'; import { FUNNEL_PRESYNC_FINISHED, InternalEventType } from '@paima/utils'; import { CarpFunnelCacheEntry } from '../FunnelCache.js'; -import { getCardanoEpoch, getCarpCursors } from '@paima/db'; +import { getCardanoEpoch, getPaginationCursors } from '@paima/db'; import type { BlockTxPair } from '@dcspark/carp-client'; const delayForWaitingForFinalityLoop = 1000; @@ -246,7 +246,7 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { Promise.all( this.sharedData.extensions .filter(extension => { - if (!('startSlot' in extension)) { + if (extension.network !== this.chainName) { return false; } @@ -456,10 +456,10 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { newEntry.updateEpoch(epoch[0].epoch); } - const cursors = await getCarpCursors.run(undefined, dbTx); + const cursors = await getPaginationCursors.run(undefined, dbTx); const extensions = sharedData.extensions - .filter(extensions => (extensions.network = config.network)) + .filter(extensions => extensions.network === config.network) .map(extension => extension.cdeId); for (const cursor of cursors) { diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 292169cf8..5c0b4335d 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -11,7 +11,7 @@ import { BaseFunnel } from '../BaseFunnel.js'; import type { FunnelSharedData } from '../BaseFunnel.js'; import type { PoolClient } from 'pg'; import { FUNNEL_PRESYNC_FINISHED, ConfigNetworkType } from '@paima/utils'; -import { getCarpCursors } from '@paima/db'; +import { getPaginationCursors } from '@paima/db'; import { getActionCdeData, getEventCdeData } from '../../cde/minaGeneric.js'; import type { MinaConfig } from '@paima/utils'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; @@ -371,10 +371,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { newEntry.updateStartingSlot(slotAsMinaTimestamp, genesisTime); - const cursors = await getCarpCursors.run(undefined, dbTx); + const cursors = await getPaginationCursors.run(undefined, dbTx); const extensions = sharedData.extensions - .filter(extensions => (extensions.network = chainName)) + .filter(extensions => extensions.network === chainName) .map(extension => extension.cdeId); for (const cursor of cursors) { diff --git a/packages/engine/paima-runtime/src/cde-config/loading.ts b/packages/engine/paima-runtime/src/cde-config/loading.ts index 4881aeb55..9df5c5746 100644 --- a/packages/engine/paima-runtime/src/cde-config/loading.ts +++ b/packages/engine/paima-runtime/src/cde-config/loading.ts @@ -19,6 +19,7 @@ import { defaultEvmMainNetworkName, defaultCardanoNetworkName, getErc1155Contract, + defaultMinaNetworkName, } from '@paima/utils'; import type { @@ -345,8 +346,7 @@ async function instantiateExtension( case CdeEntryTypeName.MinaEventGeneric: return { ...config, - // TODO: unhardcode - network: config.network || 'mina', + network: config.network || defaultMinaNetworkName, cdeId: index, hash: hashConfig(config), cdeType: ChainDataExtensionType.MinaEventGeneric, @@ -354,8 +354,7 @@ async function instantiateExtension( case CdeEntryTypeName.MinaActionGeneric: return { ...config, - // TODO: unhardcode - network: config.network || 'mina', + network: config.network || defaultMinaNetworkName, cdeId: index, hash: hashConfig(config), cdeType: ChainDataExtensionType.MinaActionGeneric, diff --git a/packages/engine/paima-sm/src/index.ts b/packages/engine/paima-sm/src/index.ts index ff1e710bc..65184b18a 100644 --- a/packages/engine/paima-sm/src/index.ts +++ b/packages/engine/paima-sm/src/index.ts @@ -28,7 +28,7 @@ import { getMainAddress, NO_USER_ID, updateCardanoEpoch, - updateCardanoPaginationCursor, + updatePaginationCursor, } from '@paima/db'; import Prando from '@paima/prando'; @@ -329,8 +329,7 @@ async function processCardanoCdeData( ): Promise { return await processCdeDataBase(cdeData, dbTx, inPresync, async () => { if ('cursor' in paginationCursor) { - console.log('cursor', paginationCursor); - await updateCardanoPaginationCursor.run( + await updatePaginationCursor.run( { cde_id: paginationCursor.cdeId, cursor: paginationCursor.cursor, @@ -339,7 +338,7 @@ async function processCardanoCdeData( dbTx ); } else { - await updateCardanoPaginationCursor.run( + await updatePaginationCursor.run( { cde_id: paginationCursor.cdeId, cursor: paginationCursor.slot.toString(), diff --git a/packages/node-sdk/paima-db/migrations/up.sql b/packages/node-sdk/paima-db/migrations/up.sql index 3bfc12861..ebee57cc2 100644 --- a/packages/node-sdk/paima-db/migrations/up.sql +++ b/packages/node-sdk/paima-db/migrations/up.sql @@ -218,7 +218,7 @@ CREATE TABLE cde_cardano_asset_utxos ( PRIMARY KEY(cde_id,tx_id,output_index,cip14_fingerprint) ); -CREATE TABLE cde_tracking_cardano_pagination ( +CREATE TABLE cde_tracking_cursor_pagination ( cde_id INTEGER PRIMARY KEY, cursor TEXT NOT NULL, finished BOOLEAN NOT NULL diff --git a/packages/node-sdk/paima-db/src/index.ts b/packages/node-sdk/paima-db/src/index.ts index 13a0639fa..c398c26f6 100644 --- a/packages/node-sdk/paima-db/src/index.ts +++ b/packages/node-sdk/paima-db/src/index.ts @@ -48,8 +48,8 @@ export { cdeSpendCardanoAssetUtxo, ICdeCardanoAssetUtxosByAddressParams, } from './sql/cde-cardano-asset-utxos.queries.js'; -export * from './sql/cde-cardano-tracking-pagination.queries.js'; -export type * from './sql/cde-cardano-tracking-pagination.queries.js'; +export * from './sql/cde-cursor-tracking-pagination.queries.js'; +export type * from './sql/cde-cursor-tracking-pagination.queries.js'; export * from './sql/cde-cardano-transfer.queries.js'; export type * from './sql/cde-cardano-transfer.queries.js'; export { cdeCardanoMintBurnInsert } from './sql/cde-cardano-mint-burn.queries.js'; diff --git a/packages/node-sdk/paima-db/src/paima-tables.ts b/packages/node-sdk/paima-db/src/paima-tables.ts index 6566161b9..52161aa3b 100644 --- a/packages/node-sdk/paima-db/src/paima-tables.ts +++ b/packages/node-sdk/paima-db/src/paima-tables.ts @@ -118,16 +118,16 @@ const TABLE_DATA_CDE_TRACKING_CARDANO: TableData = { creationQuery: QUERY_CREATE_TABLE_CDE_TRACKING_CARDANO, }; -const QUERY_CREATE_TABLE_CDE_TRACKING_CARDANO_PAGINATION = ` -CREATE TABLE cde_tracking_cardano_pagination ( +const QUERY_CREATE_TABLE_CDE_TRACKING_CURSOR_PAGINATION = ` +CREATE TABLE cde_tracking_cursor_pagination ( cde_id INTEGER PRIMARY KEY, cursor TEXT NOT NULL, finished BOOLEAN NOT NULL ); `; -const TABLE_DATA_CDE_TRACKING_CARDANO_PAGINATION: TableData = { - tableName: 'cde_tracking_cardano_pagination', +const TABLE_DATA_CDE_TRACKING_CURSOR_PAGINATION: TableData = { + tableName: 'cde_tracking_cursor_pagination', primaryKeyColumns: ['cde_id'], columnData: packTuples([ ['cde_id', 'integer', 'NO', ''], @@ -135,7 +135,7 @@ const TABLE_DATA_CDE_TRACKING_CARDANO_PAGINATION: TableData = { ['finished', 'boolean', 'NO', ''], ]), serialColumns: [], - creationQuery: QUERY_CREATE_TABLE_CDE_TRACKING_CARDANO_PAGINATION, + creationQuery: QUERY_CREATE_TABLE_CDE_TRACKING_CURSOR_PAGINATION, }; const QUERY_CREATE_TABLE_CDE = ` @@ -660,7 +660,7 @@ export const TABLES: TableData[] = [ TABLE_DATA_ADDRESSES, TABLE_DATA_DELEGATIONS, TABLE_DATA_CARDANO_LAST_EPOCH, - TABLE_DATA_CDE_TRACKING_CARDANO_PAGINATION, + TABLE_DATA_CDE_TRACKING_CURSOR_PAGINATION, TABLE_DATA_CDE_CARDANO_TRANSFER, TABLE_DATA_CDE_CARDANO_MINT_BURN, ]; diff --git a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts b/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts deleted file mode 100644 index 5e5d5a524..000000000 --- a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.queries.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** Types generated for queries found in "src/sql/cde-cardano-tracking-pagination.sql" */ -import { PreparedQuery } from '@pgtyped/runtime'; - -/** 'GetCarpCursors' parameters type */ -export type IGetCarpCursorsParams = void; - -/** 'GetCarpCursors' return type */ -export interface IGetCarpCursorsResult { - cde_id: number; - cursor: string; - finished: boolean; -} - -/** 'GetCarpCursors' query type */ -export interface IGetCarpCursorsQuery { - params: IGetCarpCursorsParams; - result: IGetCarpCursorsResult; -} - -const getCarpCursorsIR: any = {"usedParamSet":{},"params":[],"statement":"select * from cde_tracking_cardano_pagination"}; - -/** - * Query generated from SQL: - * ``` - * select * from cde_tracking_cardano_pagination - * ``` - */ -export const getCarpCursors = new PreparedQuery(getCarpCursorsIR); - - -/** 'UpdateCardanoPaginationCursor' parameters type */ -export interface IUpdateCardanoPaginationCursorParams { - cde_id: number; - cursor: string; - finished: boolean; -} - -/** 'UpdateCardanoPaginationCursor' return type */ -export type IUpdateCardanoPaginationCursorResult = void; - -/** 'UpdateCardanoPaginationCursor' query type */ -export interface IUpdateCardanoPaginationCursorQuery { - params: IUpdateCardanoPaginationCursorParams; - result: IUpdateCardanoPaginationCursorResult; -} - -const updateCardanoPaginationCursorIR: any = {"usedParamSet":{"cde_id":true,"cursor":true,"finished":true},"params":[{"name":"cde_id","required":true,"transform":{"type":"scalar"},"locs":[{"a":89,"b":96}]},{"name":"cursor","required":true,"transform":{"type":"scalar"},"locs":[{"a":101,"b":108},{"a":170,"b":177}]},{"name":"finished","required":true,"transform":{"type":"scalar"},"locs":[{"a":113,"b":122},{"a":191,"b":200}]}],"statement":"INSERT INTO cde_tracking_cardano_pagination(\n cde_id,\n cursor,\n finished\n) VALUES (\n :cde_id!,\n :cursor!,\n :finished!\n)\nON CONFLICT (cde_id)\nDO UPDATE SET cursor = :cursor!, finished = :finished!"}; - -/** - * Query generated from SQL: - * ``` - * INSERT INTO cde_tracking_cardano_pagination( - * cde_id, - * cursor, - * finished - * ) VALUES ( - * :cde_id!, - * :cursor!, - * :finished! - * ) - * ON CONFLICT (cde_id) - * DO UPDATE SET cursor = :cursor!, finished = :finished! - * ``` - */ -export const updateCardanoPaginationCursor = new PreparedQuery(updateCardanoPaginationCursorIR); - - diff --git a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql b/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql deleted file mode 100644 index 31307a50b..000000000 --- a/packages/node-sdk/paima-db/src/sql/cde-cardano-tracking-pagination.sql +++ /dev/null @@ -1,15 +0,0 @@ -/* @name getCarpCursors */ -select * from cde_tracking_cardano_pagination; - -/* @name updateCardanoPaginationCursor */ -INSERT INTO cde_tracking_cardano_pagination( - cde_id, - cursor, - finished -) VALUES ( - :cde_id!, - :cursor!, - :finished! -) -ON CONFLICT (cde_id) -DO UPDATE SET cursor = :cursor!, finished = :finished!; \ No newline at end of file diff --git a/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts new file mode 100644 index 000000000..de0533d8e --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.queries.ts @@ -0,0 +1,67 @@ +/** Types generated for queries found in "src/sql/cde-cursor-tracking-pagination.sql" */ +import { PreparedQuery } from '@pgtyped/runtime'; + +/** 'GetPaginationCursors' parameters type */ +export type IGetPaginationCursorsParams = void; + +/** 'GetPaginationCursors' return type */ +export interface IGetPaginationCursorsResult { + cde_id: number; + cursor: string; + finished: boolean; +} + +/** 'GetPaginationCursors' query type */ +export interface IGetPaginationCursorsQuery { + params: IGetPaginationCursorsParams; + result: IGetPaginationCursorsResult; +} + +const getPaginationCursorsIR: any = {"usedParamSet":{},"params":[],"statement":"select * from cde_tracking_cursor_pagination"}; + +/** + * Query generated from SQL: + * ``` + * select * from cde_tracking_cursor_pagination + * ``` + */ +export const getPaginationCursors = new PreparedQuery(getPaginationCursorsIR); + + +/** 'UpdatePaginationCursor' parameters type */ +export interface IUpdatePaginationCursorParams { + cde_id: number; + cursor: string; + finished: boolean; +} + +/** 'UpdatePaginationCursor' return type */ +export type IUpdatePaginationCursorResult = void; + +/** 'UpdatePaginationCursor' query type */ +export interface IUpdatePaginationCursorQuery { + params: IUpdatePaginationCursorParams; + result: IUpdatePaginationCursorResult; +} + +const updatePaginationCursorIR: any = {"usedParamSet":{"cde_id":true,"cursor":true,"finished":true},"params":[{"name":"cde_id","required":true,"transform":{"type":"scalar"},"locs":[{"a":88,"b":95}]},{"name":"cursor","required":true,"transform":{"type":"scalar"},"locs":[{"a":100,"b":107},{"a":169,"b":176}]},{"name":"finished","required":true,"transform":{"type":"scalar"},"locs":[{"a":112,"b":121},{"a":190,"b":199}]}],"statement":"INSERT INTO cde_tracking_cursor_pagination(\n cde_id,\n cursor,\n finished\n) VALUES (\n :cde_id!,\n :cursor!,\n :finished!\n)\nON CONFLICT (cde_id)\nDO UPDATE SET cursor = :cursor!, finished = :finished!"}; + +/** + * Query generated from SQL: + * ``` + * INSERT INTO cde_tracking_cursor_pagination( + * cde_id, + * cursor, + * finished + * ) VALUES ( + * :cde_id!, + * :cursor!, + * :finished! + * ) + * ON CONFLICT (cde_id) + * DO UPDATE SET cursor = :cursor!, finished = :finished! + * ``` + */ +export const updatePaginationCursor = new PreparedQuery(updatePaginationCursorIR); + + diff --git a/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql new file mode 100644 index 000000000..9e25d295d --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/cde-cursor-tracking-pagination.sql @@ -0,0 +1,15 @@ +/* @name getPaginationCursors */ +select * from cde_tracking_cursor_pagination; + +/* @name updatePaginationCursor */ +INSERT INTO cde_tracking_cursor_pagination( + cde_id, + cursor, + finished +) VALUES ( + :cde_id!, + :cursor!, + :finished! +) +ON CONFLICT (cde_id) +DO UPDATE SET cursor = :cursor!, finished = :finished!; \ No newline at end of file diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index 95cce721a..68cb23923 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -139,6 +139,7 @@ const minaConfigDefaults = { // will need to be removed afterwards export const defaultEvmMainNetworkName = 'evm'; export const defaultCardanoNetworkName = 'cardano'; +export const defaultMinaNetworkName = 'mina'; export async function loadConfig(): Promise | undefined> { let configFileData: string; diff --git a/packages/paima-sdk/paima-utils/src/index.ts b/packages/paima-sdk/paima-utils/src/index.ts index 8e2b3822d..4b0c7b820 100644 --- a/packages/paima-sdk/paima-utils/src/index.ts +++ b/packages/paima-sdk/paima-utils/src/index.ts @@ -14,6 +14,7 @@ import { ConfigNetworkType, defaultEvmMainNetworkName, defaultCardanoNetworkName, + defaultMinaNetworkName, } from './config/loading.js'; export * from './config.js'; @@ -39,6 +40,7 @@ export { ConfigNetworkType, defaultEvmMainNetworkName, defaultCardanoNetworkName, + defaultMinaNetworkName, }; export const DEFAULT_GAS_PRICE = '61000000000' as const; From d3a63e96c999d334f9c7fe4769f135a79bdaa01c Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Thu, 11 Apr 2024 00:00:06 -0300 Subject: [PATCH 09/31] add paginationLimit setting --- packages/paima-sdk/paima-utils/src/config/loading.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index 68cb23923..469bbd112 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -65,6 +65,7 @@ export const MinaConfigSchema = Type.Object({ archive: Type.String(), graphql: Type.String(), confirmationDepth: Type.Number(), + paginationLimit: Type.Number({ default: 50 }), //presyncStepSize: Type.Number({ default: 1000 }), }); From 6a18c160388ce9995f0f490163636b357202c384 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Thu, 11 Apr 2024 15:04:36 -0300 Subject: [PATCH 10/31] dehardcode slot duration --- .../paima-funnel/src/funnels/mina/funnel.ts | 85 ++++++++++++++----- .../paima-utils/src/config/loading.ts | 6 +- 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 5c0b4335d..3bb46e503 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -44,18 +44,20 @@ async function getGenesisTime(graphql: string): Promise { return Number.parseInt(genesisTime, 10); } -const SLOT_DURATION = 3 * 60000; - -function slotToMinaTimestamp(slot: number, genesisTime: number): number { - return slot * SLOT_DURATION + genesisTime; +function slotToMinaTimestamp(slot: number, genesisTime: number, slotDuration: number): number { + return slot * slotDuration * 1000 + genesisTime; } -function minaTimestampToSlot(ts: number, genesisTime: number): number { - return Math.max(Math.floor((ts - genesisTime) / SLOT_DURATION), 0); +function minaTimestampToSlot(ts: number, genesisTime: number, slotDuration: number): number { + return Math.max(Math.floor((ts - genesisTime) / (slotDuration * 1000)), 0); } -function baseChainTimestampToMina(baseChainTimestamp: number, confirmationDepth: number): number { - return Math.max(baseChainTimestamp * 1000 - SLOT_DURATION * confirmationDepth, 0); +function baseChainTimestampToMina( + baseChainTimestamp: number, + confirmationDepth: number, + slotDuration: number +): number { + return Math.max(baseChainTimestamp * 1000 - slotDuration * 1000 * confirmationDepth, 0); } // TODO: maybe using the node's rpc here it's not really safe? if it's out of @@ -130,7 +132,11 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { this.config.confirmationDepth ); - const confirmedTimestamp = slotToMinaTimestamp(confirmedSlot, cachedState.genesisTime); + const confirmedTimestamp = slotToMinaTimestamp( + confirmedSlot, + cachedState.genesisTime, + this.config.slotDuration + ); const fromTimestamp = this.cache.getState().lastPoint?.timestamp || cachedState.startingSlotTimestamp; @@ -139,12 +145,21 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { confirmedTimestamp, baseChainTimestampToMina( baseData[baseData.length - 1].timestamp, - this.config.confirmationDepth + this.config.confirmationDepth, + this.config.slotDuration ) ); - const fromSlot = minaTimestampToSlot(fromTimestamp, cachedState.genesisTime); - const toSlot = minaTimestampToSlot(toTimestamp, cachedState.genesisTime); + const fromSlot = minaTimestampToSlot( + fromTimestamp, + cachedState.genesisTime, + this.config.slotDuration + ); + const toSlot = minaTimestampToSlot( + toTimestamp, + cachedState.genesisTime, + this.config.slotDuration + ); const mapSlotsToEvmNumbers: { [slot: number]: number } = {}; @@ -154,8 +169,13 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { while ( curr < baseData.length && minaTimestampToSlot( - baseChainTimestampToMina(baseData[curr].timestamp, this.config.confirmationDepth), - cachedState.genesisTime + baseChainTimestampToMina( + baseData[curr].timestamp, + this.config.confirmationDepth, + this.config.slotDuration + ), + cachedState.genesisTime, + this.config.slotDuration ) < slot ) curr++; @@ -174,7 +194,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, fromTimestamp, toTimestamp, - ts => mapSlotsToEvmNumbers[minaTimestampToSlot(ts, cachedState.genesisTime)], + ts => + mapSlotsToEvmNumbers[ + minaTimestampToSlot(ts, cachedState.genesisTime, this.config.slotDuration) + ], this.chainName ); @@ -187,7 +210,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, fromTimestamp, toTimestamp, - ts => mapSlotsToEvmNumbers[minaTimestampToSlot(ts, cachedState.genesisTime)], + ts => + mapSlotsToEvmNumbers[ + minaTimestampToSlot(ts, cachedState.genesisTime, this.config.slotDuration) + ], this.chainName ); @@ -269,7 +295,11 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { }) .map(extension => { if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { - let from = slotToMinaTimestamp(extension.startSlot, cache.genesisTime); + let from = slotToMinaTimestamp( + extension.startSlot, + cache.genesisTime, + this.config.slotDuration + ); const cursor = cursors && cursors[extension.cdeId]; @@ -278,7 +308,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, from, startingSlotTimestamp - 1, - x => minaTimestampToSlot(x, cache.genesisTime), + x => minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), this.chainName, cursor?.cursor ).then(mapCursorPaginatedData(extension.cdeId)); @@ -289,7 +319,11 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { data, })); } else if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { - let from = slotToMinaTimestamp(extension.startSlot, cache.genesisTime); + let from = slotToMinaTimestamp( + extension.startSlot, + cache.genesisTime, + this.config.slotDuration + ); const cursor = cursors && cursors[extension.cdeId]; @@ -298,7 +332,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, from, startingSlotTimestamp - 1, - x => minaTimestampToSlot(x, cache.genesisTime), + x => minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), this.chainName, cursor?.cursor ).then(mapCursorPaginatedData(extension.cdeId)); @@ -363,11 +397,16 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { .timestamp as number; const slot = minaTimestampToSlot( - baseChainTimestampToMina(startingBlockTimestamp, config.confirmationDepth), - genesisTime + baseChainTimestampToMina( + startingBlockTimestamp, + config.confirmationDepth, + config.slotDuration + ), + genesisTime, + config.slotDuration ); - const slotAsMinaTimestamp = slotToMinaTimestamp(slot, genesisTime); + const slotAsMinaTimestamp = slotToMinaTimestamp(slot, genesisTime, config.slotDuration); newEntry.updateStartingSlot(slotAsMinaTimestamp, genesisTime); diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index 469bbd112..badeb292d 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -64,8 +64,10 @@ export type CardanoConfig = Static; export const MinaConfigSchema = Type.Object({ archive: Type.String(), graphql: Type.String(), + // k confirmationDepth: Type.Number(), paginationLimit: Type.Number({ default: 50 }), + slotDuration: Type.Number(), //presyncStepSize: Type.Number({ default: 1000 }), }); @@ -133,7 +135,9 @@ const cardanoConfigDefaults = { }; const minaConfigDefaults = { - confirmationDepth: 15, + // lightnet defaults + confirmationDepth: 30, + slotDuration: 20, }; // used as a placeholder name for the ENV fallback mechanism From ccdc0fc2f5d220c0c622104867520a660ecd01a8 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Fri, 12 Apr 2024 21:27:37 -0300 Subject: [PATCH 11/31] replace the node's graphql api with direct db queries --- package-lock.json | 15 +++- packages/engine/paima-funnel/package.json | 3 +- .../paima-funnel/src/funnels/FunnelCache.ts | 9 +- .../paima-funnel/src/funnels/mina/funnel.ts | 85 ++++--------------- .../paima-utils/src/config/loading.ts | 2 +- 5 files changed, 41 insertions(+), 73 deletions(-) diff --git a/package-lock.json b/package-lock.json index 658431b04..2994e3ce7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20367,6 +20367,18 @@ "node": ">= 0.4" } }, + "node_modules/postgres": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.4.tgz", + "integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, "node_modules/postgres-array": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", @@ -26338,7 +26350,8 @@ "dependencies": { "@dcspark/cardano-multiplatform-lib-nodejs": "5.2.0", "@dcspark/carp-client": "^3.1.0", - "assert-never": "^1.2.1" + "assert-never": "^1.2.1", + "postgres": "^3.3.5" }, "devDependencies": { "typescript": "^5.3.3" diff --git a/packages/engine/paima-funnel/package.json b/packages/engine/paima-funnel/package.json index d1f33c654..a9d1dad5c 100644 --- a/packages/engine/paima-funnel/package.json +++ b/packages/engine/paima-funnel/package.json @@ -19,6 +19,7 @@ "dependencies": { "assert-never": "^1.2.1", "@dcspark/carp-client": "^3.1.0", - "@dcspark/cardano-multiplatform-lib-nodejs": "5.2.0" + "@dcspark/cardano-multiplatform-lib-nodejs": "5.2.0", + "postgres": "^3.3.5" } } diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts index 78a423832..3c9ba1a8f 100644 --- a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts +++ b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts @@ -1,4 +1,5 @@ import type { ChainData } from '@paima/sm'; +import postgres from 'postgres'; export interface FunnelCacheEntry { /** @@ -184,6 +185,7 @@ export type MinaFunnelCacheEntryState = { startingSlotTimestamp: number; lastPoint: { timestamp: number } | undefined; genesisTime: number; + pg: postgres.Sql; cursors: | { [cdeId: number]: { cursor: string; finished: boolean }; @@ -195,12 +197,17 @@ export class MinaFunnelCacheEntry implements FunnelCacheEntry { private state: MinaFunnelCacheEntryState | null = null; public static readonly SYMBOL = Symbol('MinaFunnelStartingSlot'); - public updateStartingSlot(startingSlotTimestamp: number, genesisTime: number): void { + public updateStartingSlot( + startingSlotTimestamp: number, + genesisTime: number, + pg: postgres.Sql + ): void { this.state = { startingSlotTimestamp, genesisTime, lastPoint: this.state?.lastPoint, cursors: this.state?.cursors, + pg, }; } diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 3bb46e503..f69cec2d7 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -15,33 +15,19 @@ import { getPaginationCursors } from '@paima/db'; import { getActionCdeData, getEventCdeData } from '../../cde/minaGeneric.js'; import type { MinaConfig } from '@paima/utils'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; +import postgres from 'postgres'; -async function getGenesisTime(graphql: string): Promise { - const genesisTime = (await fetch(graphql, { - method: 'POST', - - headers: { - 'Content-Type': 'application/json', - }, - - body: JSON.stringify({ - query: ` - { - genesisBlock { - protocolState { - blockchainState { - utcDate - } - } - } - } - `, - }), - }) - .then(res => res.json()) - .then(res => res.data.genesisBlock.protocolState.blockchainState.utcDate)) as string; +async function getGenesisTime(pg: postgres.Sql): Promise { + const row = await pg`select timestamp from blocks where height = 1;`; - return Number.parseInt(genesisTime, 10); + return Number.parseInt(row[0]['timestamp'], 10); +} + +async function findMinaConfirmedSlot(pg: postgres.Sql): Promise { + const row = + await pg`select global_slot_since_genesis from blocks where chain_status = 'canonical' order by height desc limit 1;`; + + return Number.parseInt(row[0]['global_slot_since_genesis'], 10); } function slotToMinaTimestamp(slot: number, genesisTime: number, slotDuration: number): number { @@ -60,44 +46,6 @@ function baseChainTimestampToMina( return Math.max(baseChainTimestamp * 1000 - slotDuration * 1000 * confirmationDepth, 0); } -// TODO: maybe using the node's rpc here it's not really safe? if it's out of -// sync with the archive db we could end up skipping events -// it would be better to have an endpoint for this on the archive api -// either that, or the archive node api should take a block hash, and if it's -// not there it should return an error. -async function findMinaConfirmedSlot(graphql: string, confirmationDepth: number): Promise { - const body = JSON.stringify({ - query: ` - { - bestChain(maxLength: ${confirmationDepth}) { - stateHash - protocolState { - consensusState { - blockHeight - slotSinceGenesis - } - previousStateHash - } - } - } - `, - }); - - const confirmedSlot = await fetch(graphql, { - method: 'POST', - - headers: { - 'Content-Type': 'application/json', - }, - - body, - }) - .then(res => res.json()) - .then(res => res.data.bestChain[0].protocolState.consensusState.slotSinceGenesis); - - return Number.parseInt(confirmedSlot, 10); -} - export class MinaFunnel extends BaseFunnel implements ChainFunnel { config: MinaConfig; chainName: string; @@ -127,10 +75,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { let cachedState = this.cache.getState(); - const confirmedSlot = await findMinaConfirmedSlot( - this.config.graphql, - this.config.confirmationDepth - ); + const confirmedSlot = await findMinaConfirmedSlot(cachedState.pg); const confirmedTimestamp = slotToMinaTimestamp( confirmedSlot, @@ -391,7 +336,9 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const newEntry = new MinaFunnelCacheEntry(); sharedData.cacheManager.cacheEntries[MinaFunnelCacheEntry.SYMBOL] = newEntry; - const genesisTime = await getGenesisTime(config.graphql); + const pg = postgres(config.archiveConnectionString); + + const genesisTime = await getGenesisTime(pg); const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) .timestamp as number; @@ -408,7 +355,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const slotAsMinaTimestamp = slotToMinaTimestamp(slot, genesisTime, config.slotDuration); - newEntry.updateStartingSlot(slotAsMinaTimestamp, genesisTime); + newEntry.updateStartingSlot(slotAsMinaTimestamp, genesisTime, pg); const cursors = await getPaginationCursors.run(undefined, dbTx); diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index badeb292d..7bf7d3d7f 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -62,8 +62,8 @@ export const CardanoConfigSchema = Type.Object({ export type CardanoConfig = Static; export const MinaConfigSchema = Type.Object({ + archiveConnectionString: Type.String(), archive: Type.String(), - graphql: Type.String(), // k confirmationDepth: Type.Number(), paginationLimit: Type.Number({ default: 50 }), From 1743988c019f40135762c720741ffb481fe4a606 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Sat, 13 Apr 2024 00:48:04 -0300 Subject: [PATCH 12/31] replace the archive's graphql with direct db queries --- .../paima-funnel/src/cde/minaGeneric.ts | 365 +++++++++++++----- .../paima-funnel/src/funnels/mina/funnel.ts | 8 +- .../paima-utils/src/config/loading.ts | 2 - 3 files changed, 274 insertions(+), 101 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 74882fc69..36fd061fa 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -5,9 +5,10 @@ import type { ChainDataExtensionMinaEventGeneric, } from '@paima/sm'; import { ChainDataExtensionDatumType } from '@paima/utils'; +import postgres from 'postgres'; export async function getEventCdeData( - minaArchive: string, + pg: postgres.Sql, extension: ChainDataExtensionMinaEventGeneric, fromTimestamp: number, toTimestamp: number, @@ -24,64 +25,29 @@ export async function getEventCdeData( eventData: { data: string[][]; txHash: string }[]; }[]; - const after = cursor ? `after: "${cursor}"` : ''; - - const data = await fetch(minaArchive, { - method: 'POST', - - headers: { - 'Content-Type': 'application/json', - }, - - body: JSON.stringify({ - query: ` - { - events( - input: { - address: "${extension.address}", - fromTimestamp: "${fromTimestamp}", - toTimestamp: "${toTimestamp}", - ${after} - } - ) { - blockInfo { - height - timestamp - } - eventData { - data - transactionInfo { - hash - } - } - } - } - `, - }), - }) - .then(res => res.json()) - .then(json => json.data.events); - - const events = data as { - blockInfo: { height: number; timestamp: string }; - eventData: { data: string[]; transactionInfo: { hash: string } }[]; - }[]; + const events = await getEventsQuery( + pg, + extension.address, + toTimestamp.toString(), + fromTimestamp.toString(), + cursor + ); for (const block of events) { const eventData = [] as { data: string[][]; txHash: string }[]; - for (const blockEvent of block.eventData) { + for (const blockEvent of block.events) { if ( eventData[eventData.length - 1] && - blockEvent.transactionInfo.hash == eventData[eventData.length - 1].txHash + blockEvent.hash == eventData[eventData.length - 1].txHash ) { eventData[eventData.length - 1].data.push(blockEvent.data); } else { - eventData.push({ txHash: blockEvent.transactionInfo.hash, data: [blockEvent.data] }); + eventData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); } } - grouped.push({ blockInfo: block.blockInfo, eventData }); + grouped.push({ blockInfo: { height: block.height, timestamp: block.timestamp }, eventData }); } return grouped.flatMap(perBlock => @@ -98,7 +64,7 @@ export async function getEventCdeData( } export async function getActionCdeData( - minaArchive: string, + pg: postgres.Sql, extension: ChainDataExtensionMinaActionGeneric, fromTimestamp: number, toTimestamp: number, @@ -106,49 +72,6 @@ export async function getActionCdeData( network: string, cursor?: string ): Promise { - const after = cursor ? `after: "${cursor}"` : ''; - - const data = await fetch(minaArchive, { - method: 'POST', - - headers: { - 'Content-Type': 'application/json', - }, - - body: JSON.stringify({ - query: ` - { - actions( - input: { - address: "${extension.address}", - fromTimestamp: "${fromTimestamp}", - toTimestamp: "${toTimestamp}" - ${after} - } - ) { - blockInfo { - height - timestamp - } - actionData { - data - transactionInfo { - hash - } - } - } - } - `, - }), - }) - .then(res => res.json()) - .then(json => json.data.actions); - - const actions = data as { - blockInfo: { height: number; timestamp: string }; - actionData: { data: string[]; transactionInfo: { hash: string } }[]; - }[]; - const grouped = [] as { blockInfo: { height: number; @@ -158,21 +81,29 @@ export async function getActionCdeData( actionData: { data: string[][]; txHash: string }[]; }[]; + const actions = await getActionsQuery( + pg, + extension.address, + toTimestamp.toString(), + fromTimestamp.toString(), + cursor + ); + for (const block of actions) { const actionData = [] as { data: string[][]; txHash: string }[]; - for (const blockEvent of block.actionData) { + for (const blockEvent of block.actions) { if ( actionData[actionData.length - 1] && - blockEvent.transactionInfo.hash == actionData[actionData.length - 1].txHash + blockEvent.hash == actionData[actionData.length - 1].txHash ) { actionData[actionData.length - 1].data.push(blockEvent.data); } else { - actionData.push({ txHash: blockEvent.transactionInfo.hash, data: [blockEvent.data] }); + actionData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); } } - grouped.push({ blockInfo: block.blockInfo, actionData }); + grouped.push({ blockInfo: { height: block.height, timestamp: block.timestamp }, actionData }); } return grouped.flatMap(perBlock => @@ -187,3 +118,247 @@ export async function getActionCdeData( })) ); } + +function fullChainCTE(db_client: postgres.Sql, toTimestamp?: string, fromTimestamp?: string) { + return db_client` + full_chain AS ( + SELECT + id, state_hash, height, global_slot_since_genesis, timestamp + FROM + blocks b + WHERE + chain_status = 'canonical' + ${fromTimestamp ? db_client`AND b.timestamp::decimal >= ${fromTimestamp}::decimal` : db_client``} + ${toTimestamp ? db_client`AND b.timestamp::decimal <= ${toTimestamp}::decimal` : db_client``} + ORDER BY height + ) + `; +} + +function accountIdentifierCTE(db_client: postgres.Sql, address: string) { + return db_client` + account_identifier AS ( + SELECT + id AS requesting_zkapp_account_identifier_id + FROM + account_identifiers ai + WHERE + ai.public_key_id = (SELECT id FROM public_keys WHERE value = ${address}) + )`; +} + +function blocksAccessedCTE(db_client: postgres.Sql) { + return db_client` + blocks_accessed AS + ( + SELECT + requesting_zkapp_account_identifier_id, + block_id, + account_identifier_id, + zkapp_id, + id AS account_access_id, + state_hash, + height, + global_slot_since_genesis, + timestamp + FROM + account_identifier ai + INNER JOIN accounts_accessed aa ON ai.requesting_zkapp_account_identifier_id = aa.account_identifier_id + INNER JOIN full_chain b ON aa.block_id = b.id + )`; +} + +function emittedZkAppCommandsCTE(db_client: postgres.Sql, after?: string) { + return db_client` + emitted_zkapp_commands AS ( + SELECT + blocks_accessed.*, + zkcu.id AS zkapp_account_update_id, + zkapp_fee_payer_body_id, + zkapp_account_updates_ids, + authorization_kind, + status, + memo, + hash, + body_id, + events_id, + actions_id + FROM + blocks_accessed + INNER JOIN blocks_zkapp_commands bzkc ON blocks_accessed.block_id = bzkc.block_id + INNER JOIN zkapp_commands zkc ON bzkc.zkapp_command_id = zkc.id + INNER JOIN zkapp_account_update zkcu ON zkcu.id = ANY(zkc.zkapp_account_updates_ids) + INNER JOIN zkapp_account_update_body zkcu_body ON zkcu_body.id = zkcu.body_id + AND zkcu_body.account_identifier_id = requesting_zkapp_account_identifier_id + ${after ? db_client`AND zkc.id > (SELECT id FROM zkapp_commands WHERE zkapp_commands.hash = ${after})` : db_client``} + WHERE + bzkc.status <> 'failed' + )`; +} + +function emittedEventsCTE(db_client: postgres.Sql) { + return db_client` + emitted_events AS ( + SELECT + *, + zke.id AS zkapp_event_id, + zke.element_ids AS zkapp_event_element_ids, + zkfa.id AS zkapp_event_array_id + FROM + emitted_zkapp_commands + INNER JOIN zkapp_events zke ON zke.id = events_id + INNER JOIN zkapp_field_array zkfa ON zkfa.id = ANY(zke.element_ids) + INNER JOIN zkapp_field zkf ON zkf.id = ANY(zkfa.element_ids) + ) + `; +} + +function emittedActionsCTE(db_client: postgres.Sql) { + return db_client` + emitted_actions AS ( + SELECT + *, + zke.id AS zkapp_event_id, + zke.element_ids AS zkapp_event_element_ids, + zkfa.id AS zkapp_event_array_id + FROM + emitted_zkapp_commands + INNER JOIN zkapp_events zke ON zke.id = actions_id + INNER JOIN zkapp_field_array zkfa ON zkfa.id = ANY(zke.element_ids) + INNER JOIN zkapp_field zkf ON zkf.id = ANY(zkfa.element_ids) + ) + `; +} + +function emittedActionStateCTE( + db_client: postgres.Sql, + fromActionState?: string, + endActionState?: string +) { + return db_client` + emitted_action_state AS ( + SELECT + zkf0.field AS action_state_value1, + zkf1.field AS action_state_value2, + zkf2.field AS action_state_value3, + zkf3.field AS action_state_value4, + zkf4.field AS action_state_value5, + emitted_actions.* + FROM + emitted_actions + INNER JOIN zkapp_accounts zkacc ON zkacc.id = emitted_actions.zkapp_id + INNER JOIN zkapp_action_states zks ON zks.id = zkacc.action_state_id + INNER JOIN zkapp_field zkf0 ON zkf0.id = zks.element0 + INNER JOIN zkapp_field zkf1 ON zkf1.id = zks.element1 + INNER JOIN zkapp_field zkf2 ON zkf2.id = zks.element2 + INNER JOIN zkapp_field zkf3 ON zkf3.id = zks.element3 + INNER JOIN zkapp_field zkf4 ON zkf4.id = zks.element4 + WHERE + 1 = 1 + ${ + fromActionState + ? db_client`AND zkf0.id >= (SELECT id FROM zkapp_field WHERE field = ${fromActionState})` + : db_client`` + } + ${ + endActionState + ? db_client`AND zkf0.id <= (SELECT id FROM zkapp_field WHERE field = ${endActionState})` + : db_client`` + } + )`; +} + +interface EventsPerBlock { + timestamp: string; + height: number; + events: { + hash: string; + data: string[]; + }[]; +} + +export function getEventsQuery( + db_client: postgres.Sql, + address: string, + toTimestamp?: string, + fromTimestamp?: string, + after?: string, + limit?: string +) { + return db_client` + WITH + ${fullChainCTE(db_client, toTimestamp, fromTimestamp)}, + ${accountIdentifierCTE(db_client, address)}, + ${blocksAccessedCTE(db_client)}, + ${emittedZkAppCommandsCTE(db_client, after)}, + ${emittedEventsCTE(db_client)}, + grouped_events AS ( + SELECT + MAX(timestamp) timestamp, + MAX(hash) hash, + JSON_AGG(field) events_data, + MAX(height) height + FROM emitted_events + GROUP BY ( + zkapp_account_update_id, + zkapp_event_array_id + ) + ORDER BY height + ) + SELECT + MAX(timestamp) timestamp, + height, + JSON_AGG(JSON_BUILD_OBJECT('hash', hash, 'data', events_data)) events + FROM grouped_events + GROUP BY height + ORDER BY height + `; +} + +interface ActionsPerBlock { + timestamp: string; + height: number; + actions: { + hash: string; + data: string[]; + }[]; +} + +export function getActionsQuery( + db_client: postgres.Sql, + address: string, + toTimestamp?: string, + fromTimestamp?: string, + after?: string, + limit?: string +) { + return db_client` + WITH + ${fullChainCTE(db_client, toTimestamp, fromTimestamp)}, + ${accountIdentifierCTE(db_client, address)}, + ${blocksAccessedCTE(db_client)}, + ${emittedZkAppCommandsCTE(db_client, after)}, + ${emittedActionsCTE(db_client)}, + ${emittedActionStateCTE(db_client)}, + grouped_events AS ( + SELECT + MAX(timestamp) timestamp, + MAX(hash) hash, + JSON_AGG(field) actions_data, + MAX(height) height + FROM emitted_actions + GROUP BY ( + zkapp_account_update_id, + zkapp_event_array_id + ) + ORDER BY height + ) + SELECT + MAX(timestamp) timestamp, + height, + JSON_AGG(JSON_BUILD_OBJECT('hash', hash, 'data', actions_data)) actions + FROM grouped_events + GROUP BY height + ORDER BY height + `; +} diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index f69cec2d7..93f5727d3 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -135,7 +135,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { (promises, extension) => { if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { const promise = getEventCdeData( - this.config.archive, + cachedState.pg, extension, fromTimestamp, toTimestamp, @@ -151,7 +151,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { const promise = getActionCdeData( - this.config.archive, + cachedState.pg, extension, fromTimestamp, toTimestamp, @@ -249,7 +249,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const cursor = cursors && cursors[extension.cdeId]; const data = getEventCdeData( - this.config.archive, + cache.pg, extension, from, startingSlotTimestamp - 1, @@ -273,7 +273,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const cursor = cursors && cursors[extension.cdeId]; const data = getActionCdeData( - this.config.archive, + cache.pg, extension, from, startingSlotTimestamp - 1, diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index 7bf7d3d7f..1a42c5198 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -63,12 +63,10 @@ export type CardanoConfig = Static; export const MinaConfigSchema = Type.Object({ archiveConnectionString: Type.String(), - archive: Type.String(), // k confirmationDepth: Type.Number(), paginationLimit: Type.Number({ default: 50 }), slotDuration: Type.Number(), - //presyncStepSize: Type.Number({ default: 1000 }), }); export type MinaConfig = Static; From a6f9155a5cb38c32c37b788fe7e68b095b00abcd Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:41:45 -0300 Subject: [PATCH 13/31] add limit to the query --- .../paima-funnel/src/cde/minaGeneric.ts | 234 ++++++++++-------- .../paima-funnel/src/funnels/mina/funnel.ts | 64 +++-- 2 files changed, 162 insertions(+), 136 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 36fd061fa..22912e8a9 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -1,122 +1,158 @@ import type { CdeMinaActionGenericDatum, CdeMinaEventGenericDatum, + ChainDataExtensionDatum, ChainDataExtensionMinaActionGeneric, ChainDataExtensionMinaEventGeneric, } from '@paima/sm'; import { ChainDataExtensionDatumType } from '@paima/utils'; import postgres from 'postgres'; -export async function getEventCdeData( +export async function getEventCdeData(args: { + pg: postgres.Sql; + extension: ChainDataExtensionMinaEventGeneric; + fromTimestamp: number; + toTimestamp: number; + getBlockNumber: (minaTimestamp: number) => number; + network: string; + isPresync: boolean; + cursor?: string; + limit?: number; +}): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { + return getCdeData( + getEventsQuery, + ChainDataExtensionDatumType.MinaEventGeneric, + args.pg, + args.extension, + args.fromTimestamp, + args.toTimestamp, + args.getBlockNumber, + args.network, + args.isPresync, + args.cursor, + args.limit + ); +} + +export async function getActionCdeData(args: { + pg: postgres.Sql; + extension: ChainDataExtensionMinaActionGeneric; + fromTimestamp: number; + toTimestamp: number; + getBlockNumber: (minaTimestamp: number) => number; + network: string; + isPresync: boolean; + cursor?: string; + limit?: number; +}): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { + return getCdeData( + getActionsQuery, + ChainDataExtensionDatumType.MinaActionGeneric, + args.pg, + args.extension, + args.fromTimestamp, + args.toTimestamp, + args.getBlockNumber, + args.network, + args.isPresync, + args.cursor, + args.limit + ); +} + +export async function getCdeData( + f: typeof getEventsQuery | typeof getActionsQuery, + cdeDatumType: + | ChainDataExtensionDatumType.MinaActionGeneric + | ChainDataExtensionDatumType.MinaEventGeneric, pg: postgres.Sql, - extension: ChainDataExtensionMinaEventGeneric, + extension: ChainDataExtensionMinaEventGeneric | ChainDataExtensionMinaActionGeneric, fromTimestamp: number, toTimestamp: number, getBlockNumber: (minaTimestamp: number) => number, network: string, - cursor?: string -): Promise { - const grouped = [] as { - blockInfo: { - height: number; - timestamp: string; - }; - // TODO: could each data by just a tuple? - eventData: { data: string[][]; txHash: string }[]; - }[]; + isPresync: boolean, + cursor?: string, + limit?: number +): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { + const result = [] as ChainDataExtensionDatum[]; - const events = await getEventsQuery( - pg, - extension.address, - toTimestamp.toString(), - fromTimestamp.toString(), - cursor - ); + console.log('outside', cursor, limit, fromTimestamp, toTimestamp); + while (true) { + console.log('inside', cursor, limit, fromTimestamp, toTimestamp); - for (const block of events) { - const eventData = [] as { data: string[][]; txHash: string }[]; + const unmapped = await f( + pg, + extension.address, + toTimestamp.toString(), + fromTimestamp.toString(), + cursor, + limit?.toString() + ); - for (const blockEvent of block.events) { - if ( - eventData[eventData.length - 1] && - blockEvent.hash == eventData[eventData.length - 1].txHash - ) { - eventData[eventData.length - 1].data.push(blockEvent.data); - } else { - eventData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); - } + const grouped = groupByTx(unmapped); + + const events = grouped.flatMap(perBlock => + perBlock.eventsData.map(txEvent => ({ + cdeId: extension.cdeId, + cdeDatumType, + blockNumber: getBlockNumber(Number.parseInt(perBlock.blockInfo.timestamp, 10)), + payload: txEvent, + network, + scheduledPrefix: extension.scheduledPrefix, + paginationCursor: { cursor: txEvent.txHash, finished: false }, + })) + ); + + if (events.length > 0) { + const last = events[events.length - 1]; + + cursor = last.paginationCursor.cursor; } - grouped.push({ blockInfo: { height: block.height, timestamp: block.timestamp }, eventData }); + events.forEach(element => { + result.push(element); + }); + + if (events.length === 0 || isPresync) { + break; + } } - return grouped.flatMap(perBlock => - perBlock.eventData.map(txEvent => ({ - cdeId: extension.cdeId, - cdeDatumType: ChainDataExtensionDatumType.MinaEventGeneric, - blockNumber: getBlockNumber(Number.parseInt(perBlock.blockInfo.timestamp, 10)), - payload: txEvent, - network, - scheduledPrefix: extension.scheduledPrefix, - paginationCursor: { cursor: txEvent.txHash, finished: false }, - })) - ); + return result; } -export async function getActionCdeData( - pg: postgres.Sql, - extension: ChainDataExtensionMinaActionGeneric, - fromTimestamp: number, - toTimestamp: number, - getBlockNumber: (minaTimestamp: number) => number, - network: string, - cursor?: string -): Promise { +function groupByTx(events: postgres.RowList) { const grouped = [] as { blockInfo: { height: number; timestamp: string; }; // TODO: could each data by just a tuple? - actionData: { data: string[][]; txHash: string }[]; + eventsData: { data: string[][]; txHash: string }[]; }[]; - const actions = await getActionsQuery( - pg, - extension.address, - toTimestamp.toString(), - fromTimestamp.toString(), - cursor - ); - - for (const block of actions) { - const actionData = [] as { data: string[][]; txHash: string }[]; + for (const block of events) { + const eventData = [] as { data: string[][]; txHash: string }[]; - for (const blockEvent of block.actions) { + for (const blockEvent of block.events) { if ( - actionData[actionData.length - 1] && - blockEvent.hash == actionData[actionData.length - 1].txHash + eventData[eventData.length - 1] && + blockEvent.hash == eventData[eventData.length - 1].txHash ) { - actionData[actionData.length - 1].data.push(blockEvent.data); + eventData[eventData.length - 1].data.push(blockEvent.data); } else { - actionData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); + eventData.push({ txHash: blockEvent.hash, data: [blockEvent.data] }); } } - grouped.push({ blockInfo: { height: block.height, timestamp: block.timestamp }, actionData }); + grouped.push({ + blockInfo: { height: block.height, timestamp: block.timestamp }, + eventsData: eventData, + }); } - return grouped.flatMap(perBlock => - perBlock.actionData.map(txEvent => ({ - cdeId: extension.cdeId, - cdeDatumType: ChainDataExtensionDatumType.MinaActionGeneric, - blockNumber: getBlockNumber(Number.parseInt(perBlock.blockInfo.timestamp, 10)), - payload: txEvent, - network, - scheduledPrefix: extension.scheduledPrefix, - paginationCursor: { cursor: txEvent.txHash, finished: false }, - })) - ); + return grouped; } function fullChainCTE(db_client: postgres.Sql, toTimestamp?: string, fromTimestamp?: string) { @@ -230,11 +266,7 @@ function emittedActionsCTE(db_client: postgres.Sql) { `; } -function emittedActionStateCTE( - db_client: postgres.Sql, - fromActionState?: string, - endActionState?: string -) { +function emittedActionStateCTE(db_client: postgres.Sql) { return db_client` emitted_action_state AS ( SELECT @@ -253,29 +285,18 @@ function emittedActionStateCTE( INNER JOIN zkapp_field zkf2 ON zkf2.id = zks.element2 INNER JOIN zkapp_field zkf3 ON zkf3.id = zks.element3 INNER JOIN zkapp_field zkf4 ON zkf4.id = zks.element4 - WHERE - 1 = 1 - ${ - fromActionState - ? db_client`AND zkf0.id >= (SELECT id FROM zkapp_field WHERE field = ${fromActionState})` - : db_client`` - } - ${ - endActionState - ? db_client`AND zkf0.id <= (SELECT id FROM zkapp_field WHERE field = ${endActionState})` - : db_client`` - } )`; } -interface EventsPerBlock { +type PerBlock = { timestamp: string; height: number; + events: { hash: string; data: string[]; }[]; -} +}; export function getEventsQuery( db_client: postgres.Sql, @@ -285,7 +306,7 @@ export function getEventsQuery( after?: string, limit?: string ) { - return db_client` + return db_client` WITH ${fullChainCTE(db_client, toTimestamp, fromTimestamp)}, ${accountIdentifierCTE(db_client, address)}, @@ -312,18 +333,10 @@ export function getEventsQuery( FROM grouped_events GROUP BY height ORDER BY height + ${limit ? db_client`LIMIT ${limit}` : db_client``} `; } -interface ActionsPerBlock { - timestamp: string; - height: number; - actions: { - hash: string; - data: string[]; - }[]; -} - export function getActionsQuery( db_client: postgres.Sql, address: string, @@ -332,7 +345,7 @@ export function getActionsQuery( after?: string, limit?: string ) { - return db_client` + return db_client` WITH ${fullChainCTE(db_client, toTimestamp, fromTimestamp)}, ${accountIdentifierCTE(db_client, address)}, @@ -356,9 +369,10 @@ export function getActionsQuery( SELECT MAX(timestamp) timestamp, height, - JSON_AGG(JSON_BUILD_OBJECT('hash', hash, 'data', actions_data)) actions + JSON_AGG(JSON_BUILD_OBJECT('hash', hash, 'data', actions_data)) events FROM grouped_events GROUP BY height ORDER BY height + ${limit ? db_client`LIMIT ${limit}` : db_client``} `; } diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 93f5727d3..6f1497677 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -134,33 +134,39 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { this.sharedData.extensions.reduce( (promises, extension) => { if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { - const promise = getEventCdeData( - cachedState.pg, + const promise = getEventCdeData({ + pg: cachedState.pg, extension, fromTimestamp, toTimestamp, - ts => + getBlockNumber: ts => mapSlotsToEvmNumbers[ minaTimestampToSlot(ts, cachedState.genesisTime, this.config.slotDuration) ], - this.chainName - ); + network: this.chainName, + isPresync: false, + cursor: undefined, + limit: this.config.paginationLimit, + }); promises.push(promise); } if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { - const promise = getActionCdeData( - cachedState.pg, + const promise = getActionCdeData({ + pg: cachedState.pg, extension, fromTimestamp, toTimestamp, - ts => + getBlockNumber: ts => mapSlotsToEvmNumbers[ minaTimestampToSlot(ts, cachedState.genesisTime, this.config.slotDuration) ], - this.chainName - ); + network: this.chainName, + isPresync: false, + cursor: undefined, + limit: this.config.paginationLimit, + }); promises.push(promise); } @@ -248,15 +254,18 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const cursor = cursors && cursors[extension.cdeId]; - const data = getEventCdeData( - cache.pg, + const data = getEventCdeData({ + pg: cache.pg, extension, - from, - startingSlotTimestamp - 1, - x => minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), - this.chainName, - cursor?.cursor - ).then(mapCursorPaginatedData(extension.cdeId)); + fromTimestamp: from, + toTimestamp: startingSlotTimestamp - 1, + getBlockNumber: x => + minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), + network: this.chainName, + isPresync: true, + cursor: cursor?.cursor, + limit: this.config.paginationLimit, + }).then(mapCursorPaginatedData(extension.cdeId)); return data.then(data => ({ cdeId: extension.cdeId, @@ -272,15 +281,18 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const cursor = cursors && cursors[extension.cdeId]; - const data = getActionCdeData( - cache.pg, + const data = getActionCdeData({ + pg: cache.pg, extension, - from, - startingSlotTimestamp - 1, - x => minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), - this.chainName, - cursor?.cursor - ).then(mapCursorPaginatedData(extension.cdeId)); + fromTimestamp: from, + toTimestamp: startingSlotTimestamp - 1, + getBlockNumber: x => + minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), + network: this.chainName, + isPresync: true, + cursor: cursor?.cursor, + limit: this.config.paginationLimit, + }).then(mapCursorPaginatedData(extension.cdeId)); return data.then(data => ({ cdeId: extension.cdeId, From 5a08d13565e8ce9be846f98eca5ce262fe87df5b Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:44:11 -0300 Subject: [PATCH 14/31] add loop to wait for finality --- .../paima-funnel/src/cde/minaGeneric.ts | 2 +- .../paima-funnel/src/funnels/mina/funnel.ts | 38 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 22912e8a9..5c28fdf11 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -75,7 +75,7 @@ export async function getCdeData( cursor?: string, limit?: number ): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { - const result = [] as ChainDataExtensionDatum[]; + const result = [] as (CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]; console.log('outside', cursor, limit, fromTimestamp, toTimestamp); while (true) { diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 6f1497677..924c47899 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -1,4 +1,4 @@ -import { doLog, logError, ChainDataExtensionType } from '@paima/utils'; +import { doLog, logError, ChainDataExtensionType, delay } from '@paima/utils'; import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime'; import type { ChainData, @@ -17,6 +17,8 @@ import type { MinaConfig } from '@paima/utils'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; import postgres from 'postgres'; +const delayForWaitingForFinalityLoop = 1000; + async function getGenesisTime(pg: postgres.Sql): Promise { const row = await pg`select timestamp from blocks where height = 1;`; @@ -75,25 +77,33 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { let cachedState = this.cache.getState(); - const confirmedSlot = await findMinaConfirmedSlot(cachedState.pg); - - const confirmedTimestamp = slotToMinaTimestamp( - confirmedSlot, - cachedState.genesisTime, + const maxBaseTimestamp = baseChainTimestampToMina( + baseData[baseData.length - 1].timestamp, + this.config.confirmationDepth, this.config.slotDuration ); + let confirmedTimestamp; + while (true) { + const confirmedSlot = await findMinaConfirmedSlot(cachedState.pg); + + confirmedTimestamp = slotToMinaTimestamp( + confirmedSlot, + cachedState.genesisTime, + this.config.slotDuration + ); + + if (confirmedTimestamp >= maxBaseTimestamp) { + break; + } + + await delay(delayForWaitingForFinalityLoop); + } + const fromTimestamp = this.cache.getState().lastPoint?.timestamp || cachedState.startingSlotTimestamp; - const toTimestamp = Math.max( - confirmedTimestamp, - baseChainTimestampToMina( - baseData[baseData.length - 1].timestamp, - this.config.confirmationDepth, - this.config.slotDuration - ) - ); + const toTimestamp = Math.max(confirmedTimestamp, maxBaseTimestamp); const fromSlot = minaTimestampToSlot( fromTimestamp, From 57af769ccbfa366b9db1542495055659288ed6ef Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 15 Apr 2024 20:44:47 -0300 Subject: [PATCH 15/31] remove debugging logs --- packages/engine/paima-funnel/src/cde/minaGeneric.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 5c28fdf11..12a032d77 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -1,7 +1,6 @@ import type { CdeMinaActionGenericDatum, CdeMinaEventGenericDatum, - ChainDataExtensionDatum, ChainDataExtensionMinaActionGeneric, ChainDataExtensionMinaEventGeneric, } from '@paima/sm'; @@ -77,10 +76,7 @@ export async function getCdeData( ): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { const result = [] as (CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]; - console.log('outside', cursor, limit, fromTimestamp, toTimestamp); while (true) { - console.log('inside', cursor, limit, fromTimestamp, toTimestamp); - const unmapped = await f( pg, extension.address, From 92e7e599654952eb197df28c79f954de9dd99e19 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:23:09 -0300 Subject: [PATCH 16/31] minor refactors --- .../paima-funnel/src/cde/minaGeneric.ts | 4 ++-- .../paima-funnel/src/funnels/mina/funnel.ts | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 12a032d77..f2233e2d7 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -60,7 +60,7 @@ export async function getActionCdeData(args: { } export async function getCdeData( - f: typeof getEventsQuery | typeof getActionsQuery, + query: typeof getEventsQuery | typeof getActionsQuery, cdeDatumType: | ChainDataExtensionDatumType.MinaActionGeneric | ChainDataExtensionDatumType.MinaEventGeneric, @@ -77,7 +77,7 @@ export async function getCdeData( const result = [] as (CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]; while (true) { - const unmapped = await f( + const unmapped = await query( pg, extension.address, toTimestamp.toString(), diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 924c47899..7469a69c4 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -254,9 +254,9 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { return true; } }) - .map(extension => { + .map(async extension => { if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { - let from = slotToMinaTimestamp( + let fromTimestamp = slotToMinaTimestamp( extension.startSlot, cache.genesisTime, this.config.slotDuration @@ -264,10 +264,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const cursor = cursors && cursors[extension.cdeId]; - const data = getEventCdeData({ + const data = await getEventCdeData({ pg: cache.pg, extension, - fromTimestamp: from, + fromTimestamp, toTimestamp: startingSlotTimestamp - 1, getBlockNumber: x => minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), @@ -277,13 +277,13 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { limit: this.config.paginationLimit, }).then(mapCursorPaginatedData(extension.cdeId)); - return data.then(data => ({ + return { cdeId: extension.cdeId, cdeType: extension.cdeType, data, - })); + }; } else if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { - let from = slotToMinaTimestamp( + let fromTimestamp = slotToMinaTimestamp( extension.startSlot, cache.genesisTime, this.config.slotDuration @@ -291,10 +291,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const cursor = cursors && cursors[extension.cdeId]; - const data = getActionCdeData({ + const data = await getActionCdeData({ pg: cache.pg, extension, - fromTimestamp: from, + fromTimestamp, toTimestamp: startingSlotTimestamp - 1, getBlockNumber: x => minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), @@ -304,11 +304,11 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { limit: this.config.paginationLimit, }).then(mapCursorPaginatedData(extension.cdeId)); - return data.then(data => ({ + return { cdeId: extension.cdeId, cdeType: extension.cdeType, data, - })); + }; } else { throw new Error(`[mina funnel] unhandled extension: ${extension.cdeType}`); } From c1c0f4638f1f47461198785454c603a428e5f98f Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:28:23 -0300 Subject: [PATCH 17/31] remove unused fields from query --- packages/engine/paima-funnel/src/cde/minaGeneric.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index f2233e2d7..7421a094c 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -266,21 +266,11 @@ function emittedActionStateCTE(db_client: postgres.Sql) { return db_client` emitted_action_state AS ( SELECT - zkf0.field AS action_state_value1, - zkf1.field AS action_state_value2, - zkf2.field AS action_state_value3, - zkf3.field AS action_state_value4, - zkf4.field AS action_state_value5, emitted_actions.* FROM emitted_actions INNER JOIN zkapp_accounts zkacc ON zkacc.id = emitted_actions.zkapp_id INNER JOIN zkapp_action_states zks ON zks.id = zkacc.action_state_id - INNER JOIN zkapp_field zkf0 ON zkf0.id = zks.element0 - INNER JOIN zkapp_field zkf1 ON zkf1.id = zks.element1 - INNER JOIN zkapp_field zkf2 ON zkf2.id = zks.element2 - INNER JOIN zkapp_field zkf3 ON zkf3.id = zks.element3 - INNER JOIN zkapp_field zkf4 ON zkf4.id = zks.element4 )`; } From 19e36a68f0a09e910ae2605151b2cd05700d35d9 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:42:20 -0300 Subject: [PATCH 18/31] handle edge case for the cde generic handler --- .../engine/paima-funnel/src/funnels/mina/funnel.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 7469a69c4..a4090ef03 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -1,4 +1,4 @@ -import { doLog, logError, ChainDataExtensionType, delay } from '@paima/utils'; +import { doLog, logError, ChainDataExtensionType, delay, ENV } from '@paima/utils'; import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime'; import type { ChainData, @@ -269,8 +269,11 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, fromTimestamp, toTimestamp: startingSlotTimestamp - 1, - getBlockNumber: x => - minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), + // the handler for this cde stores the block height unmodified + // (even if the event is scheduled at the correct height), so + // handle this special case here, to have the events properly + // sorted. + getBlockNumber: _x => ENV.SM_START_BLOCKHEIGHT + 1, network: this.chainName, isPresync: true, cursor: cursor?.cursor, @@ -296,8 +299,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, fromTimestamp, toTimestamp: startingSlotTimestamp - 1, - getBlockNumber: x => - minaTimestampToSlot(x, cache.genesisTime, this.config.slotDuration), + getBlockNumber: _x => ENV.SM_START_BLOCKHEIGHT + 1, network: this.chainName, isPresync: true, cursor: cursor?.cursor, From 4c703665aa64720a0eff3cc585c51bff29f70a87 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Tue, 16 Apr 2024 01:47:52 -0300 Subject: [PATCH 19/31] fix upper range querying extra data --- packages/engine/paima-funnel/src/funnels/mina/funnel.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index a4090ef03..d0de8c0f6 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -84,6 +84,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { ); let confirmedTimestamp; + while (true) { const confirmedSlot = await findMinaConfirmedSlot(cachedState.pg); @@ -103,7 +104,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const fromTimestamp = this.cache.getState().lastPoint?.timestamp || cachedState.startingSlotTimestamp; - const toTimestamp = Math.max(confirmedTimestamp, maxBaseTimestamp); + const toTimestamp = maxBaseTimestamp; const fromSlot = minaTimestampToSlot( fromTimestamp, From c8f2f96c0e88b852e9e133c1f49a6db85eb39591 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:12:08 -0300 Subject: [PATCH 20/31] find confirmed timestamp directly instead of getting the slot first --- .../paima-funnel/src/funnels/mina/funnel.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index d0de8c0f6..d46eec906 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -25,11 +25,11 @@ async function getGenesisTime(pg: postgres.Sql): Promise { return Number.parseInt(row[0]['timestamp'], 10); } -async function findMinaConfirmedSlot(pg: postgres.Sql): Promise { +async function findMinaConfirmedTimestamp(pg: postgres.Sql): Promise { const row = - await pg`select global_slot_since_genesis from blocks where chain_status = 'canonical' order by height desc limit 1;`; + await pg`select timestamp from blocks where chain_status = 'canonical' order by height desc limit 1;`; - return Number.parseInt(row[0]['global_slot_since_genesis'], 10); + return Number.parseInt(row[0]['timestamp'], 10); } function slotToMinaTimestamp(slot: number, genesisTime: number, slotDuration: number): number { @@ -83,16 +83,8 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { this.config.slotDuration ); - let confirmedTimestamp; - while (true) { - const confirmedSlot = await findMinaConfirmedSlot(cachedState.pg); - - confirmedTimestamp = slotToMinaTimestamp( - confirmedSlot, - cachedState.genesisTime, - this.config.slotDuration - ); + const confirmedTimestamp = await findMinaConfirmedTimestamp(cachedState.pg); if (confirmedTimestamp >= maxBaseTimestamp) { break; From 5060bcae9f79bdd744171a917bc7062307afee6a Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Wed, 17 Apr 2024 15:58:44 -0300 Subject: [PATCH 21/31] rename the common paginated handling cde function --- packages/engine/paima-sm/src/index.ts | 39 +++++++++------------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/packages/engine/paima-sm/src/index.ts b/packages/engine/paima-sm/src/index.ts index 65184b18a..bd4d143d5 100644 --- a/packages/engine/paima-sm/src/index.ts +++ b/packages/engine/paima-sm/src/index.ts @@ -119,7 +119,7 @@ const SM: GameStateMachineInitializer = { ); } } else if (latestCdeData.networkType === ConfigNetworkType.CARDANO) { - const cdeDataLength = await processCardanoCdeData( + const cdeDataLength = await processPaginatedCdeData( latestCdeData.carpCursor, latestCdeData.extensionDatums, dbTx, @@ -131,8 +131,7 @@ const SM: GameStateMachineInitializer = { ); } } else if (latestCdeData.networkType === ConfigNetworkType.MINA) { - // not really correct, but should work for now. Probably that function should be renamed - const cdeDataLength = await processCardanoCdeData( + const cdeDataLength = await processPaginatedCdeData( latestCdeData.minaCursor, latestCdeData.extensionDatums, dbTx, @@ -319,34 +318,22 @@ async function processCdeData( }); } -async function processCardanoCdeData( - paginationCursor: - | { cdeId: number; cursor: string; finished: boolean } - | { cdeId: number; slot: number; finished: boolean }, +async function processPaginatedCdeData( + paginationCursor: { cdeId: number; cursor: string; finished: boolean }, cdeData: ChainDataExtensionDatum[] | undefined, dbTx: PoolClient, inPresync: boolean ): Promise { return await processCdeDataBase(cdeData, dbTx, inPresync, async () => { - if ('cursor' in paginationCursor) { - await updatePaginationCursor.run( - { - cde_id: paginationCursor.cdeId, - cursor: paginationCursor.cursor, - finished: paginationCursor.finished, - }, - dbTx - ); - } else { - await updatePaginationCursor.run( - { - cde_id: paginationCursor.cdeId, - cursor: paginationCursor.slot.toString(), - finished: paginationCursor.finished, - }, - dbTx - ); - } + await updatePaginationCursor.run( + { + cde_id: paginationCursor.cdeId, + cursor: paginationCursor.cursor, + finished: paginationCursor.finished, + }, + dbTx + ); + return; }); } From 02aaa3962311067e6be136862d253da09bf50528 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Wed, 17 Apr 2024 22:07:38 -0300 Subject: [PATCH 22/31] small refactor in funnel cursors filtering --- .../paima-funnel/src/funnels/carp/funnel.ts | 18 +++++++++++------- .../paima-funnel/src/funnels/mina/funnel.ts | 16 ++++++++++------ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts index 5475047b3..cc0ddd3a5 100644 --- a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts @@ -459,17 +459,21 @@ export class CarpFunnel extends BaseFunnel implements ChainFunnel { const cursors = await getPaginationCursors.run(undefined, dbTx); const extensions = sharedData.extensions - .filter(extensions => extensions.network === config.network) - .map(extension => extension.cdeId); - - for (const cursor of cursors) { - // TODO: performance concern? but not likely - if (extensions.find(cdeId => cdeId === cursor.cde_id)) + .filter(extensions => extensions.network === chainName) + .map(extension => extension.cdeId) + .reduce((set, cdeId) => { + set.add(cdeId); + return set; + }, new Set()); + + cursors + .filter(cursor => extensions.has(cursor.cde_id)) + .forEach(cursor => { newEntry.updateCursor(cursor.cde_id, { cursor: cursor.cursor, finished: cursor.finished, }); - } + }); return newEntry; })(); diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index d46eec906..d8c8cd649 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -378,16 +378,20 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const extensions = sharedData.extensions .filter(extensions => extensions.network === chainName) - .map(extension => extension.cdeId); - - for (const cursor of cursors) { - // TODO: performance concern? but not likely - if (extensions.find(cdeId => cdeId === cursor.cde_id)) + .map(extension => extension.cdeId) + .reduce((set, cdeId) => { + set.add(cdeId); + return set; + }, new Set()); + + cursors + .filter(cursor => extensions.has(cursor.cde_id)) + .forEach(cursor => { newEntry.updateCursor(cursor.cde_id, { cursor: cursor.cursor, finished: cursor.finished, }); - } + }); return newEntry; })(); From 6d54ea1dbcbd89b5335786d6d9ec7c72a95943d4 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:51:39 -0300 Subject: [PATCH 23/31] rework the timestamp to block mapping to not use slots --- .../paima-funnel/src/funnels/mina/funnel.ts | 51 +++++-------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index d8c8cd649..d278ab60c 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -98,40 +98,23 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const toTimestamp = maxBaseTimestamp; - const fromSlot = minaTimestampToSlot( - fromTimestamp, - cachedState.genesisTime, - this.config.slotDuration - ); - const toSlot = minaTimestampToSlot( - toTimestamp, - cachedState.genesisTime, - this.config.slotDuration - ); - - const mapSlotsToEvmNumbers: { [slot: number]: number } = {}; - - let curr = 0; - - for (let slot = fromSlot; slot <= toSlot; slot++) { + const getBlockNumber = (state: { curr: number }) => (ts: number) => { while ( - curr < baseData.length && - minaTimestampToSlot( - baseChainTimestampToMina( - baseData[curr].timestamp, - this.config.confirmationDepth, - this.config.slotDuration - ), - cachedState.genesisTime, + state.curr < baseData.length && + baseChainTimestampToMina( + baseData[state.curr].timestamp, + this.config.confirmationDepth, this.config.slotDuration - ) < slot + ) <= ts ) - curr++; + state.curr++; - if (curr < baseData.length) { - mapSlotsToEvmNumbers[slot] = baseData[curr].blockNumber; + if (state.curr < baseData.length) { + return baseData[state.curr].blockNumber; + } else { + throw new Error('got event out of the expected range'); } - } + }; const ungroupedCdeData = await Promise.all( this.sharedData.extensions.reduce( @@ -142,10 +125,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, fromTimestamp, toTimestamp, - getBlockNumber: ts => - mapSlotsToEvmNumbers[ - minaTimestampToSlot(ts, cachedState.genesisTime, this.config.slotDuration) - ], + getBlockNumber: getBlockNumber({ curr: 0 }), network: this.chainName, isPresync: false, cursor: undefined, @@ -161,10 +141,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { extension, fromTimestamp, toTimestamp, - getBlockNumber: ts => - mapSlotsToEvmNumbers[ - minaTimestampToSlot(ts, cachedState.genesisTime, this.config.slotDuration) - ], + getBlockNumber: getBlockNumber({ curr: 0 }), network: this.chainName, isPresync: false, cursor: undefined, From 5e3ce4fb647001d7a6f301f4533beb13b92ff931 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:33:47 -0300 Subject: [PATCH 24/31] replace startSlot for startBlockHeight in the extensions --- .../paima-funnel/src/cde/minaGeneric.ts | 36 +++++++++++----- .../paima-funnel/src/funnels/FunnelCache.ts | 2 +- .../paima-funnel/src/funnels/mina/funnel.ts | 41 ++++--------------- packages/engine/paima-sm/src/types.ts | 4 +- 4 files changed, 37 insertions(+), 46 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 7421a094c..5488cb973 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -17,6 +17,7 @@ export async function getEventCdeData(args: { isPresync: boolean; cursor?: string; limit?: number; + fromBlockHeight?: number; }): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { return getCdeData( getEventsQuery, @@ -29,7 +30,8 @@ export async function getEventCdeData(args: { args.network, args.isPresync, args.cursor, - args.limit + args.limit, + args.fromBlockHeight ); } @@ -43,6 +45,7 @@ export async function getActionCdeData(args: { isPresync: boolean; cursor?: string; limit?: number; + fromBlockHeight?: number; }): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { return getCdeData( getActionsQuery, @@ -55,7 +58,8 @@ export async function getActionCdeData(args: { args.network, args.isPresync, args.cursor, - args.limit + args.limit, + args.fromBlockHeight ); } @@ -72,7 +76,8 @@ export async function getCdeData( network: string, isPresync: boolean, cursor?: string, - limit?: number + limit?: number, + fromBlockHeight?: number ): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { const result = [] as (CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]; @@ -83,7 +88,8 @@ export async function getCdeData( toTimestamp.toString(), fromTimestamp.toString(), cursor, - limit?.toString() + limit?.toString(), + fromBlockHeight?.toString() ); const grouped = groupByTx(unmapped); @@ -151,9 +157,14 @@ function groupByTx(events: postgres.RowList) { return grouped; } -function fullChainCTE(db_client: postgres.Sql, toTimestamp?: string, fromTimestamp?: string) { +function canonicalChainCTE( + db_client: postgres.Sql, + toTimestamp?: string, + fromTimestamp?: string, + fromBlockHeight?: string +) { return db_client` - full_chain AS ( + canonical_chain AS ( SELECT id, state_hash, height, global_slot_since_genesis, timestamp FROM @@ -162,6 +173,7 @@ function fullChainCTE(db_client: postgres.Sql, toTimestamp?: string, fromTimesta chain_status = 'canonical' ${fromTimestamp ? db_client`AND b.timestamp::decimal >= ${fromTimestamp}::decimal` : db_client``} ${toTimestamp ? db_client`AND b.timestamp::decimal <= ${toTimestamp}::decimal` : db_client``} + ${fromBlockHeight ? db_client`AND b.height::decimal >= ${fromBlockHeight}::decimal` : db_client``} ORDER BY height ) `; @@ -196,7 +208,7 @@ function blocksAccessedCTE(db_client: postgres.Sql) { FROM account_identifier ai INNER JOIN accounts_accessed aa ON ai.requesting_zkapp_account_identifier_id = aa.account_identifier_id - INNER JOIN full_chain b ON aa.block_id = b.id + INNER JOIN canonical_chain b ON aa.block_id = b.id )`; } @@ -290,11 +302,12 @@ export function getEventsQuery( toTimestamp?: string, fromTimestamp?: string, after?: string, - limit?: string + limit?: string, + fromBlockHeight?: string ) { return db_client` WITH - ${fullChainCTE(db_client, toTimestamp, fromTimestamp)}, + ${canonicalChainCTE(db_client, toTimestamp, fromTimestamp, fromBlockHeight)}, ${accountIdentifierCTE(db_client, address)}, ${blocksAccessedCTE(db_client)}, ${emittedZkAppCommandsCTE(db_client, after)}, @@ -329,11 +342,12 @@ export function getActionsQuery( toTimestamp?: string, fromTimestamp?: string, after?: string, - limit?: string + limit?: string, + fromBlockHeight?: string ) { return db_client` WITH - ${fullChainCTE(db_client, toTimestamp, fromTimestamp)}, + ${canonicalChainCTE(db_client, toTimestamp, fromTimestamp, fromBlockHeight)}, ${accountIdentifierCTE(db_client, address)}, ${blocksAccessedCTE(db_client)}, ${emittedZkAppCommandsCTE(db_client, after)}, diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts index 3c9ba1a8f..df778c912 100644 --- a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts +++ b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts @@ -197,7 +197,7 @@ export class MinaFunnelCacheEntry implements FunnelCacheEntry { private state: MinaFunnelCacheEntryState | null = null; public static readonly SYMBOL = Symbol('MinaFunnelStartingSlot'); - public updateStartingSlot( + public updateStartingTimestamp( startingSlotTimestamp: number, genesisTime: number, pg: postgres.Sql diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index d278ab60c..b2ea34e92 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -32,14 +32,7 @@ async function findMinaConfirmedTimestamp(pg: postgres.Sql): Promise { return Number.parseInt(row[0]['timestamp'], 10); } -function slotToMinaTimestamp(slot: number, genesisTime: number, slotDuration: number): number { - return slot * slotDuration * 1000 + genesisTime; -} - -function minaTimestampToSlot(ts: number, genesisTime: number, slotDuration: number): number { - return Math.max(Math.floor((ts - genesisTime) / (slotDuration * 1000)), 0); -} - +// mina timestamps are in milliseconds, while evm timestamps are in seconds. function baseChainTimestampToMina( baseChainTimestamp: number, confirmationDepth: number, @@ -226,18 +219,13 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { }) .map(async extension => { if (extension.cdeType === ChainDataExtensionType.MinaEventGeneric) { - let fromTimestamp = slotToMinaTimestamp( - extension.startSlot, - cache.genesisTime, - this.config.slotDuration - ); - const cursor = cursors && cursors[extension.cdeId]; const data = await getEventCdeData({ pg: cache.pg, extension, - fromTimestamp, + fromTimestamp: 0, + fromBlockHeight: extension.startBlockHeight, toTimestamp: startingSlotTimestamp - 1, // the handler for this cde stores the block height unmodified // (even if the event is scheduled at the correct height), so @@ -256,18 +244,13 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { data, }; } else if (extension.cdeType === ChainDataExtensionType.MinaActionGeneric) { - let fromTimestamp = slotToMinaTimestamp( - extension.startSlot, - cache.genesisTime, - this.config.slotDuration - ); - const cursor = cursors && cursors[extension.cdeId]; const data = await getActionCdeData({ pg: cache.pg, extension, - fromTimestamp, + fromTimestamp: 0, + fromBlockHeight: extension.startBlockHeight, toTimestamp: startingSlotTimestamp - 1, getBlockNumber: _x => ENV.SM_START_BLOCKHEIGHT + 1, network: this.chainName, @@ -337,19 +320,13 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) .timestamp as number; - const slot = minaTimestampToSlot( - baseChainTimestampToMina( - startingBlockTimestamp, - config.confirmationDepth, - config.slotDuration - ), - genesisTime, + const minaTimestamp = baseChainTimestampToMina( + startingBlockTimestamp, + config.confirmationDepth, config.slotDuration ); - const slotAsMinaTimestamp = slotToMinaTimestamp(slot, genesisTime, config.slotDuration); - - newEntry.updateStartingSlot(slotAsMinaTimestamp, genesisTime, pg); + newEntry.updateStartingTimestamp(minaTimestamp, genesisTime, pg); const cursors = await getPaginationCursors.run(undefined, dbTx); diff --git a/packages/engine/paima-sm/src/types.ts b/packages/engine/paima-sm/src/types.ts index 133791ddf..ecdf2fb76 100644 --- a/packages/engine/paima-sm/src/types.ts +++ b/packages/engine/paima-sm/src/types.ts @@ -489,7 +489,7 @@ export const ChainDataExtensionMinaEventGenericConfig = Type.Object({ type: Type.Literal(CdeEntryTypeName.MinaEventGeneric), address: Type.String(), scheduledPrefix: Type.String(), - startSlot: Type.Number(), + startBlockHeight: Type.Number(), name: Type.String(), }); @@ -505,7 +505,7 @@ export const ChainDataExtensionMinaActionGenericConfig = Type.Object({ type: Type.Literal(CdeEntryTypeName.MinaActionGeneric), address: Type.String(), scheduledPrefix: Type.String(), - startSlot: Type.Number(), + startBlockHeight: Type.Number(), name: Type.String(), }); From f62b086e15c2450957851072c17cb50ae48b9b33 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 22 Apr 2024 22:47:50 -0300 Subject: [PATCH 25/31] add timestamp checkpointing event --- .../paima-funnel/src/funnels/mina/funnel.ts | 39 +++++++++-- packages/engine/paima-sm/src/index.ts | 10 +++ packages/engine/paima-sm/src/types.ts | 7 +- packages/node-sdk/paima-db/migrations/up.sql | 6 ++ packages/node-sdk/paima-db/src/index.ts | 2 + .../node-sdk/paima-db/src/paima-tables.ts | 20 ++++++ .../src/sql/mina-checkpoints.queries.ts | 64 +++++++++++++++++++ .../paima-db/src/sql/mina-checkpoints.sql | 13 ++++ .../paima-sdk/paima-utils/src/constants.ts | 1 + 9 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts create mode 100644 packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index b2ea34e92..f6e3f5f57 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -1,4 +1,11 @@ -import { doLog, logError, ChainDataExtensionType, delay, ENV } from '@paima/utils'; +import { + doLog, + logError, + ChainDataExtensionType, + delay, + ENV, + InternalEventType, +} from '@paima/utils'; import type { ChainFunnel, ReadPresyncDataFrom } from '@paima/runtime'; import type { ChainData, @@ -11,7 +18,7 @@ import { BaseFunnel } from '../BaseFunnel.js'; import type { FunnelSharedData } from '../BaseFunnel.js'; import type { PoolClient } from 'pg'; import { FUNNEL_PRESYNC_FINISHED, ConfigNetworkType } from '@paima/utils'; -import { getPaginationCursors } from '@paima/db'; +import { getMinaCheckpoint, getPaginationCursors } from '@paima/db'; import { getActionCdeData, getEventCdeData } from '../../cde/minaGeneric.js'; import type { MinaConfig } from '@paima/utils'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; @@ -86,8 +93,10 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { await delay(delayForWaitingForFinalityLoop); } - const fromTimestamp = - this.cache.getState().lastPoint?.timestamp || cachedState.startingSlotTimestamp; + const lastRoundTimestamp = this.cache.getState().lastPoint?.timestamp; + const fromTimestamp = lastRoundTimestamp + ? lastRoundTimestamp + 1 + : cachedState.startingSlotTimestamp; const toTimestamp = maxBaseTimestamp; @@ -159,6 +168,23 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const composed = composeChainData(baseData, grouped); + for (const chainData of composed) { + if (!chainData.internalEvents) { + chainData.internalEvents = []; + } + + chainData.internalEvents.push({ + type: InternalEventType.MinaLastTimestamp, + + timestamp: baseChainTimestampToMina( + chainData.timestamp, + this.config.confirmationDepth, + this.config.slotDuration + ).toString(), + network: this.chainName, + }); + } + return composed; } @@ -347,6 +373,11 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { }); }); + const checkpoint = await getMinaCheckpoint.run({ network: chainName }, dbTx); + if (checkpoint.length > 0) { + newEntry.updateLastPoint(Number.parseInt(checkpoint[0].timestamp, 10)); + } + return newEntry; })(); diff --git a/packages/engine/paima-sm/src/index.ts b/packages/engine/paima-sm/src/index.ts index bd4d143d5..c34b2df21 100644 --- a/packages/engine/paima-sm/src/index.ts +++ b/packages/engine/paima-sm/src/index.ts @@ -29,6 +29,7 @@ import { NO_USER_ID, updateCardanoEpoch, updatePaginationCursor, + updateMinaCheckpoint, } from '@paima/db'; import Prando from '@paima/prando'; @@ -488,6 +489,15 @@ async function processInternalEvents( dbTx ); break; + case InternalEventType.MinaLastTimestamp: + await updateMinaCheckpoint.run( + { + timestamp: event.timestamp, + network: event.network, + }, + dbTx + ); + break; default: assertNever(event); } diff --git a/packages/engine/paima-sm/src/types.ts b/packages/engine/paima-sm/src/types.ts index ecdf2fb76..b8f705c91 100644 --- a/packages/engine/paima-sm/src/types.ts +++ b/packages/engine/paima-sm/src/types.ts @@ -35,13 +35,18 @@ export interface ChainData { internalEvents?: InternalEvent[]; } -export type InternalEvent = CardanoEpochEvent | EvmLastBlockEvent; +export type InternalEvent = CardanoEpochEvent | EvmLastBlockEvent | MinaLastTimestampEvent; export type CardanoEpochEvent = { type: InternalEventType.CardanoBestEpoch; epoch: number }; export type EvmLastBlockEvent = { type: InternalEventType.EvmLastBlock; block: number; network: string; }; +export type MinaLastTimestampEvent = { + type: InternalEventType.MinaLastTimestamp; + timestamp: string; + network: string; +}; export interface EvmPresyncChainData { network: string; diff --git a/packages/node-sdk/paima-db/migrations/up.sql b/packages/node-sdk/paima-db/migrations/up.sql index ebee57cc2..f1eb0dd3d 100644 --- a/packages/node-sdk/paima-db/migrations/up.sql +++ b/packages/node-sdk/paima-db/migrations/up.sql @@ -241,3 +241,9 @@ CREATE TABLE cde_cardano_mint_burn( output_addresses JSONB NOT NULL, PRIMARY KEY (cde_id, tx_id) ); + +CREATE TABLE mina_checkpoint ( + timestamp TEXT NOT NULL, + network TEXT NOT NULL, + PRIMARY KEY (network) +); diff --git a/packages/node-sdk/paima-db/src/index.ts b/packages/node-sdk/paima-db/src/index.ts index c398c26f6..e2f1f3cf5 100644 --- a/packages/node-sdk/paima-db/src/index.ts +++ b/packages/node-sdk/paima-db/src/index.ts @@ -53,6 +53,8 @@ export type * from './sql/cde-cursor-tracking-pagination.queries.js'; export * from './sql/cde-cardano-transfer.queries.js'; export type * from './sql/cde-cardano-transfer.queries.js'; export { cdeCardanoMintBurnInsert } from './sql/cde-cardano-mint-burn.queries.js'; +export type * from './sql/mina-checkpoints.queries.js'; +export * from './sql/mina-checkpoints.queries.js'; export { tx, diff --git a/packages/node-sdk/paima-db/src/paima-tables.ts b/packages/node-sdk/paima-db/src/paima-tables.ts index 52161aa3b..9b197d1c4 100644 --- a/packages/node-sdk/paima-db/src/paima-tables.ts +++ b/packages/node-sdk/paima-db/src/paima-tables.ts @@ -576,6 +576,25 @@ const TABLE_DATA_DELEGATIONS: TableData = { creationQuery: QUERY_CREATE_TABLE_DELEGATIONS, }; +const QUERY_CREATE_TABLE_MINA_CHECKPOINT = ` +CREATE TABLE mina_checkpoint ( + timestamp TEXT NOT NULL, + network TEXT NOT NULL, + PRIMARY KEY (network) +); +`; + +const TABLE_DATA_MINA_CHECKPOINT: TableData = { + tableName: 'mina_checkpoint', + primaryKeyColumns: ['network'], + columnData: packTuples([ + ['timestamp', 'text', 'NO', ''], + ['network', 'text', 'NO', ''], + ]), + serialColumns: [], + creationQuery: QUERY_CREATE_TABLE_MINA_CHECKPOINT, +}; + const FUNCTION_NOTIFY_WALLET_CONNECT: string = ` create or replace function public.notify_wallet_connect() returns trigger @@ -663,4 +682,5 @@ export const TABLES: TableData[] = [ TABLE_DATA_CDE_TRACKING_CURSOR_PAGINATION, TABLE_DATA_CDE_CARDANO_TRANSFER, TABLE_DATA_CDE_CARDANO_MINT_BURN, + TABLE_DATA_MINA_CHECKPOINT, ]; diff --git a/packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts new file mode 100644 index 000000000..0c61c3d68 --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.queries.ts @@ -0,0 +1,64 @@ +/** Types generated for queries found in "src/sql/mina-checkpoints.sql" */ +import { PreparedQuery } from '@pgtyped/runtime'; + +/** 'UpdateMinaCheckpoint' parameters type */ +export interface IUpdateMinaCheckpointParams { + network: string; + timestamp: string; +} + +/** 'UpdateMinaCheckpoint' return type */ +export type IUpdateMinaCheckpointResult = void; + +/** 'UpdateMinaCheckpoint' query type */ +export interface IUpdateMinaCheckpointQuery { + params: IUpdateMinaCheckpointParams; + result: IUpdateMinaCheckpointResult; +} + +const updateMinaCheckpointIR: any = {"usedParamSet":{"timestamp":true,"network":true},"params":[{"name":"timestamp","required":true,"transform":{"type":"scalar"},"locs":[{"a":71,"b":81},{"a":149,"b":159}]},{"name":"network","required":true,"transform":{"type":"scalar"},"locs":[{"a":88,"b":96}]}],"statement":"INSERT INTO mina_checkpoint(\n timestamp,\n network\n) VALUES (\n :timestamp!,\n :network!\n) \nON CONFLICT (network) DO\nUPDATE SET timestamp = :timestamp!"}; + +/** + * Query generated from SQL: + * ``` + * INSERT INTO mina_checkpoint( + * timestamp, + * network + * ) VALUES ( + * :timestamp!, + * :network! + * ) + * ON CONFLICT (network) DO + * UPDATE SET timestamp = :timestamp! + * ``` + */ +export const updateMinaCheckpoint = new PreparedQuery(updateMinaCheckpointIR); + + +/** 'GetMinaCheckpoint' parameters type */ +export interface IGetMinaCheckpointParams { + network: string; +} + +/** 'GetMinaCheckpoint' return type */ +export interface IGetMinaCheckpointResult { + timestamp: string; +} + +/** 'GetMinaCheckpoint' query type */ +export interface IGetMinaCheckpointQuery { + params: IGetMinaCheckpointParams; + result: IGetMinaCheckpointResult; +} + +const getMinaCheckpointIR: any = {"usedParamSet":{"network":true},"params":[{"name":"network","required":true,"transform":{"type":"scalar"},"locs":[{"a":54,"b":62}]}],"statement":"SELECT timestamp FROM mina_checkpoint WHERE network = :network! LIMIT 1"}; + +/** + * Query generated from SQL: + * ``` + * SELECT timestamp FROM mina_checkpoint WHERE network = :network! LIMIT 1 + * ``` + */ +export const getMinaCheckpoint = new PreparedQuery(getMinaCheckpointIR); + + diff --git a/packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql new file mode 100644 index 000000000..da8f6e9e1 --- /dev/null +++ b/packages/node-sdk/paima-db/src/sql/mina-checkpoints.sql @@ -0,0 +1,13 @@ +/* @name updateMinaCheckpoint */ +INSERT INTO mina_checkpoint( + timestamp, + network +) VALUES ( + :timestamp!, + :network! +) +ON CONFLICT (network) DO +UPDATE SET timestamp = :timestamp!; + +/* @name getMinaCheckpoint */ +SELECT timestamp FROM mina_checkpoint WHERE network = :network! LIMIT 1; diff --git a/packages/paima-sdk/paima-utils/src/constants.ts b/packages/paima-sdk/paima-utils/src/constants.ts index 90ac1df2c..0751ed41e 100644 --- a/packages/paima-sdk/paima-utils/src/constants.ts +++ b/packages/paima-sdk/paima-utils/src/constants.ts @@ -50,4 +50,5 @@ export const FUNNEL_PRESYNC_FINISHED = 'finished'; export const enum InternalEventType { CardanoBestEpoch, EvmLastBlock, + MinaLastTimestamp, } From 88acf1c3681243b661602530969a07d9d8b11138 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:07:44 -0300 Subject: [PATCH 26/31] genesisTime field is not used anymore --- .../engine/paima-funnel/src/funnels/FunnelCache.ts | 8 +------- .../engine/paima-funnel/src/funnels/mina/funnel.ts | 10 +--------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts index df778c912..839652c06 100644 --- a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts +++ b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts @@ -184,7 +184,6 @@ export class EvmFunnelCacheEntry implements FunnelCacheEntry { export type MinaFunnelCacheEntryState = { startingSlotTimestamp: number; lastPoint: { timestamp: number } | undefined; - genesisTime: number; pg: postgres.Sql; cursors: | { @@ -197,14 +196,9 @@ export class MinaFunnelCacheEntry implements FunnelCacheEntry { private state: MinaFunnelCacheEntryState | null = null; public static readonly SYMBOL = Symbol('MinaFunnelStartingSlot'); - public updateStartingTimestamp( - startingSlotTimestamp: number, - genesisTime: number, - pg: postgres.Sql - ): void { + public updateStartingTimestamp(startingSlotTimestamp: number, pg: postgres.Sql): void { this.state = { startingSlotTimestamp, - genesisTime, lastPoint: this.state?.lastPoint, cursors: this.state?.cursors, pg, diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index f6e3f5f57..7ee71e3d1 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -26,12 +26,6 @@ import postgres from 'postgres'; const delayForWaitingForFinalityLoop = 1000; -async function getGenesisTime(pg: postgres.Sql): Promise { - const row = await pg`select timestamp from blocks where height = 1;`; - - return Number.parseInt(row[0]['timestamp'], 10); -} - async function findMinaConfirmedTimestamp(pg: postgres.Sql): Promise { const row = await pg`select timestamp from blocks where chain_status = 'canonical' order by height desc limit 1;`; @@ -341,8 +335,6 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const pg = postgres(config.archiveConnectionString); - const genesisTime = await getGenesisTime(pg); - const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) .timestamp as number; @@ -352,7 +344,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { config.slotDuration ); - newEntry.updateStartingTimestamp(minaTimestamp, genesisTime, pg); + newEntry.updateStartingTimestamp(minaTimestamp, pg); const cursors = await getPaginationCursors.run(undefined, dbTx); From 3ef89e84be3c7ea7a65ac44df58f464e193eab44 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Sat, 27 Apr 2024 13:36:23 -0300 Subject: [PATCH 27/31] fix missing cache update of the lower bound --- packages/engine/paima-funnel/src/funnels/mina/funnel.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 7ee71e3d1..7583c178f 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -162,6 +162,8 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const composed = composeChainData(baseData, grouped); + this.cache.updateLastPoint(toTimestamp); + for (const chainData of composed) { if (!chainData.internalEvents) { chainData.internalEvents = []; From 24546d0bc2c1ef62f2323d8df9bba570411f5ee0 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Mon, 29 Apr 2024 01:14:07 -0300 Subject: [PATCH 28/31] allow overriding archive's confirmation parameters --- .../paima-funnel/src/cde/minaGeneric.ts | 2 +- .../paima-funnel/src/funnels/mina/funnel.ts | 50 +++++++++++++------ .../paima-utils/src/config/loading.ts | 7 ++- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 5488cb973..149cd32ff 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -170,7 +170,7 @@ function canonicalChainCTE( FROM blocks b WHERE - chain_status = 'canonical' + 1=1 ${fromTimestamp ? db_client`AND b.timestamp::decimal >= ${fromTimestamp}::decimal` : db_client``} ${toTimestamp ? db_client`AND b.timestamp::decimal <= ${toTimestamp}::decimal` : db_client``} ${fromBlockHeight ? db_client`AND b.height::decimal >= ${fromBlockHeight}::decimal` : db_client``} diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 7583c178f..855c85213 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -26,20 +26,39 @@ import postgres from 'postgres'; const delayForWaitingForFinalityLoop = 1000; -async function findMinaConfirmedTimestamp(pg: postgres.Sql): Promise { - const row = - await pg`select timestamp from blocks where chain_status = 'canonical' order by height desc limit 1;`; +async function findMinaConfirmedTimestamp( + pg: postgres.Sql, + confirmationDepth?: number, +): Promise { + let row; + if (confirmationDepth) { + row = await pg` + WITH RECURSIVE chain AS ( + (SELECT parent_id, id, timestamp, height FROM blocks b WHERE height = (select MAX(height) from blocks) + ORDER BY timestamp ASC + LIMIT 1) + UNION ALL + SELECT b.parent_id, b.id, b.timestamp, b.height FROM blocks b + INNER JOIN chain + ON b.id = chain.parent_id AND chain.id <> chain.parent_id + ) SELECT timestamp FROM chain c + LIMIT 1 + OFFSET ${confirmationDepth}; + `; + } else { + row = + await pg`select timestamp from blocks where chain_status = 'canonical' order by height desc limit 1;`; + } - return Number.parseInt(row[0]['timestamp'], 10); + return Number.parseInt(row[0]["timestamp"], 10); } // mina timestamps are in milliseconds, while evm timestamps are in seconds. function baseChainTimestampToMina( baseChainTimestamp: number, - confirmationDepth: number, - slotDuration: number + delay: number, ): number { - return Math.max(baseChainTimestamp * 1000 - slotDuration * 1000 * confirmationDepth, 0); + return Math.max((baseChainTimestamp - delay) * 1000, 0); } export class MinaFunnel extends BaseFunnel implements ChainFunnel { @@ -73,12 +92,14 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const maxBaseTimestamp = baseChainTimestampToMina( baseData[baseData.length - 1].timestamp, - this.config.confirmationDepth, - this.config.slotDuration + this.config.delay, ); while (true) { - const confirmedTimestamp = await findMinaConfirmedTimestamp(cachedState.pg); + const confirmedTimestamp = await findMinaConfirmedTimestamp( + cachedState.pg, + this.config.confirmationDepth, + ); if (confirmedTimestamp >= maxBaseTimestamp) { break; @@ -99,8 +120,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { state.curr < baseData.length && baseChainTimestampToMina( baseData[state.curr].timestamp, - this.config.confirmationDepth, - this.config.slotDuration + this.config.delay, ) <= ts ) state.curr++; @@ -174,8 +194,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { timestamp: baseChainTimestampToMina( chainData.timestamp, - this.config.confirmationDepth, - this.config.slotDuration + this.config.delay, ).toString(), network: this.chainName, }); @@ -342,8 +361,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const minaTimestamp = baseChainTimestampToMina( startingBlockTimestamp, - config.confirmationDepth, - config.slotDuration + config.delay, ); newEntry.updateStartingTimestamp(minaTimestamp, pg); diff --git a/packages/paima-sdk/paima-utils/src/config/loading.ts b/packages/paima-sdk/paima-utils/src/config/loading.ts index 1a42c5198..18e68611a 100644 --- a/packages/paima-sdk/paima-utils/src/config/loading.ts +++ b/packages/paima-sdk/paima-utils/src/config/loading.ts @@ -63,10 +63,9 @@ export type CardanoConfig = Static; export const MinaConfigSchema = Type.Object({ archiveConnectionString: Type.String(), - // k - confirmationDepth: Type.Number(), + delay: Type.Number(), paginationLimit: Type.Number({ default: 50 }), - slotDuration: Type.Number(), + confirmationDepth: Type.Optional(Type.Number()), }); export type MinaConfig = Static; @@ -135,7 +134,7 @@ const cardanoConfigDefaults = { const minaConfigDefaults = { // lightnet defaults confirmationDepth: 30, - slotDuration: 20, + delay: 30 * 40, }; // used as a placeholder name for the ENV fallback mechanism From 27dbb08a290a28e4cee5b4155090cd20cad81e0c Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Tue, 30 Apr 2024 13:35:58 -0300 Subject: [PATCH 29/31] migrated from postgres to pg --- packages/engine/paima-funnel/package.json | 2 +- .../paima-funnel/src/cde/minaGeneric.ts | 98 ++++++++++--------- .../paima-funnel/src/funnels/FunnelCache.ts | 6 +- .../paima-funnel/src/funnels/mina/funnel.ts | 59 +++++------ packages/engine/paima-funnel/src/reading.ts | 2 + 5 files changed, 84 insertions(+), 83 deletions(-) diff --git a/packages/engine/paima-funnel/package.json b/packages/engine/paima-funnel/package.json index a9d1dad5c..f59aa0d18 100644 --- a/packages/engine/paima-funnel/package.json +++ b/packages/engine/paima-funnel/package.json @@ -20,6 +20,6 @@ "assert-never": "^1.2.1", "@dcspark/carp-client": "^3.1.0", "@dcspark/cardano-multiplatform-lib-nodejs": "5.2.0", - "postgres": "^3.3.5" + "pg": "^8.11.3" } } diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 149cd32ff..17f924e40 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -5,10 +5,10 @@ import type { ChainDataExtensionMinaEventGeneric, } from '@paima/sm'; import { ChainDataExtensionDatumType } from '@paima/utils'; -import postgres from 'postgres'; +import pg from 'pg'; export async function getEventCdeData(args: { - pg: postgres.Sql; + pg: pg.Client; extension: ChainDataExtensionMinaEventGeneric; fromTimestamp: number; toTimestamp: number; @@ -36,7 +36,7 @@ export async function getEventCdeData(args: { } export async function getActionCdeData(args: { - pg: postgres.Sql; + pg: pg.Client; extension: ChainDataExtensionMinaActionGeneric; fromTimestamp: number; toTimestamp: number; @@ -68,7 +68,7 @@ export async function getCdeData( cdeDatumType: | ChainDataExtensionDatumType.MinaActionGeneric | ChainDataExtensionDatumType.MinaEventGeneric, - pg: postgres.Sql, + pg: pg.Client, extension: ChainDataExtensionMinaEventGeneric | ChainDataExtensionMinaActionGeneric, fromTimestamp: number, toTimestamp: number, @@ -81,7 +81,10 @@ export async function getCdeData( ): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { const result = [] as (CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]; + console.log('fromTo', fromTimestamp, toTimestamp); + while (true) { + console.log('making query'); const unmapped = await query( pg, extension.address, @@ -92,7 +95,9 @@ export async function getCdeData( fromBlockHeight?.toString() ); - const grouped = groupByTx(unmapped); + console.log('unmapped', console.log(unmapped)); + + const grouped = groupByTx(unmapped.rows); const events = grouped.flatMap(perBlock => perBlock.eventsData.map(txEvent => ({ @@ -124,7 +129,7 @@ export async function getCdeData( return result; } -function groupByTx(events: postgres.RowList) { +function groupByTx(events: PerBlock[]) { const grouped = [] as { blockInfo: { height: number; @@ -157,13 +162,8 @@ function groupByTx(events: postgres.RowList) { return grouped; } -function canonicalChainCTE( - db_client: postgres.Sql, - toTimestamp?: string, - fromTimestamp?: string, - fromBlockHeight?: string -) { - return db_client` +function canonicalChainCTE(toTimestamp?: string, fromTimestamp?: string, fromBlockHeight?: string) { + return ` canonical_chain AS ( SELECT id, state_hash, height, global_slot_since_genesis, timestamp @@ -171,28 +171,28 @@ function canonicalChainCTE( blocks b WHERE 1=1 - ${fromTimestamp ? db_client`AND b.timestamp::decimal >= ${fromTimestamp}::decimal` : db_client``} - ${toTimestamp ? db_client`AND b.timestamp::decimal <= ${toTimestamp}::decimal` : db_client``} - ${fromBlockHeight ? db_client`AND b.height::decimal >= ${fromBlockHeight}::decimal` : db_client``} + ${fromTimestamp ? `AND b.timestamp::decimal >= ${fromTimestamp}::decimal` : ``} + ${toTimestamp ? `AND b.timestamp::decimal <= ${toTimestamp}::decimal` : ``} + ${fromBlockHeight ? `AND b.height::decimal >= ${fromBlockHeight}::decimal` : ``} ORDER BY height ) `; } -function accountIdentifierCTE(db_client: postgres.Sql, address: string) { - return db_client` +function accountIdentifierCTE(address: string) { + return ` account_identifier AS ( SELECT id AS requesting_zkapp_account_identifier_id FROM account_identifiers ai WHERE - ai.public_key_id = (SELECT id FROM public_keys WHERE value = ${address}) + ai.public_key_id = (SELECT id FROM public_keys WHERE value = '${address}') )`; } -function blocksAccessedCTE(db_client: postgres.Sql) { - return db_client` +function blocksAccessedCTE() { + return ` blocks_accessed AS ( SELECT @@ -212,8 +212,8 @@ function blocksAccessedCTE(db_client: postgres.Sql) { )`; } -function emittedZkAppCommandsCTE(db_client: postgres.Sql, after?: string) { - return db_client` +function emittedZkAppCommandsCTE(after?: string) { + return ` emitted_zkapp_commands AS ( SELECT blocks_accessed.*, @@ -234,14 +234,14 @@ function emittedZkAppCommandsCTE(db_client: postgres.Sql, after?: string) { INNER JOIN zkapp_account_update zkcu ON zkcu.id = ANY(zkc.zkapp_account_updates_ids) INNER JOIN zkapp_account_update_body zkcu_body ON zkcu_body.id = zkcu.body_id AND zkcu_body.account_identifier_id = requesting_zkapp_account_identifier_id - ${after ? db_client`AND zkc.id > (SELECT id FROM zkapp_commands WHERE zkapp_commands.hash = ${after})` : db_client``} + ${after ? `AND zkc.id > (SELECT id FROM zkapp_commands WHERE zkapp_commands.hash = '${after}')` : ``} WHERE bzkc.status <> 'failed' )`; } -function emittedEventsCTE(db_client: postgres.Sql) { - return db_client` +function emittedEventsCTE() { + return ` emitted_events AS ( SELECT *, @@ -257,8 +257,8 @@ function emittedEventsCTE(db_client: postgres.Sql) { `; } -function emittedActionsCTE(db_client: postgres.Sql) { - return db_client` +function emittedActionsCTE() { + return ` emitted_actions AS ( SELECT *, @@ -274,8 +274,8 @@ function emittedActionsCTE(db_client: postgres.Sql) { `; } -function emittedActionStateCTE(db_client: postgres.Sql) { - return db_client` +function emittedActionStateCTE() { + return ` emitted_action_state AS ( SELECT emitted_actions.* @@ -297,7 +297,7 @@ type PerBlock = { }; export function getEventsQuery( - db_client: postgres.Sql, + db_client: pg.Client, address: string, toTimestamp?: string, fromTimestamp?: string, @@ -305,13 +305,13 @@ export function getEventsQuery( limit?: string, fromBlockHeight?: string ) { - return db_client` + let query = ` WITH - ${canonicalChainCTE(db_client, toTimestamp, fromTimestamp, fromBlockHeight)}, - ${accountIdentifierCTE(db_client, address)}, - ${blocksAccessedCTE(db_client)}, - ${emittedZkAppCommandsCTE(db_client, after)}, - ${emittedEventsCTE(db_client)}, + ${canonicalChainCTE(toTimestamp, fromTimestamp, fromBlockHeight)}, + ${accountIdentifierCTE(address)}, + ${blocksAccessedCTE()}, + ${emittedZkAppCommandsCTE(after)}, + ${emittedEventsCTE()}, grouped_events AS ( SELECT MAX(timestamp) timestamp, @@ -332,12 +332,14 @@ export function getEventsQuery( FROM grouped_events GROUP BY height ORDER BY height - ${limit ? db_client`LIMIT ${limit}` : db_client``} + ${limit ? `LIMIT ${limit}` : ``} `; + + return db_client.query(query); } export function getActionsQuery( - db_client: postgres.Sql, + db_client: pg.Client, address: string, toTimestamp?: string, fromTimestamp?: string, @@ -345,14 +347,14 @@ export function getActionsQuery( limit?: string, fromBlockHeight?: string ) { - return db_client` + const query = ` WITH - ${canonicalChainCTE(db_client, toTimestamp, fromTimestamp, fromBlockHeight)}, - ${accountIdentifierCTE(db_client, address)}, - ${blocksAccessedCTE(db_client)}, - ${emittedZkAppCommandsCTE(db_client, after)}, - ${emittedActionsCTE(db_client)}, - ${emittedActionStateCTE(db_client)}, + ${canonicalChainCTE(toTimestamp, fromTimestamp, fromBlockHeight)}, + ${accountIdentifierCTE(address)}, + ${blocksAccessedCTE()}, + ${emittedZkAppCommandsCTE(after)}, + ${emittedActionsCTE()}, + ${emittedActionStateCTE()}, grouped_events AS ( SELECT MAX(timestamp) timestamp, @@ -373,6 +375,8 @@ export function getActionsQuery( FROM grouped_events GROUP BY height ORDER BY height - ${limit ? db_client`LIMIT ${limit}` : db_client``} + ${limit ? `LIMIT ${limit}` : ``} `; + + return db_client.query(query); } diff --git a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts index 839652c06..f8e0effcd 100644 --- a/packages/engine/paima-funnel/src/funnels/FunnelCache.ts +++ b/packages/engine/paima-funnel/src/funnels/FunnelCache.ts @@ -1,5 +1,5 @@ import type { ChainData } from '@paima/sm'; -import postgres from 'postgres'; +import pg from 'pg'; export interface FunnelCacheEntry { /** @@ -184,7 +184,7 @@ export class EvmFunnelCacheEntry implements FunnelCacheEntry { export type MinaFunnelCacheEntryState = { startingSlotTimestamp: number; lastPoint: { timestamp: number } | undefined; - pg: postgres.Sql; + pg: pg.Client; cursors: | { [cdeId: number]: { cursor: string; finished: boolean }; @@ -196,7 +196,7 @@ export class MinaFunnelCacheEntry implements FunnelCacheEntry { private state: MinaFunnelCacheEntryState | null = null; public static readonly SYMBOL = Symbol('MinaFunnelStartingSlot'); - public updateStartingTimestamp(startingSlotTimestamp: number, pg: postgres.Sql): void { + public updateStartingTimestamp(startingSlotTimestamp: number, pg: pg.Client): void { this.state = { startingSlotTimestamp, lastPoint: this.state?.lastPoint, diff --git a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts index 855c85213..845acfc4e 100644 --- a/packages/engine/paima-funnel/src/funnels/mina/funnel.ts +++ b/packages/engine/paima-funnel/src/funnels/mina/funnel.ts @@ -22,17 +22,19 @@ import { getMinaCheckpoint, getPaginationCursors } from '@paima/db'; import { getActionCdeData, getEventCdeData } from '../../cde/minaGeneric.js'; import type { MinaConfig } from '@paima/utils'; import { MinaFunnelCacheEntry } from '../FunnelCache.js'; -import postgres from 'postgres'; +import pg from 'pg'; +const { Client } = pg; const delayForWaitingForFinalityLoop = 1000; async function findMinaConfirmedTimestamp( - pg: postgres.Sql, - confirmationDepth?: number, + db: pg.Client, + confirmationDepth?: number ): Promise { let row; if (confirmationDepth) { - row = await pg` + row = ( + await db.query(` WITH RECURSIVE chain AS ( (SELECT parent_id, id, timestamp, height FROM blocks b WHERE height = (select MAX(height) from blocks) ORDER BY timestamp ASC @@ -44,20 +46,21 @@ async function findMinaConfirmedTimestamp( ) SELECT timestamp FROM chain c LIMIT 1 OFFSET ${confirmationDepth}; - `; + `) + ).rows; } else { - row = - await pg`select timestamp from blocks where chain_status = 'canonical' order by height desc limit 1;`; + const res = await db.query( + `select timestamp from blocks where chain_status = 'canonical' order by height desc limit 1;` + ); + + row = res.rows; } - return Number.parseInt(row[0]["timestamp"], 10); + return Number.parseInt(row[0]['timestamp'], 10); } // mina timestamps are in milliseconds, while evm timestamps are in seconds. -function baseChainTimestampToMina( - baseChainTimestamp: number, - delay: number, -): number { +function baseChainTimestampToMina(baseChainTimestamp: number, delay: number): number { return Math.max((baseChainTimestamp - delay) * 1000, 0); } @@ -92,13 +95,13 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const maxBaseTimestamp = baseChainTimestampToMina( baseData[baseData.length - 1].timestamp, - this.config.delay, + this.config.delay ); while (true) { const confirmedTimestamp = await findMinaConfirmedTimestamp( cachedState.pg, - this.config.confirmationDepth, + this.config.confirmationDepth ); if (confirmedTimestamp >= maxBaseTimestamp) { @@ -118,10 +121,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const getBlockNumber = (state: { curr: number }) => (ts: number) => { while ( state.curr < baseData.length && - baseChainTimestampToMina( - baseData[state.curr].timestamp, - this.config.delay, - ) <= ts + baseChainTimestampToMina(baseData[state.curr].timestamp, this.config.delay) <= ts ) state.curr++; @@ -192,10 +192,7 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { chainData.internalEvents.push({ type: InternalEventType.MinaLastTimestamp, - timestamp: baseChainTimestampToMina( - chainData.timestamp, - this.config.delay, - ).toString(), + timestamp: baseChainTimestampToMina(chainData.timestamp, this.config.delay).toString(), network: this.chainName, }); } @@ -203,9 +200,9 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { return composed; } - public override async readPresyncData( - args: ReadPresyncDataFrom - ): Promise<{ [network: string]: PresyncChainData[] | typeof FUNNEL_PRESYNC_FINISHED }> { + public override async readPresyncData(args: ReadPresyncDataFrom): Promise<{ + [network: string]: PresyncChainData[] | typeof FUNNEL_PRESYNC_FINISHED; + }> { const basePromise = this.baseFunnel.readPresyncData(args); const cache = this.cache.getState(); @@ -354,15 +351,13 @@ export class MinaFunnel extends BaseFunnel implements ChainFunnel { const newEntry = new MinaFunnelCacheEntry(); sharedData.cacheManager.cacheEntries[MinaFunnelCacheEntry.SYMBOL] = newEntry; - const pg = postgres(config.archiveConnectionString); + const pg = new Client({ connectionString: config.archiveConnectionString }); + await pg.connect(); - const startingBlockTimestamp = (await sharedData.web3.eth.getBlock(startingBlockHeight)) - .timestamp as number; + const startingBlock = await sharedData.web3.eth.getBlock(startingBlockHeight); + const startingBlockTimestamp = startingBlock.timestamp as number; - const minaTimestamp = baseChainTimestampToMina( - startingBlockTimestamp, - config.delay, - ); + const minaTimestamp = baseChainTimestampToMina(startingBlockTimestamp, config.delay); newEntry.updateStartingTimestamp(minaTimestamp, pg); diff --git a/packages/engine/paima-funnel/src/reading.ts b/packages/engine/paima-funnel/src/reading.ts index 1e2b4bebf..7f8abf78e 100644 --- a/packages/engine/paima-funnel/src/reading.ts +++ b/packages/engine/paima-funnel/src/reading.ts @@ -80,6 +80,8 @@ export async function getMultipleBlockData( ): Promise { const batch = new web3.BatchRequest(); + // console.log('getting multiple block data', fromBlock, toBlock); + const blockRange = Array.from({ length: toBlock - fromBlock + 1 }, (_, i) => i + fromBlock); const blockPromises = blockRange.map(blockNumber => { return new Promise((resolve, reject) => { From 26e543ed455b89cf963a3e27de3071e97dbfbecd Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Tue, 30 Apr 2024 13:49:06 -0300 Subject: [PATCH 30/31] remove debugging logs --- packages/engine/paima-funnel/src/cde/minaGeneric.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 17f924e40..51c55bede 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -81,10 +81,7 @@ export async function getCdeData( ): Promise<(CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]> { const result = [] as (CdeMinaActionGenericDatum | CdeMinaEventGenericDatum)[]; - console.log('fromTo', fromTimestamp, toTimestamp); - while (true) { - console.log('making query'); const unmapped = await query( pg, extension.address, @@ -95,8 +92,6 @@ export async function getCdeData( fromBlockHeight?.toString() ); - console.log('unmapped', console.log(unmapped)); - const grouped = groupByTx(unmapped.rows); const events = grouped.flatMap(perBlock => From 9c65d8647c6160978002e894c70419007b460bb1 Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini <48031343+ecioppettini@users.noreply.github.com> Date: Thu, 2 May 2024 23:59:47 -0300 Subject: [PATCH 31/31] cleanup --- packages/engine/paima-funnel/src/cde/minaGeneric.ts | 1 - packages/engine/paima-funnel/src/reading.ts | 2 -- packages/engine/paima-sm/src/types.ts | 1 - 3 files changed, 4 deletions(-) diff --git a/packages/engine/paima-funnel/src/cde/minaGeneric.ts b/packages/engine/paima-funnel/src/cde/minaGeneric.ts index 51c55bede..b78647197 100644 --- a/packages/engine/paima-funnel/src/cde/minaGeneric.ts +++ b/packages/engine/paima-funnel/src/cde/minaGeneric.ts @@ -130,7 +130,6 @@ function groupByTx(events: PerBlock[]) { height: number; timestamp: string; }; - // TODO: could each data by just a tuple? eventsData: { data: string[][]; txHash: string }[]; }[]; diff --git a/packages/engine/paima-funnel/src/reading.ts b/packages/engine/paima-funnel/src/reading.ts index 7f8abf78e..1e2b4bebf 100644 --- a/packages/engine/paima-funnel/src/reading.ts +++ b/packages/engine/paima-funnel/src/reading.ts @@ -80,8 +80,6 @@ export async function getMultipleBlockData( ): Promise { const batch = new web3.BatchRequest(); - // console.log('getting multiple block data', fromBlock, toBlock); - const blockRange = Array.from({ length: toBlock - fromBlock + 1 }, (_, i) => i + fromBlock); const blockPromises = blockRange.map(blockNumber => { return new Promise((resolve, reject) => { diff --git a/packages/engine/paima-sm/src/types.ts b/packages/engine/paima-sm/src/types.ts index b8f705c91..01a0234cb 100644 --- a/packages/engine/paima-sm/src/types.ts +++ b/packages/engine/paima-sm/src/types.ts @@ -62,7 +62,6 @@ export interface CardanoPresyncChainData { extensionDatums: ChainDataExtensionDatum[]; } -// TODO: potentially this and Cardano can be collapsed export interface MinaPresyncChainData { network: string; networkType: ConfigNetworkType.MINA;