From af91567d6c835cf25096aaf519e1a1369866fe0d Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 25 Nov 2024 13:47:25 +1300 Subject: [PATCH 1/3] Improve event iteration performance --- packages/node/src/indexer/indexer.manager.ts | 32 ++++++++++++++--- packages/node/src/utils/substrate.ts | 38 ++++++++++---------- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index 1401c339ad..fa6f7a2d40 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -22,6 +22,7 @@ import { ProcessBlockResponse, BaseIndexerManager, IBlock, + getLogger, } from '@subql/node-core'; import { LightSubstrateEvent, @@ -39,6 +40,8 @@ import { DynamicDsService } from './dynamic-ds.service'; import { ApiAt, BlockContent, isFullBlock, LightBlockContent } from './types'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; +const logger = getLogger('indexer'); + @Injectable() export class IndexerManager extends BaseIndexerManager< ApiPromise, @@ -105,9 +108,31 @@ export class IndexerManager extends BaseIndexerManager< const { block, events, extrinsics } = blockContent; await this.indexBlockContent(block, dataSources, getVM); + // Group the events so they only need to be iterated over a single time + const groupedEvents = events.reduce( + (acc, evt, idx) => { + if (evt.phase.isInitialization) { + acc.init ??= []; + acc.init.push(evt); + } else if (evt.phase.isFinalization) { + acc.finalize ??= []; + acc.finalize.push(evt); + } else if (evt.extrinsic?.idx) { + const idx = evt.extrinsic.idx; + acc[idx] = acc[idx] || []; + acc[idx].push(evt); + } else { + logger.warn( + `Unrecognized event type, skipping. block="${block.block.header.number.toNumber()}" eventIdx="${idx}"`, + ); + } + return acc; + }, + {} as Record, + ); + // Run initialization events - const initEvents = events.filter((evt) => evt.phase.isInitialization); - for (const event of initEvents) { + for (const event of groupedEvents.init ?? []) { await this.indexEvent(event, dataSources, getVM); } @@ -125,8 +150,7 @@ export class IndexerManager extends BaseIndexerManager< } // Run finalization events - const finalizeEvents = events.filter((evt) => evt.phase.isFinalization); - for (const event of finalizeEvents) { + for (const event of groupedEvents.finalize ?? []) { await this.indexEvent(event, dataSources, getVM); } } else { diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts index 86c4c15152..a0e563f707 100644 --- a/packages/node/src/utils/substrate.ts +++ b/packages/node/src/utils/substrate.ts @@ -91,8 +91,9 @@ export function wrapExtrinsics( wrappedBlock: SubstrateBlock, allEvents: EventRecord[], ): SubstrateExtrinsic[] { + const groupedEvents = groupEventsByExtrinsic(allEvents); return wrappedBlock.block.extrinsics.map((extrinsic, idx) => { - const events = filterExtrinsicEvents(idx, allEvents); + const events = groupedEvents[idx]; return { idx, extrinsic, @@ -109,13 +110,24 @@ function getExtrinsicSuccess(events: EventRecord[]): boolean { ); } -function filterExtrinsicEvents( - extrinsicIdx: number, +function groupEventsByExtrinsic( events: EventRecord[], -): EventRecord[] { - return events.filter( - ({ phase }) => - phase.isApplyExtrinsic && phase.asApplyExtrinsic.eqn(extrinsicIdx), +): Record { + return events.reduce( + (acc, event) => { + const extrinsicIdx = event.phase.isApplyExtrinsic + ? event.phase.asApplyExtrinsic.toNumber() + : undefined; + if (extrinsicIdx === undefined) { + return acc; + } + if (!acc[extrinsicIdx]) { + acc[extrinsicIdx] = []; + } + acc[extrinsicIdx].push(event); + return acc; + }, + {} as Record, ); } @@ -311,18 +323,6 @@ export async function getHeaderByHeight( return header; } -export async function fetchBlocksRange( - api: ApiPromise, - startHeight: number, - endHeight: number, -): Promise { - return Promise.all( - range(startHeight, endHeight + 1).map(async (height) => - getBlockByHeight(api, height), - ), - ); -} - export async function fetchBlocksArray( api: ApiPromise, blockArray: number[], From d5374dd56f24b3302bd365e89e1d2999ae4a5841 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 25 Nov 2024 13:52:57 +1300 Subject: [PATCH 2/3] Update changelog --- packages/node/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 593c2076bf..c881dc65d5 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Improve event iteration, this improves performance with large blocks (#2601) ## [5.3.0] - 2024-11-22 ### Added From b639369f689793d32852624d1c1f4f9d3e018a4c Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 25 Nov 2024 15:13:37 +1300 Subject: [PATCH 3/3] Address comments --- packages/node/src/indexer/indexer.manager.ts | 13 +++++++------ packages/node/src/utils/substrate.ts | 4 +--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index fa6f7a2d40..dde11c5be7 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -112,14 +112,12 @@ export class IndexerManager extends BaseIndexerManager< const groupedEvents = events.reduce( (acc, evt, idx) => { if (evt.phase.isInitialization) { - acc.init ??= []; acc.init.push(evt); } else if (evt.phase.isFinalization) { - acc.finalize ??= []; acc.finalize.push(evt); } else if (evt.extrinsic?.idx) { const idx = evt.extrinsic.idx; - acc[idx] = acc[idx] || []; + acc[idx] ??= []; acc[idx].push(evt); } else { logger.warn( @@ -128,11 +126,14 @@ export class IndexerManager extends BaseIndexerManager< } return acc; }, - {} as Record, + { init: [], finalize: [] } as Record< + number | 'init' | 'finalize', + SubstrateEvent[] + >, ); // Run initialization events - for (const event of groupedEvents.init ?? []) { + for (const event of groupedEvents.init) { await this.indexEvent(event, dataSources, getVM); } @@ -150,7 +151,7 @@ export class IndexerManager extends BaseIndexerManager< } // Run finalization events - for (const event of groupedEvents.finalize ?? []) { + for (const event of groupedEvents.finalize) { await this.indexEvent(event, dataSources, getVM); } } else { diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts index a0e563f707..882a41d027 100644 --- a/packages/node/src/utils/substrate.ts +++ b/packages/node/src/utils/substrate.ts @@ -121,9 +121,7 @@ function groupEventsByExtrinsic( if (extrinsicIdx === undefined) { return acc; } - if (!acc[extrinsicIdx]) { - acc[extrinsicIdx] = []; - } + acc[extrinsicIdx] ??= []; acc[extrinsicIdx].push(event); return acc; },