diff --git a/packages/common-substrate/CHANGELOG.md b/packages/common-substrate/CHANGELOG.md index f2edaa1c5e..4ddaa994fa 100644 --- a/packages/common-substrate/CHANGELOG.md +++ b/packages/common-substrate/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update common (#2584) ## [4.3.2] - 2024-10-23 ### Changed diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md index 97cf37aa99..ef8f5e2b82 100644 --- a/packages/common/CHANGELOG.md +++ b/packages/common/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed - Removed imports of `@polkadot/util` without it being a dependency (#2592) +### Removed +- Support for cockroach DB (#2584) ## [5.1.4] - 2024-10-23 ### Fixed diff --git a/packages/common/src/constants.ts b/packages/common/src/constants.ts index 83deebadf8..ac9a07b954 100644 --- a/packages/common/src/constants.ts +++ b/packages/common/src/constants.ts @@ -39,12 +39,6 @@ export const runnerMapping = { '@subql/node-concordium': NETWORK_FAMILY.concordium, }; -// DATABASE TYPE -export enum SUPPORT_DB { - cockRoach = 'CockroachDB', - postgres = 'PostgreSQL', -} - // DATABASE ERROR REGEX export const CONNECTION_SSL_ERROR_REGEX = 'not support SSL'; diff --git a/packages/common/src/project/database/databaseUtil.ts b/packages/common/src/project/database/databaseUtil.ts deleted file mode 100644 index bab1e72e72..0000000000 --- a/packages/common/src/project/database/databaseUtil.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import {Sequelize} from '@subql/x-sequelize'; -import {Pool} from 'pg'; -import {SUPPORT_DB} from '../../constants'; - -export async function getDbType(queryFrom: Sequelize | Pool): Promise { - const result = await (queryFrom as any).query('select version()'); - // sequelize return an array, Promise<[unknown[], unknown] - // pgPool return a single string object with rows - const cleanResult = result instanceof Array ? result[0][0] : result.rows[0]; - const matchDB = Object.values(SUPPORT_DB).find((db) => (cleanResult as {version: string}).version.includes(db)); - if (!matchDB) { - throw new Error(`Database type not supported, got ${result}`); - } - return matchDB; -} diff --git a/packages/common/src/project/database/index.ts b/packages/common/src/project/database/index.ts deleted file mode 100644 index 336a2e8794..0000000000 --- a/packages/common/src/project/database/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -export * from './databaseUtil'; diff --git a/packages/common/src/project/index.ts b/packages/common/src/project/index.ts index 78ff1229f7..e87c0facde 100644 --- a/packages/common/src/project/index.ts +++ b/packages/common/src/project/index.ts @@ -4,6 +4,5 @@ export * from './load'; export * from './versioned'; export * from './readers'; -export * from './database'; export * from './utils'; export * from './IpfsHttpClientLite'; diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index d9c2f26ea8..07e899a302 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -7,10 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - lazy loading for monitor service (#2583) -- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations. +- Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations. ### Fixed - When using a GET query to retrieve an entity, it will include a “store” field. +- Support for historical indexing by timestamp as well as block height (#2584) + +### Removed +- Support for cockroach DB (#2584) ### Fixed - When configuring multiple endpoints, poor network conditions may lead to block crawling delays. (#2572) diff --git a/packages/node-core/src/configure/NodeConfig.ts b/packages/node-core/src/configure/NodeConfig.ts index c8ecd3baf2..acfefaac71 100644 --- a/packages/node-core/src/configure/NodeConfig.ts +++ b/packages/node-core/src/configure/NodeConfig.ts @@ -8,6 +8,7 @@ import {getFileContent, loadFromJsonOrYaml, normalizeNetworkEndpoints} from '@su import {IEndpointConfig} from '@subql/types-core'; import {last} from 'lodash'; import {LevelWithSilent} from 'pino'; +import {HistoricalMode} from '../indexer'; import {getLogger} from '../logger'; import {assign} from '../utils/object'; @@ -38,7 +39,7 @@ export interface IConfig { readonly profiler?: boolean; readonly unsafe?: boolean; readonly subscription: boolean; - readonly disableHistorical: boolean; + readonly historical: HistoricalMode; readonly multiChain: boolean; readonly reindex?: number; readonly unfinalizedBlocks?: boolean; @@ -75,7 +76,7 @@ const DEFAULT_CONFIG = { dictionaryQuerySize: 10000, profiler: false, subscription: false, - disableHistorical: false, + historical: 'height', multiChain: false, unfinalizedBlocks: false, storeCacheThreshold: 1000, @@ -257,8 +258,15 @@ export class NodeConfig implements IConfig { return this._config.subscription; } - get disableHistorical(): boolean { - return this._isTest ? true : this._config.disableHistorical; + get historical(): HistoricalMode { + if (this._isTest) return false; + + const val = this._config.historical; + // Runtime check, option can come from cli, project or config file + if (val !== false && val !== 'height' && val !== 'timestamp') { + throw new Error(`Historical mode is invalid. Received: ${val}`); + } + return val; } get multiChain(): boolean { @@ -326,7 +334,7 @@ export class NodeConfig implements IConfig { const defaultMonitorFileSize = 200; // If user passed though yarg, we will record monitor file by this size, no matter poi or not // if user didn't pass through yarg, we will record monitor file by this default size only when poi is enabled - return this._config.monitorFileSize ?? this._config.proofOfIndex ? defaultMonitorFileSize : 0; + return (this._config.monitorFileSize ?? this._config.proofOfIndex) ? defaultMonitorFileSize : 0; } get enableCache(): boolean { diff --git a/packages/node-core/src/configure/configure.module.ts b/packages/node-core/src/configure/configure.module.ts index 0766a3a7cf..74d3c2c400 100644 --- a/packages/node-core/src/configure/configure.module.ts +++ b/packages/node-core/src/configure/configure.module.ts @@ -38,8 +38,8 @@ export function validDbSchemaName(name: string): boolean { } } -// TODO once yargs is in node core we can update -type Args = Record; // typeof yargsOptions.argv['argv'] +// Cant seem to use the inferred types, strings arent converted to unions +type Args = Record; //ReturnType['argv'] function processEndpointConfig(raw?: string | string[]): IEndpointConfig[] { if (!raw) return []; @@ -67,10 +67,13 @@ export function yargsToIConfig(yargs: Args, nameMapping: Record value = [value]; } if (Array.isArray(value)) { - value = value.reduce((acc, endpoint, index) => { - acc[endpoint] = endpointConfig[index] ?? {}; - return acc; - }, {} as Record); + value = value.reduce( + (acc, endpoint, index) => { + acc[endpoint] = endpointConfig[index] ?? {}; + return acc; + }, + {} as Record + ); } } if (key === 'primary-network-endpoint') { @@ -79,6 +82,13 @@ export function yargsToIConfig(yargs: Args, nameMapping: Record } if (['network-endpoint-config', 'primary-network-endpoint-config'].includes(key)) return acc; + if (key === 'disable-historical' && value) { + acc.historical = false; + } + if (key === 'historical' && value === 'false') { + value = false; + } + acc[nameMapping[key] ?? camelCase(key)] = value; return acc; }, {} as any); diff --git a/packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts b/packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts index 6f6d859361..f3952976a4 100644 --- a/packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts +++ b/packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts @@ -227,7 +227,7 @@ describe('SchemaMigration integration tests', () => { schemaName, initialSchema, sequelize, - new NodeConfig({disableHistorical: true} as any) + new NodeConfig({historical: false} as any) ); await migrationService.run(initialSchema, loadGqlSchema('test_10_1000.graphql')); @@ -323,7 +323,7 @@ WHERE schemaName, initialSchema, sequelize, - new NodeConfig({disableHistorical: true} as any) + new NodeConfig({historical: false} as any) ); await migrationService.run(initialSchema, loadGqlSchema('test_13_2000.graphql')); @@ -363,7 +363,7 @@ WHERE schemaName, initialSchema, sequelize, - new NodeConfig({disableHistorical: true} as any) + new NodeConfig({historical: false} as any) ); await migrationService.run(initialSchema, loadGqlSchema('test_14_1000.graphql')); diff --git a/packages/node-core/src/db/migration-service/SchemaMigration.service.ts b/packages/node-core/src/db/migration-service/SchemaMigration.service.ts index 9271a9d300..e2a7472d56 100644 --- a/packages/node-core/src/db/migration-service/SchemaMigration.service.ts +++ b/packages/node-core/src/db/migration-service/SchemaMigration.service.ts @@ -1,7 +1,6 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {SUPPORT_DB} from '@subql/common'; import {getAllEntitiesRelations, GraphQLModelsType, GraphQLRelationsType} from '@subql/utils'; import {Sequelize, Transaction} from '@subql/x-sequelize'; import {GraphQLSchema} from 'graphql'; @@ -30,8 +29,7 @@ export class SchemaMigrationService { private sequelize: Sequelize, private storeService: StoreService, private dbSchema: string, - private config: NodeConfig, - private dbType: SUPPORT_DB = SUPPORT_DB.postgres + private config: NodeConfig ) {} static validateSchemaChanges(currentSchema: GraphQLSchema, nextSchema: GraphQLSchema): boolean { @@ -116,13 +114,7 @@ export class SchemaMigrationService { await cacheProviderFlushData(this.storeService.modelProvider, true); - const migrationAction = await Migration.create( - this.sequelize, - this.storeService, - this.dbSchema, - this.config, - this.dbType - ); + const migrationAction = await Migration.create(this.sequelize, this.storeService, this.dbSchema, this.config); if (this.config.debug) { logger.debug(`${schemaChangesLoggerMessage(schemaDifference)}`); diff --git a/packages/node-core/src/db/migration-service/migration.ts b/packages/node-core/src/db/migration-service/migration.ts index 321f976099..aa5cdf35cb 100644 --- a/packages/node-core/src/db/migration-service/migration.ts +++ b/packages/node-core/src/db/migration-service/migration.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'node:assert'; -import {SUPPORT_DB} from '@subql/common'; import { GraphQLEntityField, GraphQLEntityIndex, @@ -10,7 +9,6 @@ import { GraphQLModelsType, GraphQLRelationsType, hashName, - IndexType, } from '@subql/utils'; import { IndexesOptions, @@ -23,7 +21,7 @@ import { } from '@subql/x-sequelize'; import {isEqual, uniq} from 'lodash'; import {NodeConfig} from '../../configure/NodeConfig'; -import {StoreService} from '../../indexer'; +import {HistoricalMode, StoreService} from '../../indexer'; import {getLogger} from '../../logger'; import {EnumType, getColumnOption, modelsTypeToModelAttributes} from '../../utils'; import {formatAttributes, formatColumnName, modelToTableName} from '../sequelizeUtil'; @@ -43,21 +41,19 @@ export class Migration { */ private mainQueries: syncHelper.QueryString[] = []; private extraQueries: syncHelper.QueryString[] = []; - private readonly historical: boolean; + private readonly historical: HistoricalMode; private readonly useSubscription: boolean; private foreignKeyMap: Map> = new Map< string, Map >(); private enumTypeMap: Map; - private removedIndexes: RemovedIndexes = {}; private constructor( private sequelize: Sequelize, private storeService: StoreService, private readonly schemaName: string, private readonly config: NodeConfig, - private readonly dbType: SUPPORT_DB, private readonly existingForeignKeys: string[], // this the source of truth from the db private initEnumTypeMap: Map, private existingIndexes: {indexname: string}[] @@ -67,15 +63,12 @@ export class Migration { this.historical = storeService.historical; this.useSubscription = config.subscription; - if (this.useSubscription && dbType === SUPPORT_DB.cockRoach) { - this.useSubscription = false; - logger.warn(`Subscription is not support with ${this.dbType}`); - } - this.enumTypeMap = this.initEnumTypeMap; if (this.useSubscription) { this.extraQueries.push(syncHelper.createSendNotificationTriggerFunction(schemaName)); + } else { + this.extraQueries.push(syncHelper.dropNotifyFunction(this.schemaName)); } } @@ -83,8 +76,7 @@ export class Migration { sequelize: Sequelize, storeService: StoreService, schemaName: string, - config: NodeConfig, - dbType: SUPPORT_DB + config: NodeConfig ): Promise { const existingForeignKeys = await syncHelper.getExistingForeignKeys(schemaName, sequelize); const enumTypeMap = await syncHelper.getExistingEnums(schemaName, sequelize); @@ -93,16 +85,7 @@ export class Migration { })) as { indexname: string; }[]; - return new Migration( - sequelize, - storeService, - schemaName, - config, - dbType, - existingForeignKeys, - enumTypeMap, - indexesResult - ); + return new Migration(sequelize, storeService, schemaName, config, existingForeignKeys, enumTypeMap, indexesResult); } async run(transaction?: Transaction): Promise<{modifiedModels: ModelStatic[]; removedModels: string[]}> { @@ -137,8 +120,6 @@ export class Migration { throw e; } - this.afterHandleCockroachIndex(); - return { modifiedModels: this.modifiedModels, removedModels: this.removedModels, @@ -189,8 +170,6 @@ export class Migration { } syncHelper.updateIndexesName(model.name, indexes, existedIndexes as string[]); - // Update index query for cockroach db - this.beforeHandleCockroachIndex(model.name, indexes, existedIndexes as string[]); const sequelizeModel = this.storeService.defineModel(model, attributes, indexes, this.schemaName); @@ -220,15 +199,8 @@ export class Migration { syncHelper.validateNotifyTriggers(triggerName, notifyTriggers as syncHelper.NotifyTriggerPayload[]); } } else { - //TODO: DROP TRIGGER IF EXIST is not valid syntax for cockroach, better check trigger exist at first. - if (this.dbType !== SUPPORT_DB.cockRoach) { - // trigger drop should be prioritized - this.extraQueries.unshift(syncHelper.dropNotifyTrigger(this.schemaName, sequelizeModel.tableName)); - } - } - - if (!this.useSubscription && this.dbType !== SUPPORT_DB.cockRoach) { - this.extraQueries.push(syncHelper.dropNotifyFunction(this.schemaName)); + // should prioritise dropping the triggers before dropNotifyFunction + this.extraQueries.unshift(syncHelper.dropNotifyTrigger(this.schemaName, sequelizeModel.tableName)); } this.addModelToSequelizeCache(sequelizeModel); @@ -237,7 +209,7 @@ export class Migration { dropTable(model: GraphQLModelsType): void { const tableName = modelToTableName(model.name); - // should prioritise dropping the triggers + // should prioritise dropping the triggers before dropNotifyFunction this.mainQueries.unshift(syncHelper.dropNotifyTrigger(this.schemaName, tableName)); this.mainQueries.push(`DROP TABLE IF EXISTS "${this.schemaName}"."${tableName}";`); this.removedModels.push(model.name); @@ -311,9 +283,7 @@ export class Migration { const rel = model.belongsTo(relatedModel, {foreignKey: relation.foreignKey}); const fkConstraint = syncHelper.getFkConstraint(rel.source.tableName, rel.foreignKey); if (this.existingForeignKeys.includes(fkConstraint)) break; - if (this.dbType !== SUPPORT_DB.cockRoach) { - this.extraQueries.push(syncHelper.constraintDeferrableQuery(model.getTableName().toString(), fkConstraint)); - } + this.extraQueries.push(syncHelper.constraintDeferrableQuery(model.getTableName().toString(), fkConstraint)); break; } case 'hasOne': { @@ -415,17 +385,9 @@ export class Migration { queries.unshift(syncHelper.createEnumQuery(type, escapedEnumValues)); } - if (this.dbType === SUPPORT_DB.cockRoach) { - logger.warn( - `Comment on enum ${e.description} is not supported with ${this.dbType}, enum name may display incorrectly in query service` - ); - } else { - const comment = this.sequelize.escape( - `@enum\\n@enumName ${e.name}${e.description ? `\\n ${e.description}` : ''}` - ); + const comment = this.sequelize.escape(`@enum\\n@enumName ${e.name}${e.description ? `\\n ${e.description}` : ''}`); - queries.push(syncHelper.commentOnEnumQuery(type, comment)); - } + queries.push(syncHelper.commentOnEnumQuery(type, comment)); this.mainQueries.unshift(...queries); this.enumTypeMap.set(enumTypeName, { enumValues: e.values, @@ -473,44 +435,4 @@ export class Migration { this.mainQueries.push(...queries); } - - // Sequelize model will generate follow query to create hash indexes - // Example SQL: CREATE INDEX "accounts_person_id" ON "polkadot-starter"."accounts" USING hash ("person_id") - // This will be rejected from cockroach db due to syntax error - // To avoid this we need to create index manually and add to extraQueries in order to create index in db - private beforeHandleCockroachIndex(modelName: string, indexes: IndexesOptions[], existedIndexes: string[]): void { - if (this.dbType !== SUPPORT_DB.cockRoach) { - return; - } - indexes.forEach((index, i) => { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - if (index.using === IndexType.HASH && !existedIndexes.includes(index.name!)) { - // TODO double check with idempotent on cockroach - const cockroachDbIndexQuery = `CREATE INDEX "${index.name}" ON "${this.schemaName}"."${modelToTableName( - modelName - )}"(${index.fields}) USING HASH;`; - this.extraQueries.push(cockroachDbIndexQuery); - if (this.removedIndexes[modelName] === undefined) { - this.removedIndexes[modelName] = []; - } - this.removedIndexes[modelName].push(indexes[i]); - delete indexes[i]; - } - }); - } - - // Due to we have removed hash index, it will be missing from the model, we need temp store it under `this.removedIndexes` - // And force add back to the model use `afterHandleCockroachIndex()` after db is synced - private afterHandleCockroachIndex(): void { - if (this.dbType !== SUPPORT_DB.cockRoach) { - return; - } - const removedIndexes = Object.entries(this.removedIndexes); - if (removedIndexes.length > 0) { - for (const [model, indexes] of removedIndexes) { - const sqModel = this.sequelize.model(model); - (sqModel as any)._indexes = (sqModel as any)._indexes.concat(indexes); - } - } - } } diff --git a/packages/node-core/src/db/sync-helper.ts b/packages/node-core/src/db/sync-helper.ts index 2545ac12e3..6f5cd8dc89 100644 --- a/packages/node-core/src/db/sync-helper.ts +++ b/packages/node-core/src/db/sync-helper.ts @@ -170,7 +170,7 @@ export async function getFunctions(sequelize: Sequelize, schema: string, functio ); } -export function createSendNotificationTriggerFunction(schema: string) { +export function createSendNotificationTriggerFunction(schema: string): string { return ` CREATE OR REPLACE FUNCTION "${schema}".send_notification() RETURNS trigger AS $$ diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index f3d98557da..45c2794357 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -17,14 +17,13 @@ import {SmartBatchService} from '../smartBatch.service'; import {StoreService} from '../store.service'; import {IStoreModelProvider} from '../storeModelProvider'; import {IPoi} from '../storeModelProvider/poi'; -import {IBlock, IProjectService, ISubqueryProject} from '../types'; +import {Header, IBlock, IProjectService, ISubqueryProject} from '../types'; const logger = getLogger('BaseBlockDispatcherService'); export type ProcessBlockResponse = { dynamicDsCreated: boolean; - blockHash: string; - reindexBlockHeight: number | null; + reindexBlockHeader: Header | null; }; export interface IBlockDispatcher { @@ -52,7 +51,7 @@ export abstract class BaseBlockDispatcher implements IB protected _latestProcessedHeight = 0; protected currentProcessingHeight = 0; private _onDynamicDsCreated?: (height: number) => void; - private _pendingRewindHeight?: number; + private _pendingRewindHeader?: Header; protected smartBatchService: SmartBatchService; @@ -138,14 +137,14 @@ export abstract class BaseBlockDispatcher implements IB * @param lastCorrectHeight */ @mainThreadOnly() - protected async rewind(lastCorrectHeight: number): Promise { - if (lastCorrectHeight <= this.currentProcessingHeight) { - logger.info(`Found last verified block at height ${lastCorrectHeight}, rewinding...`); - await this.projectService.reindex(lastCorrectHeight); - this.setLatestProcessedHeight(lastCorrectHeight); - logger.info(`Successful rewind to block ${lastCorrectHeight}!`); + protected async rewind(lastCorrectHeader: Header): Promise { + if (lastCorrectHeader.blockHeight <= this.currentProcessingHeight) { + logger.info(`Found last verified block at height ${lastCorrectHeader.blockHeight}, rewinding...`); + await this.projectService.reindex(lastCorrectHeader); + this.setLatestProcessedHeight(lastCorrectHeader.blockHeight); + logger.info(`Successful rewind to block ${lastCorrectHeader.blockHeight}!`); } - this.flushQueue(lastCorrectHeight); + this.flushQueue(lastCorrectHeader.blockHeight); logger.info(`Queued blocks flushed!`); //Also last buffered height reset, next fetching should start after lastCorrectHeight } @@ -156,42 +155,43 @@ export abstract class BaseBlockDispatcher implements IB // Is called directly before a block is processed @mainThreadOnly() - protected async preProcessBlock(height: number): Promise { - monitorCreateBlockStart(height); - await this.storeService.setBlockHeight(height); + protected async preProcessBlock(header: Header): Promise { + const {blockHeight} = header; + monitorCreateBlockStart(blockHeight); + await this.storeService.setBlockHeader(header); - await this.projectUpgradeService.setCurrentHeight(height); + await this.projectUpgradeService.setCurrentHeight(blockHeight); - this.currentProcessingHeight = height; + this.currentProcessingHeight = blockHeight; this.eventEmitter.emit(IndexerEvent.BlockProcessing, { - height, + height: blockHeight, timestamp: Date.now(), }); } // Is called directly after a block is processed @mainThreadOnly() - protected async postProcessBlock(height: number, processBlockResponse: ProcessBlockResponse): Promise { - const {blockHash, dynamicDsCreated, reindexBlockHeight: processReindexBlockHeight} = processBlockResponse; + protected async postProcessBlock(header: Header, processBlockResponse: ProcessBlockResponse): Promise { + const {blockHash, blockHeight: height} = header; + const {dynamicDsCreated, reindexBlockHeader: processReindexBlockHeader} = processBlockResponse; // Rewind height received from admin api have higher priority than processed reindexBlockHeight - const reindexBlockHeight = this._pendingRewindHeight ?? processReindexBlockHeight; + const reindexBlockHeader = this._pendingRewindHeader ?? processReindexBlockHeader; monitorWrite(`Finished block ${height}`); - if (reindexBlockHeight !== null && reindexBlockHeight !== undefined) { + if (reindexBlockHeader !== null && reindexBlockHeader !== undefined) { try { if (this.nodeConfig.proofOfIndex) { await this.poiSyncService.stopSync(); this.poiSyncService.clear(); monitorWrite(`poiSyncService stopped, cache cleared`); } - monitorCreateBlockFork(reindexBlockHeight); - this.resetPendingRewindHeight(); - await this.rewind(reindexBlockHeight); - this.setLatestProcessedHeight(reindexBlockHeight); + monitorCreateBlockFork(reindexBlockHeader.blockHeight); + this.resetPendingRewindHeader(); + await this.rewind(reindexBlockHeader); // Bring poi sync service back to sync again. if (this.nodeConfig.proofOfIndex) { void this.poiSyncService.syncPoi(); } - this.eventEmitter.emit(IndexerEvent.RewindSuccess, {success: true, height: reindexBlockHeight}); + this.eventEmitter.emit(IndexerEvent.RewindSuccess, {success: true, height: reindexBlockHeader.blockHeight}); return; } catch (e: any) { this.eventEmitter.emit(IndexerEvent.RewindFailure, {success: false, message: e.message}); @@ -199,7 +199,7 @@ export abstract class BaseBlockDispatcher implements IB throw e; } } else { - await this.updateStoreMetadata(height, undefined, this.storeService.transaction); + await this.updateStoreMetadata(height, header.timestamp, undefined, this.storeService.transaction); const operationHash = this.storeService.getOperationMerkleRoot(); await this.createPOI(height, blockHash, operationHash, this.storeService.transaction); @@ -231,14 +231,18 @@ export abstract class BaseBlockDispatcher implements IB `Current processing block ${this.currentProcessingHeight}, can not rewind to future block ${blockPayload.height}` ); } - this._pendingRewindHeight = Number(blockPayload.height); + + // TODO can this work without + this._pendingRewindHeader = { + blockHeight: Number(blockPayload.height), + } as Header; const message = `Received admin command to rewind to block ${blockPayload.height}`; monitorWrite(`***** [ADMIN] ${message}`); logger.warn(message); } - private resetPendingRewindHeight(): void { - this._pendingRewindHeight = undefined; + private resetPendingRewindHeader(): void { + this._pendingRewindHeader = undefined; } /** @@ -273,7 +277,12 @@ export abstract class BaseBlockDispatcher implements IB } @mainThreadOnly() - private async updateStoreMetadata(height: number, updateProcessed = true, tx?: Transaction): Promise { + private async updateStoreMetadata( + height: number, + blockTimestamp?: Date, + updateProcessed = true, + tx?: Transaction + ): Promise { const meta = this.storeModelProvider.metadata; // Update store metadata await meta.setBulk( @@ -287,6 +296,9 @@ export abstract class BaseBlockDispatcher implements IB if (updateProcessed) { await meta.setIncrement('processedBlockCount', undefined, tx); } + if (blockTimestamp) { + await meta.set('lastProcessedBlockTimestamp', blockTimestamp.getTime(), tx); + } } private get poi(): IPoi { diff --git a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts index 01ca467238..3962d0c074 100644 --- a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts @@ -149,23 +149,26 @@ export abstract class BlockDispatcher }) .then( (block) => { - const {blockHeight} = block.getHeader(); + const header = block.getHeader(); return this.processQueue.put(async () => { // Check if the queues have been flushed between queue.takeMany and fetchBlocksBatches resolving // Peeking the queue is because the latestBufferedHeight could have regrown since fetching block const peeked = this.queue.peek(); - if (bufferedHeight > this._latestBufferedHeight || (peeked && getBlockHeight(peeked) < blockHeight)) { + if ( + bufferedHeight > this._latestBufferedHeight || + (peeked && getBlockHeight(peeked) < header.blockHeight) + ) { logger.info(`Queue was reset for new DS, discarding fetched blocks`); return; } try { - await this.preProcessBlock(blockHeight); + await this.preProcessBlock(header); monitorWrite(`Processing from main thread`); // Inject runtimeVersion here to enhance api.at preparation const processBlockResponse = await this.indexBlock(block); - await this.postProcessBlock(blockHeight, processBlockResponse); + await this.postProcessBlock(header, processBlockResponse); //set block to null for garbage collection (block as any) = null; } catch (e: any) { @@ -175,7 +178,7 @@ export abstract class BlockDispatcher } logger.error( e, - `Failed to index block at height ${blockHeight} ${ + `Failed to index block at height ${header.blockHeight} ${ e.handler ? `${e.handler}(${e.stack ?? ''})` : '' }` ); diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts index cc8e5da01e..e0d49ecc10 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.spec.ts @@ -1,17 +1,19 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {EventEmitter2} from '@nestjs/event-emitter'; -import {IProjectUpgradeService, NodeConfig} from '../../configure'; -import {PoiSyncService} from '../poi'; -import {StoreService} from '../store.service'; -import {StoreCacheService} from '../storeModelProvider'; -import {IProjectService, ISubqueryProject} from '../types'; -import {WorkerBlockDispatcher} from './worker-block-dispatcher'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { IProjectUpgradeService, NodeConfig } from '../../configure'; +import { PoiSyncService } from '../poi'; +import { StoreService } from '../store.service'; +import { StoreCacheService } from '../storeModelProvider'; +import { Header, IProjectService, ISubqueryProject } from '../types'; +import { WorkerBlockDispatcher } from './worker-block-dispatcher'; class TestWorkerBlockDispatcher extends WorkerBlockDispatcher { - async fetchBlock(worker: any, height: number): Promise { - return Promise.resolve(); + async fetchBlock(worker: any, height: number): Promise
{ + return Promise.resolve({ + blockHeight: height, + } as Header); } get minimumHeapLimit(): number { @@ -23,14 +25,14 @@ describe('WorkerBlockDispatcher', () => { // Mock workers const mockWorkers = [ - {getMemoryLeft: jest.fn().mockResolvedValue(100), waitForWorkerBatchSize: jest.fn().mockResolvedValue(undefined)}, - {getMemoryLeft: jest.fn().mockResolvedValue(200), waitForWorkerBatchSize: jest.fn().mockResolvedValue(undefined)}, - {getMemoryLeft: jest.fn().mockResolvedValue(300), waitForWorkerBatchSize: jest.fn().mockResolvedValue(undefined)}, + { getMemoryLeft: jest.fn().mockResolvedValue(100), waitForWorkerBatchSize: jest.fn().mockResolvedValue(undefined) }, + { getMemoryLeft: jest.fn().mockResolvedValue(200), waitForWorkerBatchSize: jest.fn().mockResolvedValue(undefined) }, + { getMemoryLeft: jest.fn().mockResolvedValue(300), waitForWorkerBatchSize: jest.fn().mockResolvedValue(undefined) }, ]; beforeEach(() => { dispatcher = new TestWorkerBlockDispatcher( - {workers: 3} as unknown as NodeConfig, + { workers: 3 } as unknown as NodeConfig, null as unknown as EventEmitter2, null as unknown as IProjectService, null as unknown as IProjectUpgradeService, diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts index 8b105e1272..3031994c8e 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts @@ -2,23 +2,23 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {OnApplicationShutdown} from '@nestjs/common'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {Interval} from '@nestjs/schedule'; -import {last} from 'lodash'; -import {NodeConfig} from '../../configure'; -import {IProjectUpgradeService} from '../../configure/ProjectUpgrade.service'; -import {IndexerEvent} from '../../events'; -import {IBlock, PoiSyncService} from '../../indexer'; -import {getLogger} from '../../logger'; -import {monitorWrite} from '../../process'; -import {AutoQueue, isTaskFlushedError} from '../../utils'; -import {MonitorServiceInterface} from '../monitor.service'; -import {StoreService} from '../store.service'; -import {IStoreModelProvider} from '../storeModelProvider'; -import {ISubqueryProject, IProjectService} from '../types'; -import {isBlockUnavailableError} from '../worker/utils'; -import {BaseBlockDispatcher} from './base-block-dispatcher'; +import { OnApplicationShutdown } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { Interval } from '@nestjs/schedule'; +import { last } from 'lodash'; +import { NodeConfig } from '../../configure'; +import { IProjectUpgradeService } from '../../configure/ProjectUpgrade.service'; +import { IndexerEvent } from '../../events'; +import { IBlock, PoiSyncService } from '../../indexer'; +import { getLogger } from '../../logger'; +import { monitorWrite } from '../../process'; +import { AutoQueue, isTaskFlushedError } from '../../utils'; +import { MonitorServiceInterface } from '../monitor.service'; +import { StoreService } from '../store.service'; +import { IStoreModelProvider } from '../storeModelProvider'; +import { ISubqueryProject, IProjectService, Header } from '../types'; +import { isBlockUnavailableError } from '../worker/utils'; +import { BaseBlockDispatcher } from './base-block-dispatcher'; const logger = getLogger('WorkerBlockDispatcherService'); @@ -43,14 +43,13 @@ function initAutoQueue( export abstract class WorkerBlockDispatcher extends BaseBlockDispatcher, DS, B> - implements OnApplicationShutdown -{ + implements OnApplicationShutdown { protected workers: W[] = []; private numWorkers: number; private isShutdown = false; private currentWorkerIndex = 0; - protected abstract fetchBlock(worker: W, height: number): Promise; + protected abstract fetchBlock(worker: W, height: number): Promise
; constructor( nodeConfig: NodeConfig, @@ -151,25 +150,26 @@ export abstract class WorkerBlockDispatcher const processBlock = async () => { try { - await pendingBlock; + const header = await pendingBlock; if (bufferedHeight > this.latestBufferedHeight) { logger.debug(`Queue was reset for new DS, discarding fetched blocks`); return; } - await this.preProcessBlock(height); + await this.preProcessBlock(header); monitorWrite(`Processing from worker #${workerIdx}`); - const {blockHash, dynamicDsCreated, reindexBlockHeight} = await worker.processBlock(height); + const { dynamicDsCreated, reindexBlockHeader } = await worker.processBlock(height); - await this.postProcessBlock(height, { + await this.postProcessBlock(header, { dynamicDsCreated, - blockHash, - reindexBlockHeight, + reindexBlockHeader, }); } catch (e: any) { // TODO discard any cache changes from this block height - + if (isTaskFlushedError(e)) { + return; + } if (isBlockUnavailableError(e)) { return; } diff --git a/packages/node-core/src/indexer/dictionary/utils.spec.ts b/packages/node-core/src/indexer/dictionary/utils.spec.ts index c97bc0852f..df25711f0c 100644 --- a/packages/node-core/src/indexer/dictionary/utils.spec.ts +++ b/packages/node-core/src/indexer/dictionary/utils.spec.ts @@ -7,7 +7,7 @@ import {mergeNumAndBlocks, mergeNumAndBlocksToNums} from './utils'; function mockIblock(n: number): IBlock<{height: number; hash: string}> { return { getHeader: () => { - return {blockHeight: n, parentHash: `0x${n - 1}`, blockHash: `0x${n}`}; + return {blockHeight: n, parentHash: `0x${n - 1}`, blockHash: `0x${n}`, timestamp: new Date()}; }, block: {height: n, hash: `0x${n}`}, }; @@ -56,7 +56,7 @@ it('mergeNumAndBlocksToNums, turn all blocks into number', () => { function mockIblock(n: number): IBlock<{height: number; hash: string}> { return { getHeader: () => { - return {blockHeight: n, parentHash: `0x${n - 1}`, blockHash: `0x${n}`}; + return {blockHeight: n, parentHash: `0x${n - 1}`, blockHash: `0x${n}`, timestamp: new Date()}; }, block: {height: n, hash: `0x${n}`}, }; diff --git a/packages/node-core/src/indexer/entities/Metadata.entity.ts b/packages/node-core/src/indexer/entities/Metadata.entity.ts index b3d7056442..60cdbbdf91 100644 --- a/packages/node-core/src/indexer/entities/Metadata.entity.ts +++ b/packages/node-core/src/indexer/entities/Metadata.entity.ts @@ -4,14 +4,16 @@ import {getMetadataTableName} from '@subql/utils'; import {BuildOptions, DataTypes, Model, QueryTypes, Sequelize} from '@subql/x-sequelize'; import {DatasourceParams} from '../dynamic-ds.service'; +import {HistoricalMode} from '../types'; export interface MetadataKeys { chain: string; genesisHash: string; startHeight: number; - historicalStateEnabled: boolean; + historicalStateEnabled: HistoricalMode; indexerNodeVersion: string; lastProcessedHeight: number; + lastProcessedBlockTimestamp: number; // The unix timestamp of the block in MS lastProcessedTimestamp: string; processedBlockCount: number; blockOffset: number; diff --git a/packages/node-core/src/indexer/fetch.service.spec.ts b/packages/node-core/src/indexer/fetch.service.spec.ts index 2e4d4eda6e..3a5d7b45ea 100644 --- a/packages/node-core/src/indexer/fetch.service.spec.ts +++ b/packages/node-core/src/indexer/fetch.service.spec.ts @@ -72,7 +72,12 @@ class TestFetchService extends BaseFetchService { - return Promise.resolve({blockHeight: this.finalizedHeight, blockHash: '0xxx', parentHash: '0xxx'}); + return Promise.resolve({ + blockHeight: this.finalizedHeight, + blockHash: '0xxx', + parentHash: '0xxx', + timestamp: new Date(), + }); } } @@ -344,7 +349,7 @@ describe('Fetch Service', () => { expect(finalizedSpy).toHaveBeenCalledTimes(2); expect(bestSpy).toHaveBeenCalledTimes(2); - await expect(fetchService.getFinalizedHeader()).resolves.toEqual({ + await expect(fetchService.getFinalizedHeader()).resolves.toMatchObject({ blockHeight: fetchService.finalizedHeight, blockHash: '0xxx', parentHash: '0xxx', diff --git a/packages/node-core/src/indexer/indexer.manager.ts b/packages/node-core/src/indexer/indexer.manager.ts index 24647b8cae..44c108d37a 100644 --- a/packages/node-core/src/indexer/indexer.manager.ts +++ b/packages/node-core/src/indexer/indexer.manager.ts @@ -13,7 +13,7 @@ import {ProcessBlockResponse} from './blockDispatcher'; import {asSecondLayerHandlerProcessor_1_0_0, BaseDsProcessorService} from './ds-processor.service'; import {DynamicDsService} from './dynamic-ds.service'; import {IndexerSandbox} from './sandbox'; -import {IBlock, IIndexerManager} from './types'; +import {Header, IBlock, IIndexerManager} from './types'; import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service'; const logger = getLogger('indexer'); @@ -95,10 +95,10 @@ export abstract class BaseIndexerManager< this.assertDataSources(filteredDataSources, blockHeight); let apiAt: SA; - const reindexBlockHeight = (await this.processUnfinalizedBlocks(block)) ?? null; + const reindexBlockHeader = (await this.processUnfinalizedBlocks(block)) ?? null; // Only index block if we're not going to reindex - if (!reindexBlockHeight) { + if (!reindexBlockHeader) { await this.indexBlockData(block.block, filteredDataSources, async (ds: DS) => { // Injected runtimeVersion from fetch service might be outdated apiAt ??= await getApi(); @@ -123,12 +123,11 @@ export abstract class BaseIndexerManager< return { dynamicDsCreated, - blockHash: block.getHeader().blockHash, - reindexBlockHeight, + reindexBlockHeader, }; } - protected async processUnfinalizedBlocks(block: IBlock): Promise { + protected async processUnfinalizedBlocks(block: IBlock): Promise
{ if (this.nodeConfig.unfinalizedBlocks) { return this.unfinalizedBlocksService.processUnfinalizedBlocks(block); } diff --git a/packages/node-core/src/indexer/poi/poi.service.spec.ts b/packages/node-core/src/indexer/poi/poi.service.spec.ts index 353d6e7b3d..da629939fe 100644 --- a/packages/node-core/src/indexer/poi/poi.service.spec.ts +++ b/packages/node-core/src/indexer/poi/poi.service.spec.ts @@ -64,7 +64,7 @@ describe('PoiService', () => { const sequelize = new Sequelize(); storeCache = new StoreCacheService(sequelize, nodeConfig, new EventEmitter2(), new SchedulerRegistry()); - storeCache.init(true, true, {} as any, {} as any); + storeCache.init('height', {} as any, {} as any); (storeCache as any).cachedModels._metadata = { find: jest.fn().mockResolvedValue(10), bulkRemove: jest.fn(), diff --git a/packages/node-core/src/indexer/project.service.spec.ts b/packages/node-core/src/indexer/project.service.spec.ts index 0585b3bd8b..49ae4bf431 100644 --- a/packages/node-core/src/indexer/project.service.spec.ts +++ b/packages/node-core/src/indexer/project.service.spec.ts @@ -1,14 +1,13 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {EventEmitter2} from '@nestjs/event-emitter'; -import {buildSchemaFromString} from '@subql/utils'; -import {NodeConfig, ProjectUpgradeService} from '../configure'; -import {BaseDsProcessorService} from './ds-processor.service'; -import {DynamicDsService} from './dynamic-ds.service'; -import {BaseProjectService} from './project.service'; -import {StoreService} from './store.service'; -import {Header, ISubqueryProject} from './types'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { buildSchemaFromString } from '@subql/utils'; +import { NodeConfig, ProjectUpgradeService } from '../configure'; +import { BaseDsProcessorService } from './ds-processor.service'; +import { DynamicDsService } from './dynamic-ds.service'; +import { BaseProjectService } from './project.service'; +import { Header, ISubqueryProject } from './types'; import { BaseUnfinalizedBlocksService, METADATA_LAST_FINALIZED_PROCESSED_KEY, @@ -38,6 +37,7 @@ class TestUnfinalizedBlocksService extends BaseUnfinalizedBlocksService { blockHash: 'asdf', blockHeight: 1000, parentHash: 'efgh', + timestamp: new Date(), }; } @@ -48,15 +48,17 @@ class TestUnfinalizedBlocksService extends BaseUnfinalizedBlocksService { blockHeight: num, blockHash: hash, parentHash: `b${num - 1}`, + timestamp: new Date(), }; } // eslint-disable-next-line @typescript-eslint/require-await - protected async getHeaderForHeight(height: number): Promise
{ + async getHeaderForHeight(height: number): Promise
{ return { blockHeight: height, blockHash: `b${height}`, parentHash: `b${height - 1}`, + timestamp: new Date(), }; } } @@ -71,11 +73,11 @@ describe('BaseProjectService', () => { null as unknown as any, null as unknown as any, null as unknown as any, - {dataSources: []} as unknown as ISubqueryProject, + { dataSources: [] } as unknown as ISubqueryProject, null as unknown as any, null as unknown as any, - {unsafe: false} as unknown as NodeConfig, - {getDynamicDatasources: jest.fn()} as unknown as DynamicDsService, + { unsafe: false } as unknown as NodeConfig, + { getDynamicDatasources: jest.fn() } as unknown as DynamicDsService, null as unknown as any, null as unknown as any ); @@ -89,11 +91,11 @@ describe('BaseProjectService', () => { 1, { dataSources: [ - {startBlock: 1, endBlock: 300}, - {startBlock: 10, endBlock: 20}, - {startBlock: 1, endBlock: 100}, - {startBlock: 50, endBlock: 200}, - {startBlock: 500}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 10, endBlock: 20 }, + { startBlock: 1, endBlock: 100 }, + { startBlock: 50, endBlock: 200 }, + { startBlock: 500 }, ], }, ], @@ -114,10 +116,10 @@ describe('BaseProjectService', () => { 1, { dataSources: [ - {startBlock: 1, endBlock: 300}, - {startBlock: 10, endBlock: 20}, - {startBlock: 1, endBlock: 100}, - {startBlock: 50, endBlock: 200}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 10, endBlock: 20 }, + { startBlock: 1, endBlock: 100 }, + { startBlock: 50, endBlock: 200 }, ], }, ], @@ -130,17 +132,17 @@ describe('BaseProjectService', () => { it('getDataSources', async () => { (service as any).project.dataSources = [ - {startBlock: 100, endBlock: 200}, - {startBlock: 1, endBlock: 100}, + { startBlock: 100, endBlock: 200 }, + { startBlock: 1, endBlock: 100 }, ]; (service as any).dynamicDsService.getDynamicDatasources = jest .fn() - .mockResolvedValue([{startBlock: 150, endBlock: 250}]); + .mockResolvedValue([{ startBlock: 150, endBlock: 250 }]); const result = await service.getDataSources(175); expect(result).toEqual([ - {startBlock: 100, endBlock: 200}, - {startBlock: 150, endBlock: 250}, + { startBlock: 100, endBlock: 200 }, + { startBlock: 150, endBlock: 250 }, ]); }); @@ -153,11 +155,11 @@ describe('BaseProjectService', () => { 1, { dataSources: [ - {startBlock: 1, endBlock: 300}, - {startBlock: 10, endBlock: 20}, - {startBlock: 1, endBlock: 100}, - {startBlock: 50, endBlock: 200}, - {startBlock: 500}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 10, endBlock: 20 }, + { startBlock: 1, endBlock: 100 }, + { startBlock: 50, endBlock: 200 }, + { startBlock: 500 }, ], }, ], @@ -170,43 +172,43 @@ describe('BaseProjectService', () => { [ 1, [ - {startBlock: 1, endBlock: 300}, - {startBlock: 1, endBlock: 100}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 1, endBlock: 100 }, ], ], [ 10, [ - {startBlock: 1, endBlock: 300}, - {startBlock: 1, endBlock: 100}, - {startBlock: 10, endBlock: 20}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 1, endBlock: 100 }, + { startBlock: 10, endBlock: 20 }, ], ], [ 21, [ - {startBlock: 1, endBlock: 300}, - {startBlock: 1, endBlock: 100}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 1, endBlock: 100 }, ], ], [ 50, [ - {startBlock: 1, endBlock: 300}, - {startBlock: 1, endBlock: 100}, - {startBlock: 50, endBlock: 200}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 1, endBlock: 100 }, + { startBlock: 50, endBlock: 200 }, ], ], [ 101, [ - {startBlock: 1, endBlock: 300}, - {startBlock: 50, endBlock: 200}, + { startBlock: 1, endBlock: 300 }, + { startBlock: 50, endBlock: 200 }, ], ], - [201, [{startBlock: 1, endBlock: 300}]], + [201, [{ startBlock: 1, endBlock: 300 }]], [301, []], - [500, [{startBlock: 500}]], + [500, [{ startBlock: 500 }]], ]) ); }); @@ -218,13 +220,13 @@ describe('BaseProjectService', () => { [ 1, { - dataSources: [{startBlock: 1}, {startBlock: 200}], + dataSources: [{ startBlock: 1 }, { startBlock: 200 }], }, ], [ 100, { - dataSources: [{startBlock: 100}], + dataSources: [{ startBlock: 100 }], }, ], ], @@ -233,8 +235,8 @@ describe('BaseProjectService', () => { const result = service.getDataSourcesMap(); expect(result.getAll()).toEqual( new Map([ - [1, [{startBlock: 1}]], - [100, [{startBlock: 100}]], + [1, [{ startBlock: 1 }]], + [100, [{ startBlock: 100 }]], ]) ); }); @@ -246,13 +248,13 @@ describe('BaseProjectService', () => { [ 7408909, { - dataSources: [{startBlock: 7408909}], + dataSources: [{ startBlock: 7408909 }], }, ], [ 7880532, { - dataSources: [{startBlock: 7408909}], + dataSources: [{ startBlock: 7408909 }], }, ], ], @@ -261,8 +263,8 @@ describe('BaseProjectService', () => { const result = service.getDataSourcesMap(); expect(result.getAll()).toEqual( new Map([ - [7408909, [{startBlock: 7408909}]], - [7880532, [{startBlock: 7408909}]], + [7408909, [{ startBlock: 7408909 }]], + [7880532, [{ startBlock: 7408909 }]], ]) ); }); @@ -281,7 +283,7 @@ describe('BaseProjectService', () => { network: { chainId: '1', }, - dataSources: [{startBlock: 1}], + dataSources: [{ startBlock: 1 }], schema: buildSchemaFromString(`type TestEntity @entity { id: ID! fieldOne: String @@ -304,7 +306,7 @@ describe('BaseProjectService', () => { Promise.resolve(projects[parseInt(id, 10)]) ); - const nodeConfig = {unsafe: false} as unknown as NodeConfig; + const nodeConfig = { unsafe: false } as unknown as NodeConfig; const storeService = { init: jest.fn(), @@ -326,7 +328,7 @@ describe('BaseProjectService', () => { result = startBlock - 1; break; case 'deployments': - result = JSON.stringify({1: '1'}); + result = JSON.stringify({ 1: '1' }); break; default: result = undefined; @@ -348,7 +350,7 @@ describe('BaseProjectService', () => { { validateProjectCustomDatasources: jest.fn(), } as unknown as BaseDsProcessorService, // dsProcessorService - {networkMeta: {}} as unknown as any, //apiService + { networkMeta: {} } as unknown as any, //apiService null as unknown as any, // poiService null as unknown as any, // poiSyncService { @@ -382,7 +384,7 @@ describe('BaseProjectService', () => { { ...defaultProjects[0], id: '0', - parent: {block: 20, untilBlock: 20, reference: '1'}, + parent: { block: 20, untilBlock: 20, reference: '1' }, schema: buildSchemaFromString(`type TestEntity @entity { id: ID! fieldOne: String @@ -404,8 +406,8 @@ describe('BaseProjectService', () => { await setupProject( 95, [ - {blockHeight: 100, blockHash: 'a100', parentHash: 'a99'}, - {blockHeight: 99, blockHash: 'a99', parentHash: 'a98'}, + { blockHeight: 100, blockHash: 'a100', parentHash: 'a99', timestamp: new Date() }, + { blockHeight: 99, blockHash: 'a99', parentHash: 'a98', timestamp: new Date() }, ], 90 ); diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index 0ab35f227b..6e847091f0 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -2,26 +2,26 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {isMainThread} from 'worker_threads'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {BaseDataSource, IProjectNetworkConfig} from '@subql/types-core'; -import {Sequelize} from '@subql/x-sequelize'; -import {IApi} from '../api.service'; -import {IProjectUpgradeService, NodeConfig} from '../configure'; -import {IndexerEvent} from '../events'; -import {getLogger} from '../logger'; -import {exitWithError, monitorWrite} from '../process'; -import {getExistingProjectSchema, getStartHeight, hasValue, initDbSchema, mainThreadOnly, reindex} from '../utils'; -import {BlockHeightMap} from '../utils/blockHeightMap'; -import {BaseDsProcessorService} from './ds-processor.service'; -import {DynamicDsService} from './dynamic-ds.service'; -import {MetadataKeys} from './entities'; -import {PoiSyncService} from './poi'; -import {PoiService} from './poi/poi.service'; -import {StoreService} from './store.service'; -import {cacheProviderFlushData} from './storeModelProvider'; -import {ISubqueryProject, IProjectService, BypassBlocks} from './types'; -import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service'; +import { isMainThread } from 'worker_threads'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { BaseDataSource, IProjectNetworkConfig } from '@subql/types-core'; +import { Sequelize } from '@subql/x-sequelize'; +import { IApi } from '../api.service'; +import { IProjectUpgradeService, NodeConfig } from '../configure'; +import { IndexerEvent } from '../events'; +import { getLogger } from '../logger'; +import { exitWithError, monitorWrite } from '../process'; +import { getExistingProjectSchema, getStartHeight, hasValue, initDbSchema, mainThreadOnly, reindex } from '../utils'; +import { BlockHeightMap } from '../utils/blockHeightMap'; +import { BaseDsProcessorService } from './ds-processor.service'; +import { DynamicDsService } from './dynamic-ds.service'; +import { MetadataKeys } from './entities'; +import { PoiSyncService } from './poi'; +import { PoiService } from './poi/poi.service'; +import { StoreService } from './store.service'; +import { cacheProviderFlushData } from './storeModelProvider'; +import { ISubqueryProject, IProjectService, BypassBlocks, HistoricalMode, Header } from './types'; +import { IUnfinalizedBlocksService } from './unfinalizedBlocks.service'; const logger = getLogger('Project'); @@ -35,8 +35,7 @@ export abstract class BaseProjectService< API extends IApi, DS extends BaseDataSource, UnfinalizedBlocksService extends IUnfinalizedBlocksService = IUnfinalizedBlocksService, -> implements IProjectService -{ +> implements IProjectService { private _schema?: string; private _startHeight?: number; private _blockOffset?: number; @@ -88,7 +87,7 @@ export abstract class BaseProjectService< return this.project.network.bypassBlocks ?? []; } - protected get isHistorical(): boolean { + protected get isHistorical(): HistoricalMode { return this.storeService.historical; } @@ -142,7 +141,7 @@ export abstract class BaseProjectService< const reindexedUnfinalized = await this.initUnfinalizedInternal(); if (reindexedUnfinalized !== undefined) { - this._startHeight = reindexedUnfinalized; + this._startHeight = reindexedUnfinalized.blockHeight; } if (reindexedUpgrade !== undefined) { @@ -219,20 +218,20 @@ export abstract class BaseProjectService< const existing = await metadata.findMany(keys); - const {chain, genesisHash, specName} = this.apiService.networkMeta; + const { chain, genesisHash, specName } = this.apiService.networkMeta; if (this.project.runner) { - const {node, query} = this.project.runner; + const { node, query } = this.project.runner; - metadata.setBulk([ - {key: 'runnerNode', value: node.name}, - {key: 'runnerNodeVersion', value: node.version}, - {key: 'runnerQuery', value: query.name}, - {key: 'runnerQueryVersion', value: query.version}, + await metadata.setBulk([ + { key: 'runnerNode', value: node.name }, + { key: 'runnerNodeVersion', value: node.version }, + { key: 'runnerQuery', value: query.name }, + { key: 'runnerQueryVersion', value: query.version }, ]); } if (!existing.genesisHash) { - metadata.set('genesisHash', genesisHash); + await metadata.set('genesisHash', genesisHash); } else { // Check if the configured genesisHash matches the currently stored genesisHash assert( @@ -241,34 +240,34 @@ export abstract class BaseProjectService< ); } if (existing.chain !== chain) { - metadata.set('chain', chain); + await metadata.set('chain', chain); } if (existing.specName !== specName) { - metadata.set('specName', specName); + await metadata.set('specName', specName); } // If project was created before this feature, don't add the key. If it is project created after, add this key. if (!existing.processedBlockCount && !existing.lastProcessedHeight) { - metadata.set('processedBlockCount', 0); + await metadata.set('processedBlockCount', 0); } if (existing.indexerNodeVersion !== this.packageVersion) { - metadata.set('indexerNodeVersion', this.packageVersion); + await metadata.set('indexerNodeVersion', this.packageVersion); } if (!existing.schemaMigrationCount) { - metadata.set('schemaMigrationCount', 0); + await metadata.set('schemaMigrationCount', 0); } if (!existing.startHeight) { - metadata.set('startHeight', this.getStartBlockFromDataSources()); + await metadata.set('startHeight', this.getStartBlockFromDataSources()); } if (!existing.dynamicDatasources) { - metadata.set('dynamicDatasources', []); + await metadata.set('dynamicDatasources', []); } else if (typeof existing.dynamicDatasources === 'string') { // Migration Step: In versions < 4.7.2 dynamic datasources was stored as a string in a json field. logger.info('Migrating dynamic datasources from string to object'); - metadata.set('dynamicDatasources', JSON.parse(existing.dynamicDatasources)); + await metadata.set('dynamicDatasources', JSON.parse(existing.dynamicDatasources)); } } @@ -285,7 +284,6 @@ export abstract class BaseProjectService< return undefined; } - // @ts-ignore getStartBlockFromDataSources(): number { try { return getStartHeight(this.project.dataSources); @@ -342,7 +340,7 @@ export abstract class BaseProjectService< const nextProject = projects[i + 1][1]; nextMinStartHeight = Math.max( nextProject.dataSources - .filter((ds): ds is DS & {startBlock: number} => !!ds.startBlock) + .filter((ds): ds is DS & { startBlock: number } => !!ds.startBlock) .sort((a, b) => a.startBlock - b.startBlock)[0].startBlock, projects[i + 1][0] ); @@ -357,12 +355,12 @@ export abstract class BaseProjectService< }[] = []; [...project.dataSources, ...dynamicDs] - .filter((ds): ds is DS & {startBlock: number} => { + .filter((ds): ds is DS & { startBlock: number } => { return !!ds.startBlock && (!nextMinStartHeight || nextMinStartHeight > ds.startBlock); }) .forEach((ds) => { - events.push({block: Math.max(height, ds.startBlock), start: true, ds}); - if (ds.endBlock) events.push({block: ds.endBlock + 1, start: false, ds}); + events.push({ block: Math.max(height, ds.startBlock), start: true, ds }); + if (ds.endBlock) events.push({ block: ds.endBlock + 1, start: false, ds }); }); // sort events by block in ascending order, start events come before end events @@ -377,7 +375,7 @@ export abstract class BaseProjectService< return new BlockHeightMap(dsMap); } - private async initUnfinalizedInternal(): Promise { + private async initUnfinalizedInternal(): Promise
{ if (this.nodeConfig.unfinalizedBlocks && !this.isHistorical) { exitWithError( 'Unfinalized blocks cannot be enabled without historical. You will need to reindex your project to enable historical', @@ -388,7 +386,7 @@ export abstract class BaseProjectService< return this.initUnfinalized(); } - protected async initUnfinalized(): Promise { + protected async initUnfinalized(): Promise
{ return this.unfinalizedBlockService.init(this.reindex.bind(this)); } @@ -431,7 +429,13 @@ export abstract class BaseProjectService< const msg = `Rewinding project to preform project upgrade. Block height="${upgradePoint}"`; logger.info(msg); monitorWrite(msg); - await this.reindex(upgradePoint); + + const timestamp = await this.getBlockTimestamp(upgradePoint); + // Only timestamp and blockHeight are used with reindexing so its safe to convert to a header + await this.reindex({ + blockHeight: upgradePoint, + timestamp, + } as Header); return upgradePoint + 1; } } @@ -450,17 +454,20 @@ export abstract class BaseProjectService< await this.onProjectChange(this.project); } - async reindex(targetBlockHeight: number): Promise { - const lastProcessedHeight = await this.getLastProcessedHeight(); + async reindex(targetBlockHeader: Header): Promise { + const [height, timestamp] = await Promise.all([ + this.getLastProcessedHeight(), + this.storeService.modelProvider.metadata.find('lastProcessedBlockTimestamp'), + ]); - if (lastProcessedHeight === undefined) { + if (height === undefined) { throw new Error('Cannot reindex with missing lastProcessedHeight'); } return reindex( this.getStartBlockFromDataSources(), - targetBlockHeight, - lastProcessedHeight, + targetBlockHeader, + { height, timestamp }, this.storeService, this.unfinalizedBlockService, this.dynamicDsService, diff --git a/packages/node-core/src/indexer/store.service.test.ts b/packages/node-core/src/indexer/store.service.test.ts index 1b42fac89b..7ca16aeab5 100644 --- a/packages/node-core/src/indexer/store.service.test.ts +++ b/packages/node-core/src/indexer/store.service.test.ts @@ -1,13 +1,13 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {EventEmitter2} from '@nestjs/event-emitter'; -import {buildSchemaFromString} from '@subql/utils'; -import {Sequelize, QueryTypes} from '@subql/x-sequelize'; -import {NodeConfig} from '../configure'; -import {DbOption} from '../db'; -import {StoreService} from './store.service'; -import {CachedModel, PlainStoreModelService, StoreCacheService} from './storeModelProvider'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { buildSchemaFromString } from '@subql/utils'; +import { Sequelize, QueryTypes } from '@subql/x-sequelize'; +import { NodeConfig } from '../configure'; +import { DbOption } from '../db'; +import { StoreService } from './store.service'; +import { CachedModel, PlainStoreModelService, StoreCacheService } from './storeModelProvider'; const option: DbOption = { host: process.env.DB_HOST ?? '127.0.0.1', port: process.env.DB_PORT ? Number(process.env.DB_PORT) : 5432, @@ -38,8 +38,8 @@ describe('Check whether the db store and cache store are consistent.', () => { await sequelize.authenticate(); await sequelize.query(`CREATE SCHEMA ${testSchemaName};`); - const nodeConfig = new NodeConfig({subquery: 'test', proofOfIndex: true, enableCache: false}); - const project = {network: {chainId: '1'}, schema} as any; + const nodeConfig = new NodeConfig({ subquery: 'test', proofOfIndex: true, enableCache: false }); + const project = { network: { chainId: '1' }, schema } as any; const dbModel = new PlainStoreModelService(sequelize, nodeConfig); storeService = new StoreService(sequelize, nodeConfig, dbModel, project); await storeService.initCoreTables(testSchemaName); @@ -57,9 +57,9 @@ describe('Check whether the db store and cache store are consistent.', () => { }); it('Same block, Execute the set method multiple times.', async () => { - await storeService.setBlockHeight(1); + await storeService.setBlockHeader({ blockHeight: 1, blockHash: '0x01', parentHash: '0x00' }); - const accountEntity = {id: 'block-001', balance: 100}; + const accountEntity = { id: 'block-001', balance: 100 }; // account not exist. let account = await storeService.getStore().get('Account', accountEntity.id); @@ -72,21 +72,21 @@ describe('Check whether the db store and cache store are consistent.', () => { // block range check. const [dbData] = await sequelize.query(`SELECT * FROM "${testSchemaName}"."accounts" WHERE id = :id`, { - replacements: {id: accountEntity.id}, + replacements: { id: accountEntity.id }, type: QueryTypes.SELECT, transaction: storeService.transaction, }); expect(dbData._block_range).toEqual([ - {value: '1', inclusive: true}, - {value: null, inclusive: false}, + { value: '1', inclusive: true }, + { value: null, inclusive: false }, ]); // update account success. - const account001 = {id: 'block-001', balance: 10000}; + const account001 = { id: 'block-001', balance: 10000 }; await storeService.getStore().set('Account', account001.id, account001 as any); const account001After = await storeService.getStore().get('Account', account001.id); expect(account001After).toEqual(account001); - console.log({accountAfter: account001After, accountEntityAfter: account001}); + console.log({ accountAfter: account001After, accountEntityAfter: account001 }); // only one record in db and block range check. const allDatas = await sequelize.query(`SELECT * FROM "${testSchemaName}"."accounts"`, { @@ -95,11 +95,11 @@ describe('Check whether the db store and cache store are consistent.', () => { }); expect(allDatas).toHaveLength(1); expect(allDatas[0]._block_range).toEqual([ - {value: '1', inclusive: true}, - {value: null, inclusive: false}, + { value: '1', inclusive: true }, + { value: null, inclusive: false }, ]); - const account002 = {id: 'block-002', balance: 100}; + const account002 = { id: 'block-002', balance: 100 }; await storeService.getStore().bulkCreate('Account', [account002, account001]); const account002After = await storeService.getStore().get('Account', account002.id); expect(account002After).toEqual(account002); @@ -111,20 +111,20 @@ describe('Check whether the db store and cache store are consistent.', () => { }); expect(allDatas2).toHaveLength(2); expect(allDatas2[0]._block_range).toEqual([ - {value: '1', inclusive: true}, - {value: null, inclusive: false}, + { value: '1', inclusive: true }, + { value: null, inclusive: false }, ]); expect(allDatas2[1]._block_range).toEqual([ - {value: '1', inclusive: true}, - {value: null, inclusive: false}, + { value: '1', inclusive: true }, + { value: null, inclusive: false }, ]); }, 30000); it('_block_range update check', async () => { - await storeService.setBlockHeight(1000); + await storeService.setBlockHeader({ blockHeight: 1000, blockHash: '0x1000', parentHash: '0x0999' }); // insert new account. - const account1000Data = {id: 'block-1000', balance: 999}; + const account1000Data = { id: 'block-1000', balance: 999 }; await storeService.getStore().set('Account', account1000Data.id, account1000Data as any); const account1000 = await storeService.getStore().get('Account', account1000Data.id); expect(account1000).toEqual(account1000Data); @@ -136,7 +136,7 @@ describe('Check whether the db store and cache store are consistent.', () => { expect(allDatas).toHaveLength(3); // set old account. - const account002 = {id: 'block-002', balance: 222222}; + const account002 = { id: 'block-002', balance: 222222 }; await storeService.getStore().set('Account', account002.id, account002 as any); const account002After = await storeService.getStore().get('Account', account002.id); expect(account002After).toEqual(account002); @@ -153,12 +153,12 @@ describe('Check whether the db store and cache store are consistent.', () => { expect(account002Datas).toHaveLength(2); expect(account002Datas.map((v) => v._block_range).sort((a, b) => b[0].value - a[0].value)).toEqual([ [ - {value: '1000', inclusive: true}, - {value: null, inclusive: false}, + { value: '1000', inclusive: true }, + { value: null, inclusive: false }, ], [ - {value: '1', inclusive: true}, - {value: '1000', inclusive: false}, + { value: '1', inclusive: true }, + { value: '1000', inclusive: false }, ], ]); }, 100000); @@ -187,7 +187,7 @@ describe('Cache Provider', () => { storeCacheUpperLimit: 1, storeFlushInterval: 0, }); - const project = {network: {chainId: '1'}, schema} as any; + const project = { network: { chainId: '1' }, schema } as any; cacheModel = new StoreCacheService(sequelize, nodeConfig, new EventEmitter2(), null as any); storeService = new StoreService(sequelize, nodeConfig, cacheModel, project); await storeService.initCoreTables(testSchemaName); @@ -204,7 +204,7 @@ describe('Cache Provider', () => { tx.afterCommit(() => { Account.clear(blockHeight); }); - await storeService.setBlockHeight(blockHeight); + await storeService.setBlockHeader({ blockHeight, blockHash: `0x${blockHeight}`, parentHash: `0x${blockHeight - 1}` }); await handle(blockHeight); await Account.runFlush(tx, blockHeight); await tx.commit(); @@ -216,7 +216,7 @@ describe('Cache Provider', () => { type: QueryTypes.SELECT, }); - const accountEntity1 = {id: 'accountEntity-001', balance: 100}; + const accountEntity1 = { id: 'accountEntity-001', balance: 100 }; await cacheFlush(1, async (blockHeight) => { await Account.set(accountEntity1.id, accountEntity1, blockHeight); }); @@ -226,7 +226,7 @@ describe('Cache Provider', () => { expect(allDatas).toHaveLength(1); // next block 999 - const accountEntity2 = {id: 'accountEntity-002', balance: 9999}; + const accountEntity2 = { id: 'accountEntity-002', balance: 9999 }; await cacheFlush(999, async (blockHeight) => { await Account.remove(accountEntity1.id, blockHeight); const oldAccunt = await Account.get(accountEntity1.id); @@ -251,7 +251,7 @@ describe('Cache Provider', () => { oldAccunt2 = await Account.get(accountEntity2.id); expect(oldAccunt2).toBeUndefined(); - await Account.set(accountEntity2.id, {id: 'accountEntity-002', balance: 999999} as any, blockHeight); + await Account.set(accountEntity2.id, { id: 'accountEntity-002', balance: 999999 } as any, blockHeight); oldAccunt2 = await Account.get(accountEntity2.id); expect(oldAccunt2.balance).toEqual(999999); }); diff --git a/packages/node-core/src/indexer/store.service.ts b/packages/node-core/src/indexer/store.service.ts index a48119a815..306ea896e9 100644 --- a/packages/node-core/src/indexer/store.service.ts +++ b/packages/node-core/src/indexer/store.service.ts @@ -2,9 +2,8 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Inject, Injectable} from '@nestjs/common'; -import {getDbType, SUPPORT_DB} from '@subql/common'; -import {IProjectNetworkConfig} from '@subql/types-core'; +import { Inject, Injectable } from '@nestjs/common'; +import { IProjectNetworkConfig } from '@subql/types-core'; import { GraphQLModelsRelationsEnums, hashName, @@ -23,8 +22,8 @@ import { Transaction, Deferrable, } from '@subql/x-sequelize'; -import {camelCase, flatten, last, upperFirst} from 'lodash'; -import {NodeConfig} from '../configure'; +import { camelCase, flatten, last, upperFirst } from 'lodash'; +import { NodeConfig } from '../configure'; import { BTREE_GIST_EXTENSION_EXIST_QUERY, createSchemaTrigger, @@ -33,14 +32,14 @@ import { getTriggers, SchemaMigrationService, } from '../db'; -import {getLogger} from '../logger'; -import {exitWithError} from '../process'; -import {camelCaseObjectKey, customCamelCaseGraphqlKey} from '../utils'; -import {MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo} from './entities'; -import {Store} from './store'; -import {IMetadata, IStoreModelProvider, PlainStoreModelService} from './storeModelProvider'; -import {StoreOperations} from './StoreOperations'; -import {ISubqueryProject} from './types'; +import { getLogger } from '../logger'; +import { exitWithError } from '../process'; +import { camelCaseObjectKey, customCamelCaseGraphqlKey, getHistoricalUnit } from '../utils'; +import { MetadataFactory, MetadataRepo, PoiFactory, PoiFactoryDeprecate, PoiRepo } from './entities'; +import { Store } from './store'; +import { IMetadata, IStoreModelProvider, PlainStoreModelService } from './storeModelProvider'; +import { StoreOperations } from './StoreOperations'; +import { Header, HistoricalMode, ISubqueryProject } from './types'; const logger = getLogger('StoreService'); const NULL_MERKEL_ROOT = hexToU8a('0x00'); @@ -65,12 +64,11 @@ export class StoreService { private _modelIndexedFields?: IndexField[]; private _modelsRelations?: GraphQLModelsRelationsEnums; private _metaDataRepo?: MetadataRepo; - private _historical?: boolean; - private _dbType?: SUPPORT_DB; + private _historical?: HistoricalMode; private _metadataModel?: IMetadata; private _schema?: string; // Should be updated each block - private _blockHeight?: number; + private _blockHeader?: Header; private _operationStack?: StoreOperations; private _lastTimeDbSizeChecked?: number; @@ -106,12 +104,12 @@ export class StoreService { return this._operationStack; } - get blockHeight(): number { - assert(this._blockHeight, new Error('StoreService.setBlockHeight has not been called')); - return this._blockHeight; + get blockHeader(): Header { + assert(this._blockHeader, new Error('StoreService.setBlockHeader has not been called')); + return this._blockHeader; } - get historical(): boolean { + get historical(): HistoricalMode { assert(this._historical !== undefined, new NoInitError()); return this._historical; } @@ -136,11 +134,6 @@ export class StoreService { } } - private get dbType(): SUPPORT_DB { - assert(this._dbType, new NoInitError()); - return this._dbType; - } - private get metadataModel(): IMetadata { assert(this._metadataModel, new NoInitError()); return this._metadataModel; @@ -165,19 +158,14 @@ export class StoreService { this.subqueryProject.network.chainId ); - this._dbType = await getDbType(this.sequelize); this._schema = schema; await this.sequelize.sync(); this._historical = await this.getHistoricalStateEnabled(schema); - if (this.historical && this.dbType === SUPPORT_DB.cockRoach) { - this._historical = false; - logger.warn(`Historical feature is not supported with ${this.dbType}`); - } - logger.info(`Historical state is ${this.historical ? 'enabled' : 'disabled'}`); + logger.info(`Historical state is ${this.historical || 'disabled'}`); - this.modelProvider.init(this.historical, this.dbType === SUPPORT_DB.cockRoach, this.metaDataRepo, this.poiRepo); + this.modelProvider.init(this.historical, this.metaDataRepo, this.poiRepo); this._metadataModel = this.modelProvider.metadata; @@ -216,16 +204,11 @@ export class StoreService { await this.metadataModel.setIncrement('schemaMigrationCount'); } } catch (e: any) { - exitWithError(new Error(`Having a problem when syncing schema`, {cause: e}), logger); + exitWithError(new Error(`Having a problem when syncing schema`, { cause: e }), logger); } } private async initHotSchemaReloadQueries(schema: string): Promise { - if (this.dbType === SUPPORT_DB.cockRoach) { - logger.warn(`Hot schema reload feature is not supported with ${this.dbType}`); - return; - } - /* These SQL queries are to allow hot-schema reload on query service */ const schemaTriggerName = hashName(schema, 'schema_trigger', this.metaDataRepo.tableName); const schemaTriggers = await getTriggers(this.sequelize, schemaTriggerName); @@ -250,7 +233,7 @@ export class StoreService { try { this._modelIndexedFields = await this.getAllIndexFields(schema); } catch (e: any) { - exitWithError(new Error(`Having a problem when getting indexed fields`, {cause: e}), logger); + exitWithError(new Error(`Having a problem when getting indexed fields`, { cause: e }), logger); } } @@ -279,17 +262,17 @@ export class StoreService { sequelizeModel.addHook('beforeFind', (options) => { (options.where as any).__block_range = { - [Op.contains]: this.blockHeight as any, + [Op.contains]: this.getHistoricalUnit(), }; }); sequelizeModel.addHook('beforeValidate', (attributes, options) => { - attributes.__block_range = [this.blockHeight, null]; + attributes.__block_range = [this.getHistoricalUnit(), null]; }); if (!this.config.enableCache) { sequelizeModel.addHook('beforeBulkCreate', (instances, options) => { instances.forEach((item) => { - item.__block_range = [this.blockHeight, null]; + item.__block_range = [this.getHistoricalUnit(), null]; }); }); } @@ -302,77 +285,75 @@ export class StoreService { private async useDeprecatePoi(schema: string): Promise { const sql = `SELECT * FROM information_schema.columns WHERE table_schema = ? AND table_name = '_poi' AND column_name = 'projectId'`; - const [result] = await this.sequelize.query(sql, {replacements: [schema]}); + const [result] = await this.sequelize.query(sql, { replacements: [schema] }); return !!result.length; } - async getHistoricalStateEnabled(schema: string): Promise { - const {disableHistorical, multiChain} = this.config; + async getHistoricalStateEnabled(schema: string): Promise { + const { historical, multiChain } = this.config; try { const tableRes = await this.sequelize.query>( `SELECT table_name FROM information_schema.tables where table_schema='${schema}'`, - {type: QueryTypes.SELECT} + { type: QueryTypes.SELECT } ); const metadataTableNames = flatten(tableRes).filter( (value: string) => METADATA_REGEX.test(value) || MULTI_METADATA_REGEX.test(value) ); - if (metadataTableNames.length > 1 && !multiChain) { - exitWithError( - 'There are multiple projects in the database schema, if you are trying to multi-chain index use --multi-chain', - logger - ); + if (metadataTableNames.length <= 0) { + throw new Error('Metadata table does not exist'); } - if (metadataTableNames.length === 1) { - const res = await this.sequelize.query<{key: string; value: boolean | string}>( - `SELECT key, value FROM "${schema}"."${metadataTableNames[0]}" WHERE (key = 'historicalStateEnabled' OR key = 'genesisHash')`, - {type: QueryTypes.SELECT} - ); + const res = await this.sequelize.query<{ key: string; value: boolean | string }>( + `SELECT key, value FROM "${schema}"."${metadataTableNames[0]}" WHERE (key = 'historicalStateEnabled')`, + { type: QueryTypes.SELECT } + ); - const store = res.reduce( - function (total, current) { - total[current.key] = current.value; - return total; - }, - {} as {[key: string]: string | boolean} - ); + if (res[0]?.key !== 'historicalStateEnabled') { + throw new Error('Metadata table does not have historicalStateEnabled key'); + } - const useHistorical = - store.historicalStateEnabled === undefined ? !disableHistorical : (store.historicalStateEnabled as boolean); + const value = res[0].value; - if (useHistorical && multiChain) { - throw new Error( - 'Historical feature is enabled and not compatible with multi-chain, to multi-chain index clear postgres schema and re-index project using --multichain' - ); + if (typeof value === 'string') { + if (value === 'height' || value === 'timestamp') { + return value; } - return useHistorical; + throw new Error(`Invalid value for historicalStateEnabled. Received "${value}"`); } - throw new Error('Metadata table does not exist'); - } catch (e) { - if (multiChain && !disableHistorical) { - logger.info('Historical state is not compatible with multi chain indexing, disabling historical..'); - return false; + + if ((value === true || value.toString() === 'height') && multiChain) { + throw new Error( + 'Historical indexing by height is enabled and not compatible with multi-chain, to multi-chain index clear postgres schema and re-index project using --multichain' + ); } + // TODO parse through CLI/Project option and consider multichain + return value ? 'height' : false; + } catch (e) { + if (multiChain && historical === 'height') { + logger.warn('Historical state by height is not compatible with multi chain indexing, using timestamp instead.'); + return 'timestamp'; + } // Will trigger on first startup as metadata table doesn't exist - return !disableHistorical; + // Default fallback to "height" for backwards compatible + return historical; } } - async setBlockHeight(blockHeight: number): Promise { - this._blockHeight = blockHeight; + + async setBlockHeader(header: Header): Promise { + this._blockHeader = header; if (this.modelProvider instanceof PlainStoreModelService) { - assert(!this.#transaction, new Error(`Transaction already exists for height: ${blockHeight}`)); + assert(!this.#transaction, new Error(`Transaction already exists for height: ${header.blockHeight}`)); this.#transaction = await this.sequelize.transaction({ - deferrable: this._historical || this.dbType === SUPPORT_DB.cockRoach ? undefined : Deferrable.SET_DEFERRED(), + deferrable: this._historical ? undefined : Deferrable.SET_DEFERRED(), }); this.#transaction.afterCommit(() => (this.#transaction = undefined)); } - if (this.config.proofOfIndex) { this.operationStack = new StoreOperations(this.modelsRelations.models); } @@ -441,21 +422,25 @@ group by * @param targetBlockHeight * @param transaction */ - async rewind(targetBlockHeight: number, transaction: Transaction): Promise { + async rewind(targetBlockHeader: Header, transaction: Transaction): Promise { if (!this.historical) { throw new Error('Unable to reindex, historical state not enabled'); } // This should only been called from CLI, blockHeight in storeService never been set and is required for`beforeFind` hook // Height no need to change for rewind during indexing - if (this._blockHeight === undefined) { - await this.setBlockHeight(targetBlockHeight); + if (this._blockHeader === undefined) { + await this.setBlockHeader(targetBlockHeader); } for (const model of Object.values(this.sequelize.models)) { if ('__block_range' in model.getAttributes()) { - await batchDeleteAndThenUpdate(this.sequelize, model, transaction, targetBlockHeight); + await batchDeleteAndThenUpdate(this.sequelize, model, transaction, this.getHistoricalUnit()); } } - await this.metadataModel.set('lastProcessedHeight', targetBlockHeight, transaction); + + await this.metadataModel.set('lastProcessedHeight', targetBlockHeader.blockHeight, transaction); + if (targetBlockHeader.timestamp) { + await this.metadataModel.set('lastProcessedBlockTimestamp', targetBlockHeader.timestamp.getTime(), transaction); + } // metadataModel will be flushed in reindex.ts#reindex() } @@ -486,6 +471,12 @@ group by getStore(): Store { return new Store(this.config, this.modelProvider, this); } + + // Get the right unit depending on the historical mode + getHistoricalUnit(): number { + // Cant throw here because even with historical disabled the current height is used by the store + return getHistoricalUnit(this.historical, this.blockHeader); + } } // REMOVE 10,000 record per batch @@ -493,7 +484,7 @@ async function batchDeleteAndThenUpdate( sequelize: Sequelize, model: ModelStatic, transaction: Transaction, - targetBlockHeight: number, + targetBlockUnit: number, // Height or timestamp batchSize = 10000 ): Promise { let destroyCompleted = false; @@ -504,33 +495,33 @@ async function batchDeleteAndThenUpdate( destroyCompleted ? 0 : model.destroy({ - transaction, - hooks: false, - limit: batchSize, - where: sequelize.where(sequelize.fn('lower', sequelize.col('_block_range')), Op.gt, targetBlockHeight), - }), + transaction, + hooks: false, + limit: batchSize, + where: sequelize.where(sequelize.fn('lower', sequelize.col('_block_range')), Op.gt, targetBlockUnit), + }), updateCompleted ? [0] : model.update( - { - __block_range: sequelize.fn('int8range', sequelize.fn('lower', sequelize.col('_block_range')), null), - }, - { - transaction, - limit: batchSize, - hooks: false, - where: { - [Op.and]: [ - { - __block_range: { - [Op.contains]: targetBlockHeight, - }, + { + __block_range: sequelize.fn('int8range', sequelize.fn('lower', sequelize.col('_block_range')), null), + }, + { + transaction, + limit: batchSize, + hooks: false, + where: { + [Op.and]: [ + { + __block_range: { + [Op.contains]: targetBlockUnit, }, - sequelize.where(sequelize.fn('upper', sequelize.col('_block_range')), Op.not, null), - ], - }, - } - ), + }, + sequelize.where(sequelize.fn('upper', sequelize.col('_block_range')), Op.not, null), + ], + }, + } + ), ]); logger.debug(`${model.name} deleted ${numDestroyRows} records, updated ${numUpdatedRows} records`); if (numDestroyRows === 0) { diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index e5a13acb80..c305eccc01 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -2,20 +2,19 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Inject} from '@nestjs/common'; -import {Store as IStore, Entity, FieldsExpression, GetOptions} from '@subql/types-core'; -import {Transaction} from '@subql/x-sequelize'; -import {NodeConfig} from '../../configure'; -import {monitorWrite} from '../../process'; -import {handledStringify} from '../../utils'; -import {IStoreModelProvider} from '../storeModelProvider'; -import {StoreOperations} from '../StoreOperations'; -import {OperationType} from '../types'; -import {EntityClass} from './entity'; +import { Store as IStore, Entity, FieldsExpression, GetOptions } from '@subql/types-core'; +import { Transaction } from '@subql/x-sequelize'; +import { NodeConfig } from '../../configure'; +import { monitorWrite } from '../../process'; +import { handledStringify } from '../../utils'; +import { IStoreModelProvider } from '../storeModelProvider'; +import { StoreOperations } from '../StoreOperations'; +import { OperationType } from '../types'; +import { EntityClass } from './entity'; /* A context is provided to allow it to be updated by the owner of the class instance */ type Context = { - blockHeight: number; + getHistoricalUnit: () => number; transaction?: Transaction; operationStack?: StoreOperations; isIndexed: (entity: string, field: string) => boolean; @@ -102,7 +101,7 @@ export class Store implements IStore { assert(indexed, `to query by field ${String(field)}, a unique index must be created on model ${entity}`); const [raw] = await this.#modelProvider .getModel(entity) - .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], {limit: 1}); + .getByFields([Array.isArray(value) ? [field, 'in', value] : [field, '=', value]], { limit: 1 }); monitorWrite(() => `-- [Store][getOneByField] Entity ${entity}, data: ${handledStringify(raw)}`); return EntityClass.create(entity, raw, this); } catch (e) { @@ -112,10 +111,9 @@ export class Store implements IStore { async set(entity: string, _id: string, data: Entity): Promise { try { - await this.#modelProvider.getModel(entity).set(_id, data, this.#context.blockHeight, this.#context.transaction); - monitorWrite( - () => `-- [Store][set] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` - ); + const historicalUnit = this.#context.getHistoricalUnit(); + await this.#modelProvider.getModel(entity).set(_id, data, historicalUnit, this.#context.transaction); + monitorWrite(() => `-- [Store][set] Entity ${entity}, height: ${historicalUnit}, data: ${handledStringify(data)}`); this.#context.operationStack?.put(OperationType.Set, entity, data); } catch (e) { throw new Error(`Failed to set Entity ${entity} with _id ${_id}: ${e}`); @@ -124,13 +122,13 @@ export class Store implements IStore { async bulkCreate(entity: string, data: Entity[]): Promise { try { - await this.#modelProvider.getModel(entity).bulkCreate(data, this.#context.blockHeight, this.#context.transaction); + const historicalUnit = this.#context.getHistoricalUnit(); + await this.#modelProvider.getModel(entity).bulkCreate(data, historicalUnit, this.#context.transaction); for (const item of data) { this.#context.operationStack?.put(OperationType.Set, entity, item); } monitorWrite( - () => - `-- [Store][bulkCreate] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` + () => `-- [Store][bulkCreate] Entity ${entity}, height: ${historicalUnit}, data: ${handledStringify(data)}` ); } catch (e) { throw new Error(`Failed to bulkCreate Entity ${entity}: ${e}`); @@ -139,15 +137,13 @@ export class Store implements IStore { async bulkUpdate(entity: string, data: Entity[], fields?: string[]): Promise { try { - await this.#modelProvider - .getModel(entity) - .bulkUpdate(data, this.#context.blockHeight, fields, this.#context.transaction); + const historicalUnit = this.#context.getHistoricalUnit(); + await this.#modelProvider.getModel(entity).bulkUpdate(data, historicalUnit, fields, this.#context.transaction); for (const item of data) { this.#context.operationStack?.put(OperationType.Set, entity, item); } monitorWrite( - () => - `-- [Store][bulkUpdate] Entity ${entity}, height: ${this.#context.blockHeight}, data: ${handledStringify(data)}` + () => `-- [Store][bulkUpdate] Entity ${entity}, height: ${historicalUnit}, data: ${handledStringify(data)}` ); } catch (e) { throw new Error(`Failed to bulkCreate Entity ${entity}: ${e}`); @@ -155,25 +151,18 @@ export class Store implements IStore { } async remove(entity: string, id: string): Promise { - try { - await this.#modelProvider.getModel(entity).bulkRemove([id], this.#context.blockHeight, this.#context.transaction); - this.#context.operationStack?.put(OperationType.Remove, entity, id); - monitorWrite(() => `-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, id: ${id}`); - } catch (e) { - throw new Error(`Failed to remove Entity ${entity} with id ${id}: ${e}`); - } + return this.bulkRemove(entity, [id]); } async bulkRemove(entity: string, ids: string[]): Promise { try { - await this.#modelProvider.getModel(entity).bulkRemove(ids, this.#context.blockHeight, this.#context.transaction); + const historicalUnit = this.#context.getHistoricalUnit(); + await this.#modelProvider.getModel(entity).bulkRemove(ids, historicalUnit, this.#context.transaction); for (const id of ids) { this.#context.operationStack?.put(OperationType.Remove, entity, id); } - monitorWrite( - () => `-- [Store][remove] Entity ${entity}, height: ${this.#context.blockHeight}, ids: ${handledStringify(ids)}` - ); + monitorWrite(() => `-- [Store][remove] Entity ${entity}, height: ${historicalUnit}, ids: ${handledStringify(ids)}`); } catch (e) { throw new Error(`Failed to bulkRemove Entity ${entity}: ${e}`); } diff --git a/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts b/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts index 524662894d..cabe23b545 100644 --- a/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/baseStoreModel.service.ts @@ -1,31 +1,30 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {BeforeApplicationShutdown} from '@nestjs/common'; -import {getLogger} from '@subql/node-core/logger'; -import {ModelStatic} from '@subql/x-sequelize'; -import {MetadataRepo, PoiRepo} from '../entities'; -import {METADATA_ENTITY_NAME} from './metadata/utils'; -import {BaseEntity, IModel} from './model'; -import {POI_ENTITY_NAME} from './poi'; -import {Exporter} from './types'; +import { BeforeApplicationShutdown } from '@nestjs/common'; +import { getLogger } from '@subql/node-core/logger'; +import { ModelStatic } from '@subql/x-sequelize'; +import { MetadataRepo, PoiRepo } from '../entities'; +import { HistoricalMode } from '../types'; +import { METADATA_ENTITY_NAME } from './metadata/utils'; +import { BaseEntity, IModel } from './model'; +import { POI_ENTITY_NAME } from './poi'; +import { Exporter } from './types'; const logger = getLogger('BaseStoreModelService'); export abstract class BaseStoreModelService> implements BeforeApplicationShutdown { - protected historical = true; + protected historical: HistoricalMode = 'height'; protected poiRepo?: PoiRepo; protected metadataRepo?: MetadataRepo; protected cachedModels: Record = {}; - protected useCockroachDb?: boolean; protected exports: Exporter[] = []; protected abstract createModel(entity: string): M; - init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { + init(historical: HistoricalMode, meta: MetadataRepo, poi?: PoiRepo): void { this.historical = historical; this.metadataRepo = meta; this.poiRepo = poi; - this.useCockroachDb = useCockroachDb; } getModel(entity: string): IModel { @@ -41,7 +40,7 @@ export abstract class BaseStoreModelService> implements BeforeAp return this.cachedModels[entity] as IModel; } - updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void { + updateModels({ modifiedModels, removedModels }: { modifiedModels: ModelStatic[]; removedModels: string[] }): void { modifiedModels.forEach((m) => { this.cachedModels[m.name] = this.createModel(m.name); }); diff --git a/packages/node-core/src/indexer/storeModelProvider/cacheable.ts b/packages/node-core/src/indexer/storeModelProvider/cacheable.ts index dc3dd961f5..6defb9999c 100644 --- a/packages/node-core/src/indexer/storeModelProvider/cacheable.ts +++ b/packages/node-core/src/indexer/storeModelProvider/cacheable.ts @@ -7,19 +7,19 @@ import {Mutex} from 'async-mutex'; export abstract class Cacheable { protected mutex = new Mutex(); - abstract clear(blockHeight?: number): void; - protected abstract runFlush(tx: Transaction, blockHeight: number): Promise; + abstract clear(historicalUnit?: number): void; + protected abstract runFlush(tx: Transaction, historicalUnit: number): Promise; - async flush(tx: Transaction, blockHeight: number): Promise { + async flush(tx: Transaction, historicalUnit: number): Promise { const release = await this.mutex.acquire(); try { tx.afterCommit(() => { - this.clear(blockHeight); + this.clear(historicalUnit); release(); }); - const pendingFlush = this.runFlush(tx, blockHeight); + const pendingFlush = this.runFlush(tx, historicalUnit); await pendingFlush; } catch (e) { release(); diff --git a/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.spec.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.spec.ts index 519dfac1bf..eabc05c1e0 100644 --- a/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.spec.ts +++ b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.spec.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {CacheMetadataModel} from './cacheMetadata'; +import { CacheMetadataModel } from './cacheMetadata'; const incrementKey = 'processedBlockCount'; @@ -9,28 +9,28 @@ describe('CacheMetadata', () => { let cacheMetadata: CacheMetadataModel; beforeEach(() => { - cacheMetadata = new CacheMetadataModel(null as any); + cacheMetadata = new CacheMetadataModel(null as any, false); }); // Clearing the cache used to set the setCache and getCache to the same empty object // The set cache has increment amounts while the get cache has the actual value - it('clears the caches properly', () => { + it('clears the caches properly', async () => { cacheMetadata.clear(); (cacheMetadata as any).getCache[incrementKey] = 100; - cacheMetadata.setIncrement(incrementKey); + await cacheMetadata.setIncrement(incrementKey); expect((cacheMetadata as any).setCache[incrementKey]).toBe(1); }); // This tested a very specific use case where `cacheModel.getByFields`` was called on the start block which could trigger a flush and "lastProcessedHeight" was not yet set - it('clears the caches properly with blockHeight', () => { + it('clears the caches properly with blockHeight', async () => { cacheMetadata.clear(1); (cacheMetadata as any).getCache[incrementKey] = 100; - cacheMetadata.setIncrement(incrementKey); + await cacheMetadata.setIncrement(incrementKey); expect(Object.keys((cacheMetadata as any).setCache)).not.toContain('lastProcessedHeight'); }); @@ -38,14 +38,17 @@ describe('CacheMetadata', () => { it('builds the correct dynamicDatasources query', () => { const queryFn = jest.fn(); - const cacheMetadata = new CacheMetadataModel({ - getTableName: () => '"Schema"."_metadata"', - sequelize: { - query: queryFn, - }, - } as any); + const cacheMetadata = new CacheMetadataModel( + { + getTableName: () => '"Schema"."_metadata"', + sequelize: { + query: queryFn, + }, + } as any, + false + ); - (cacheMetadata as any).appendDynamicDatasources([{foo: 'bar'}]); + (cacheMetadata as any).appendDynamicDatasources([{ foo: 'bar' }]); expect(queryFn).toHaveBeenCalledWith( ` UPDATE "Schema"."_metadata" @@ -56,7 +59,7 @@ describe('CacheMetadata', () => { undefined ); - (cacheMetadata as any).appendDynamicDatasources([{foo: 'bar'}, {baz: 'buzz'}]); + (cacheMetadata as any).appendDynamicDatasources([{ foo: 'bar' }, { baz: 'buzz' }]); expect(queryFn).toHaveBeenCalledWith( ` UPDATE "Schema"."_metadata" diff --git a/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.test.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.test.ts index 73594431de..e262e0c4e5 100644 --- a/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.test.ts +++ b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.test.ts @@ -34,7 +34,7 @@ describe('cacheMetadata integration', () => { await metaDataRepo.sync(); - cacheMetadataModel = new CacheMetadataModel(metaDataRepo); + cacheMetadataModel = new CacheMetadataModel(metaDataRepo, false); }); const queryMeta = async (key: K): Promise => { diff --git a/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.ts b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.ts index d070a131c1..7816a6d58e 100644 --- a/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.ts +++ b/packages/node-core/src/indexer/storeModelProvider/metadata/cacheMetadata.ts @@ -2,14 +2,15 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Transaction} from '@subql/x-sequelize'; -import {hasValue} from '../../../utils'; -import {DatasourceParams} from '../../dynamic-ds.service'; -import {Metadata, MetadataKeys, MetadataRepo} from '../../entities'; -import {Cacheable} from '../cacheable'; -import {ICachedModelControl} from '../types'; -import {IMetadata} from './metadata'; -import {MetadataKey, incrementKeys, IncrementalMetadataKey, INCREMENT_QUERY, APPEND_DS_QUERY} from './utils'; +import { Transaction } from '@subql/x-sequelize'; +import { hasValue } from '../../../utils'; +import { DatasourceParams } from '../../dynamic-ds.service'; +import { Metadata, MetadataKeys, MetadataRepo } from '../../entities'; +import { HistoricalMode } from '../../types'; +import { Cacheable } from '../cacheable'; +import { ICachedModelControl } from '../types'; +import { IMetadata } from './metadata'; +import { MetadataKey, incrementKeys, IncrementalMetadataKey, INCREMENT_QUERY, APPEND_DS_QUERY } from './utils'; // type MetadataKey = keyof MetadataKeys; // const incrementKeys: MetadataKey[] = ['processedBlockCount', 'schemaMigrationCount']; @@ -28,7 +29,10 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM flushableRecordCounter = 0; - constructor(readonly model: MetadataRepo) { + constructor( + readonly model: MetadataRepo, + readonly historical: HistoricalMode = 'height' + ) { super(); } @@ -117,7 +121,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); - await this.model.sequelize.query(INCREMENT_QUERY(schemaTable, key, amount), tx && {transaction: tx}); + await this.model.sequelize.query(INCREMENT_QUERY(schemaTable, key, amount), tx && { transaction: tx }); } private async appendDynamicDatasources(items: DatasourceParams[], tx?: Transaction): Promise { @@ -125,7 +129,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM assert(this.model.sequelize, `Sequelize is not available on ${this.model.name}`); - await this.model.sequelize.query(APPEND_DS_QUERY(schemaTable, items), tx && {transaction: tx}); + await this.model.sequelize.query(APPEND_DS_QUERY(schemaTable, items), tx && { transaction: tx }); } private async handleSpecialKeys(tx?: Transaction): Promise { @@ -140,7 +144,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM **/ const val = this.setCache[key]; if (val !== undefined) { - await this.model.bulkCreate([{key, value: val}], {transaction: tx, updateOnDuplicate: ['key', 'value']}); + await this.model.bulkCreate([{ key, value: val }], { transaction: tx, updateOnDuplicate: ['key', 'value'] }); } else if (this.datasourceUpdates.length) { await this.appendDynamicDatasources(this.datasourceUpdates, tx); } @@ -167,16 +171,21 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM async runFlush(tx: Transaction, blockHeight?: number): Promise { const ops = Object.entries(this.setCache) .filter(([key]) => !specialKeys.includes(key as MetadataKey)) - .map(([key, value]) => ({key, value}) as Metadata); + .map(([key, value]) => ({ key, value }) as Metadata); - const lastProcessedHeightIdx = ops.findIndex((k) => k.key === 'lastProcessedHeight'); + const key = this.historical === 'timestamp' ? 'lastProcessedBlockTimestamp' : 'lastProcessedHeight'; + + const lastProcessedHeightIdx = ops.findIndex((k) => k.key === key); if (blockHeight !== undefined && lastProcessedHeightIdx >= 0) { const lastProcessedHeight = Number(ops[lastProcessedHeightIdx].value); // Before flush, lastProcessedHeight was obtained from metadata // During the flush, we are expecting metadata not being updated. Therefore, we exit here to ensure data accuracy and integrity. // This is unlikely happened. However, we need to observe how often this occurs, we need to adjust this logic if frequently. // Also, we can remove `lastCreatedPoiHeight` from metadata, as it will recreate again with indexing . - assert(blockHeight === lastProcessedHeight, 'metadata was updated before getting flushed'); + assert( + blockHeight === lastProcessedHeight, + `metadata was updated before getting flushed. BlockHeight="${blockHeight}", LastProcessed="${lastProcessedHeight}"` + ); } await Promise.all([ @@ -185,7 +194,7 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM updateOnDuplicate: ['key', 'value'], }), this.handleSpecialKeys(tx), - this.model.destroy({where: {key: this.removeCache}}), + this.model.destroy({ where: { key: this.removeCache } }), ]); } @@ -211,8 +220,8 @@ export class CacheMetadataModel extends Cacheable implements IMetadata, ICachedM newSetCache.lastProcessedHeight = this.setCache.lastProcessedHeight; this.flushableRecordCounter = 1; } - this.setCache = {...newSetCache}; - this.getCache = {...newSetCache}; + this.setCache = { ...newSetCache }; + this.getCache = { ...newSetCache }; this.datasourceUpdates = []; } } diff --git a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts index 5c1cb03b80..981c7d0a17 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.spec.ts @@ -54,7 +54,7 @@ jest.mock('@subql/x-sequelize', () => { count: 5, findAll: jest.fn(() => Object.values(data).map(({__block_range, ...d}) => ({toJSON: () => d}))), findOne: jest.fn(({transaction, where: {id}}) => ({ - toJSON: () => (transaction ? pendingData[id] ?? data[id] : data[id]), + toJSON: () => (transaction ? (pendingData[id] ?? data[id]) : data[id]), })), bulkCreate: jest.fn((records: {id: string}[]) => { records.map((r) => (pendingData[r.id] = r)); @@ -226,7 +226,7 @@ describe('cacheModel', () => { jest.clearAllMocks(); let i = 0; sequelize = new Sequelize(); - testModel = new CachedModel(sequelize.model('entity1'), true, {} as NodeConfig, () => i++); + testModel = new CachedModel(sequelize.model('entity1'), 'height', {} as NodeConfig, () => i++); }); it('throws when trying to set undefined', async () => { diff --git a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts index cc1699fb85..208119e531 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/cacheModel.ts @@ -2,16 +2,17 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {FieldsExpression, GetOptions} from '@subql/types-core'; -import {CreationAttributes, Model, ModelStatic, Op, Sequelize, Transaction} from '@subql/x-sequelize'; -import {flatten, uniq, cloneDeep, orderBy, unionBy} from 'lodash'; -import {NodeConfig} from '../../../configure'; -import {Cacheable} from '../cacheable'; -import {CsvStoreService} from '../csvStore.service'; -import {SetValueModel} from '../setValueModel'; -import {ICachedModelControl, RemoveValue, SetData, GetData, FilteredHeightRecords, SetValue, Exporter} from '../types'; -import {BaseEntity, IModel} from './model'; -import {getFullOptions, operatorsMap} from './utils'; +import { FieldsExpression, GetOptions } from '@subql/types-core'; +import { CreationAttributes, Model, ModelStatic, Op, Sequelize, Transaction } from '@subql/x-sequelize'; +import { flatten, uniq, cloneDeep, orderBy, unionBy } from 'lodash'; +import { NodeConfig } from '../../../configure'; +import { HistoricalMode } from '../../types'; +import { Cacheable } from '../cacheable'; +import { CsvStoreService } from '../csvStore.service'; +import { SetValueModel } from '../setValueModel'; +import { ICachedModelControl, RemoveValue, SetData, GetData, FilteredHeightRecords, SetValue, Exporter } from '../types'; +import { BaseEntity, IModel } from './model'; +import { getFullOptions, operatorsMap } from './utils'; const getCacheOptions = { max: 500, // default value @@ -21,8 +22,7 @@ const getCacheOptions = { export class CachedModel extends Cacheable - implements IModel, ICachedModelControl -{ + implements IModel, ICachedModelControl { // Null value indicates its not defined in the db private getCache: GetData; private setCache: SetData = {}; @@ -34,7 +34,7 @@ export class CachedModel constructor( readonly model: ModelStatic>, - private readonly historical = true, + private readonly historical: HistoricalMode = 'height', config: NodeConfig, private getNextStoreOperationIndex: () => number ) { @@ -82,7 +82,7 @@ export class CachedModel record = ( await this.model.findOne({ // https://github.com/sequelize/sequelize/issues/15179 - where: {id} as any, + where: { id } as any, }) )?.toJSON(); if (record) { @@ -152,8 +152,8 @@ export class CachedModel const records = await this.model.findAll({ where: { [Op.and]: [ - ...filters.map(([field, operator, value]) => ({[field]: {[operatorsMap[operator]]: value}})), - {id: {[Op.notIn]: this.allCachedIds()}}, + ...filters.map(([field, operator, value]) => ({ [field]: { [operatorsMap[operator]]: value } })), + { id: { [Op.notIn]: this.allCachedIds() } }, ] as any, // Types not working properly }, limit: fullOptions.limit - offsetCacheData.length, // Only get enough to fill the limit @@ -234,7 +234,7 @@ export class CachedModel async runFlush(tx: Transaction, blockHeight: number): Promise { // Get records relevant to the block height - const {removeRecords, setRecords} = this.filterRecordsWithHeight(blockHeight); + const { removeRecords, setRecords } = this.filterRecordsWithHeight(blockHeight); // Filter non-historical could return undefined due to it been removed let records = this.applyBlockRange(setRecords).filter((r) => !!r); let dbOperation: Promise; @@ -272,12 +272,12 @@ export class CachedModel dbOperation = Promise.all([ records.length && - this.model.bulkCreate(records, { - transaction: tx, - updateOnDuplicate: Object.keys(records[0]) as unknown as (keyof T)[], - }), + this.model.bulkCreate(records, { + transaction: tx, + updateOnDuplicate: Object.keys(records[0]) as unknown as (keyof T)[], + }), Object.keys(removeRecords).length && - this.model.destroy({where: {id: Object.keys(removeRecords)} as any, transaction: tx}), + this.model.destroy({ where: { id: Object.keys(removeRecords) } as any, transaction: tx }), ]); } this.exporters.forEach((store: Exporter) => { @@ -294,7 +294,7 @@ export class CachedModel (key) => this.removeCache[key].operationIndex === operationIndex ); if (removeRecordKey !== undefined) { - await this.model.destroy({where: {id: removeRecordKey} as any, transaction: tx}); + await this.model.destroy({ where: { id: removeRecordKey } as any, transaction: tx }); delete this.removeCache[removeRecordKey]; } else { let setRecord: SetValue | undefined; @@ -303,7 +303,7 @@ export class CachedModel if (setRecord) break; } if (setRecord) { - await this.model.upsert(setRecord.data as unknown as CreationAttributes>, {transaction: tx}); + await this.model.upsert(setRecord.data as unknown as CreationAttributes>, { transaction: tx }); } return; } @@ -401,15 +401,15 @@ export class CachedModel setRecords: SetData, removeRecords: Record ): Promise { - const closeSetRecords: {id: string; blockHeight: number}[] = []; + const closeSetRecords: { id: string; blockHeight: number }[] = []; for (const [id, value] of Object.entries(setRecords)) { const firstValue = value.getFirst(); if (firstValue !== undefined) { - closeSetRecords.push({id, blockHeight: firstValue.startHeight}); + closeSetRecords.push({ id, blockHeight: firstValue.startHeight }); } } const closeRemoveRecords = Object.entries(removeRecords).map(([id, value]) => { - return {id, blockHeight: value.removedAtBlock}; + return { id, blockHeight: value.removedAtBlock }; }); const mergedRecords = closeSetRecords.concat(closeRemoveRecords); @@ -420,10 +420,10 @@ export class CachedModel await this.sequelize.query( `UPDATE ${this.model.getTableName()} table1 SET _block_range = int8range(lower("_block_range"), table2._block_end) from (SELECT UNNEST(array[${mergedRecords.map((r) => - this.sequelize.escape(r.id) - )}]) AS id, UNNEST(array[${mergedRecords.map((r) => r.blockHeight)}]) AS _block_end) AS table2 + this.sequelize.escape(r.id) + )}]) AS id, UNNEST(array[${mergedRecords.map((r) => r.blockHeight)}]) AS _block_end) AS table2 WHERE table1.id = table2.id and "_block_range" @> _block_end-1::int8;`, - {transaction: tx} + { transaction: tx } ); } diff --git a/packages/node-core/src/indexer/storeModelProvider/model/model.ts b/packages/node-core/src/indexer/storeModelProvider/model/model.ts index 015cdc5e62..9d5f962d99 100644 --- a/packages/node-core/src/indexer/storeModelProvider/model/model.ts +++ b/packages/node-core/src/indexer/storeModelProvider/model/model.ts @@ -1,15 +1,15 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {FieldsExpression, GetOptions} from '@subql/types-core'; -import {Op, Model, ModelStatic, Transaction, CreationAttributes, Sequelize} from '@subql/x-sequelize'; -import {Fn} from '@subql/x-sequelize/types/utils'; -import _ from 'lodash'; -import {CsvStoreService} from '../csvStore.service'; -import {Exporter} from '../types'; -import {getFullOptions, operatorsMap} from './utils'; +import { FieldsExpression, GetOptions } from '@subql/types-core'; +import { Op, Model, ModelStatic, Transaction, CreationAttributes, Sequelize } from '@subql/x-sequelize'; +import { Fn } from '@subql/x-sequelize/types/utils'; +import { HistoricalMode } from '../../types'; +import { CsvStoreService } from '../csvStore.service'; +import { Exporter } from '../types'; +import { getFullOptions, operatorsMap } from './utils'; -export type BaseEntity = {id: string; __block_range?: (number | null)[] | Fn}; +export type BaseEntity = { id: string; __block_range?: (number | null)[] | Fn }; export interface IModel { get(id: string, tx?: Transaction): Promise; @@ -29,13 +29,13 @@ export class PlainModel implements IModel constructor( readonly model: ModelStatic>, - private readonly historical = true + private readonly historical: HistoricalMode = 'height' ) {} async get(id: string, tx?: Transaction): Promise { const record = await this.model.findOne({ // https://github.com/sequelize/sequelize/issues/15179 - where: {id} as any, + where: { id } as any, transaction: tx, }); @@ -47,7 +47,7 @@ export class PlainModel implements IModel // Query DB with all params const records = await this.model.findAll({ where: { - [Op.and]: [...filters.map(([field, operator, value]) => ({[field]: {[operatorsMap[operator]]: value}}))] as any, // Types not working properly + [Op.and]: [...filters.map(([field, operator, value]) => ({ [field]: { [operatorsMap[operator]]: value } }))] as any, // Types not working properly }, limit: fullOptions.limit, offset: fullOptions.offset, @@ -110,14 +110,14 @@ export class PlainModel implements IModel }); }); } else { - Promise.all(this.exporters.map(async (store: Exporter) => store.export(data))); + await Promise.all(this.exporters.map(async (store: Exporter) => store.export(data))); } } async bulkRemove(ids: string[], blockHeight: number, tx?: Transaction): Promise { if (!ids.length) return; if (!this.historical) { - await this.model.destroy({where: {id: ids} as any, transaction: tx}); + await this.model.destroy({ where: { id: ids } as any, transaction: tx }); } else { await this.markAsDeleted(ids, blockHeight, tx); } @@ -146,7 +146,7 @@ export class PlainModel implements IModel hooks: false, where: { id: ids, - __block_range: {[Op.contains]: blockHeight}, + __block_range: { [Op.contains]: blockHeight }, } as any, transaction: tx, } diff --git a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.spec.ts b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.spec.ts index 87efc16a10..6185fbd5e1 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.spec.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.spec.ts @@ -1,17 +1,17 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {EventEmitter2} from '@nestjs/event-emitter'; -import {SchedulerRegistry} from '@nestjs/schedule'; -import {Sequelize} from '@subql/x-sequelize'; -import {NodeConfig} from '../../configure'; -import {delay} from '../../utils'; -import {BaseEntity} from './model'; -import {StoreCacheService} from './storeCache.service'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { Sequelize } from '@subql/x-sequelize'; +import { NodeConfig } from '../../configure'; +import { delay } from '../../utils'; +import { BaseEntity } from './model'; +import { StoreCacheService } from './storeCache.service'; const eventEmitter = new EventEmitter2(); -type TestEntity = BaseEntity & {field1: string}; +type TestEntity = BaseEntity & { field1: string }; jest.mock('@subql/x-sequelize', () => { const mSequelize = { @@ -24,7 +24,7 @@ jest.mock('@subql/x-sequelize', () => { findOne: jest.fn(), create: (input: any) => input, }), - query: () => [{nextval: 1}], + query: () => [{ nextval: 1 }], showAllSchemas: () => ['subquery_1'], model: (entity: string) => ({ upsert: jest.fn(), @@ -152,7 +152,7 @@ describe('Store Cache flush with order', () => { beforeEach(() => { storeCacheService = new StoreCacheService(sequelize, nodeConfig, eventEmitter, new SchedulerRegistry()); - storeCacheService.init(false, true, {} as any, undefined); + storeCacheService.init(false, {} as any, undefined); }); it('when set/remove multiple model entities, operation index should added to record in sequential order', async () => { @@ -185,11 +185,11 @@ describe('Store Cache flush with non-historical', () => { let storeCacheService: StoreCacheService; const sequelize = new Sequelize(); - const nodeConfig: NodeConfig = {disableHistorical: true} as any; + const nodeConfig: NodeConfig = { disableHistorical: true } as any; beforeEach(() => { storeCacheService = new StoreCacheService(sequelize, nodeConfig, eventEmitter, new SchedulerRegistry()); - storeCacheService.init(false, false, {} as any, undefined); + storeCacheService.init(false, {} as any, undefined); }); it('Same Id with multiple operations, when flush it should always pick up the latest operation', async () => { @@ -227,12 +227,12 @@ describe('Store Cache flush with non-historical', () => { const spyModel1Destroy = jest.spyOn(sequelizeModel1, 'destroy'); // Only last set record with block 5 is created - expect(spyModel1Create).toHaveBeenCalledWith([{field1: 'set at block 5', id: 'entity1_id_0x01'}], { + expect(spyModel1Create).toHaveBeenCalledWith([{ field1: 'set at block 5', id: 'entity1_id_0x01' }], { transaction: tx, updateOnDuplicate: ['id', 'field1'], }); // remove id 2 only - expect(spyModel1Destroy).toHaveBeenCalledWith({transaction: tx, where: {id: ['entity1_id_0x02']}}); + expect(spyModel1Destroy).toHaveBeenCalledWith({ transaction: tx, where: { id: ['entity1_id_0x02'] } }); }); }); @@ -247,7 +247,7 @@ describe('Store cache upper threshold', () => { beforeEach(() => { storeCacheService = new StoreCacheService(sequelize, nodeConfig, eventEmitter, new SchedulerRegistry()); - storeCacheService.init(false, false, {findByPk: () => Promise.resolve({toJSON: () => 1})} as any, undefined); + storeCacheService.init(false, { findByPk: () => Promise.resolve({ toJSON: () => 1 }) } as any, undefined); }); it('doesnt wait for flushing cache when threshold not met', async () => { diff --git a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts index 8b65628ba3..55cfaab9fd 100644 --- a/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts +++ b/packages/node-core/src/indexer/storeModelProvider/storeCache.service.ts @@ -2,24 +2,25 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Injectable} from '@nestjs/common'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {SchedulerRegistry} from '@nestjs/schedule'; -import {DatabaseError, Deferrable, ModelStatic, Sequelize, Transaction} from '@subql/x-sequelize'; -import {sum} from 'lodash'; -import {NodeConfig} from '../../configure'; -import {IndexerEvent} from '../../events'; -import {getLogger} from '../../logger'; -import {exitWithError} from '../../process'; -import {profiler} from '../../profiler'; -import {MetadataRepo, PoiRepo} from '../entities'; -import {BaseCacheService} from './baseCache.service'; -import {CsvStoreService} from './csvStore.service'; -import {CacheMetadataModel} from './metadata'; -import {METADATA_ENTITY_NAME} from './metadata/utils'; -import {CachedModel} from './model'; -import {CachePoiModel, POI_ENTITY_NAME} from './poi'; -import {ICachedModelControl, IStoreModelProvider} from './types'; +import { Injectable } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { DatabaseError, Deferrable, Sequelize } from '@subql/x-sequelize'; +import { sum } from 'lodash'; +import { NodeConfig } from '../../configure'; +import { IndexerEvent } from '../../events'; +import { getLogger } from '../../logger'; +import { exitWithError } from '../../process'; +import { profiler } from '../../profiler'; +import { MetadataRepo, PoiRepo } from '../entities'; +import { HistoricalMode } from '../types'; +import { BaseCacheService } from './baseCache.service'; +import { CsvStoreService } from './csvStore.service'; +import { CacheMetadataModel } from './metadata'; +import { METADATA_ENTITY_NAME } from './metadata/utils'; +import { CachedModel } from './model'; +import { CachePoiModel, POI_ENTITY_NAME } from './poi'; +import { IStoreModelProvider } from './types'; const logger = getLogger('StoreCacheService'); @@ -28,7 +29,6 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr private readonly storeCacheThreshold: number; private readonly cacheUpperLimit: number; private _storeOperationIndex = 0; - private _lastFlushedOperationIndex = 0; constructor( private sequelize: Sequelize, @@ -45,8 +45,8 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr } } - init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void { - super.init(historical, useCockroachDb, meta, poi); + init(historical: HistoricalMode, meta: MetadataRepo, poi?: PoiRepo): void { + super.init(historical, meta, poi); if (this.config.storeFlushInterval > 0) { this.schedulerRegistry.addInterval( @@ -99,7 +99,7 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr if (!this.metadataRepo) { throw new Error('Metadata entity has not been set on store cache'); } - this.cachedModels[METADATA_ENTITY_NAME] = new CacheMetadataModel(this.metadataRepo); + this.cachedModels[METADATA_ENTITY_NAME] = new CacheMetadataModel(this.metadataRepo, this.historical); } return this.cachedModels[METADATA_ENTITY_NAME] as unknown as CacheMetadataModel; } @@ -116,41 +116,21 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr return this.cachedModels[POI_ENTITY_NAME] as unknown as CachePoiModel; } - private async flushRelationalModelsInOrder(updatableModels: ICachedModelControl[], tx: Transaction): Promise { - const relationalModels = updatableModels.filter((m) => m.hasAssociations); - // _storeOperationIndex could increase while we are still flushing - // therefore we need to store this index in memory first. - - const flushToIndex = this._storeOperationIndex; - for (let i = this._lastFlushedOperationIndex; i < flushToIndex; i++) { - // Flush operation can be a no-op if it doesn't have that index - await Promise.all(relationalModels.map((m) => m.flushOperation?.(i, tx))); - } - this._lastFlushedOperationIndex = flushToIndex; - } - @profiler() async _flushCache(): Promise { this.logger.debug('Flushing cache'); // With historical disabled we defer the constraints check so that it doesn't matter what order entities are modified const tx = await this.sequelize.transaction({ - deferrable: this.historical || this.useCockroachDb ? undefined : Deferrable.SET_DEFERRED(), + deferrable: this.historical ? undefined : Deferrable.SET_DEFERRED(), }); try { - // Get the block height of all data we want to flush up to - const blockHeight = await this.metadata.find('lastProcessedHeight'); + // Get the unit for historical + const historicalUnit = await this.metadata.find( + this.historical === 'timestamp' ? 'lastProcessedBlockTimestamp' : 'lastProcessedHeight' + ); // Get models that have data to flush const updatableModels = Object.values(this.cachedModels).filter((m) => m.isFlushable); - if (this.useCockroachDb) { - // 1. Independent(no associations) models can flush simultaneously - await Promise.all( - updatableModels.filter((m) => !m.hasAssociations).map((model) => model.flush(tx, blockHeight)) - ); - // 2. Models with associations will flush in orders, - await this.flushRelationalModelsInOrder(updatableModels, tx); - } else { - await Promise.all(updatableModels.map((model) => model.flush(tx, blockHeight))); - } + await Promise.all(updatableModels.map((model) => model.flush(tx, historicalUnit))); await tx.commit(); } catch (e: any) { if (e instanceof DatabaseError) { @@ -195,7 +175,7 @@ export class StoreCacheService extends BaseCacheService implements IStoreModelPr if (this.config.storeCacheAsync) { // Flush all completed block data and don't wait await this.flushAndWaitForCapacity(false)?.catch((e) => { - exitWithError(new Error(`Flushing cache failed`, {cause: e}), logger); + exitWithError(new Error(`Flushing cache failed`, { cause: e }), logger); }); } else { // Flush all data from cache and wait diff --git a/packages/node-core/src/indexer/storeModelProvider/types.ts b/packages/node-core/src/indexer/storeModelProvider/types.ts index 22dcb2232f..76544eeb8d 100644 --- a/packages/node-core/src/indexer/storeModelProvider/types.ts +++ b/packages/node-core/src/indexer/storeModelProvider/types.ts @@ -1,21 +1,22 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {ENUM, ModelStatic, Transaction} from '@subql/x-sequelize'; -import {LRUCache} from 'lru-cache'; -import {MetadataRepo, PoiRepo} from '../entities'; -import {IMetadata} from './metadata'; -import {BaseEntity, IModel} from './model'; -import {IPoi} from './poi'; -import {SetValueModel} from './setValueModel'; +import { ModelStatic, Transaction } from '@subql/x-sequelize'; +import { LRUCache } from 'lru-cache'; +import { MetadataRepo, PoiRepo } from '../entities'; +import { HistoricalMode } from '../types'; +import { IMetadata } from './metadata'; +import { BaseEntity, IModel } from './model'; +import { IPoi } from './poi'; +import { SetValueModel } from './setValueModel'; -export type HistoricalModel = {__block_range: any}; +export type HistoricalModel = { __block_range: any }; export interface IStoreModelProvider { poi: IPoi | null; metadata: IMetadata; - init(historical: boolean, useCockroachDb: boolean, meta: MetadataRepo, poi?: PoiRepo): void; + init(historical: HistoricalMode, meta: MetadataRepo, poi?: PoiRepo): void; getModel(entity: string): IModel; @@ -23,7 +24,7 @@ export interface IStoreModelProvider { applyPendingChanges(height: number, dataSourcesCompleted: boolean, tx?: Transaction): Promise; - updateModels({modifiedModels, removedModels}: {modifiedModels: ModelStatic[]; removedModels: string[]}): void; + updateModels({ modifiedModels, removedModels }: { modifiedModels: ModelStatic[]; removedModels: string[] }): void; } export interface ICachedModelControl { diff --git a/packages/node-core/src/indexer/test.runner.spec.ts b/packages/node-core/src/indexer/test.runner.spec.ts index c166f90ce2..0be0cd3846 100644 --- a/packages/node-core/src/indexer/test.runner.spec.ts +++ b/packages/node-core/src/indexer/test.runner.spec.ts @@ -1,14 +1,14 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Sequelize} from '@subql/x-sequelize'; -import {TestRunner} from './test.runner'; +import { Sequelize } from '@subql/x-sequelize'; +import { TestRunner } from './test.runner'; jest.mock('@subql/x-sequelize'); const mockStoreCache = { flushCache: jest.fn().mockResolvedValue(undefined), - metadata: {set: jest.fn()}, + metadata: { set: jest.fn() }, }; describe('TestRunner', () => { @@ -21,11 +21,11 @@ describe('TestRunner', () => { beforeEach(() => { sequelizeMock = new Sequelize() as any; apiServiceMock = { - fetchBlocks: jest.fn().mockResolvedValue([{}]), + fetchBlocks: jest.fn().mockResolvedValue([{ getHeader: jest.fn() }]), }; storeServiceMock = { - setBlockHeight: jest.fn(), + setBlockHeader: jest.fn(), getStore: jest.fn().mockReturnValue({}), modelProvider: mockStoreCache, }; @@ -82,18 +82,12 @@ describe('TestRunner', () => { }; const indexBlock = jest.fn().mockResolvedValue(undefined); - const storeMock = { + storeServiceMock.getStore = () => ({ get: jest.fn().mockResolvedValue(expectedEntity), - }; - (testRunner as any).storeService = { - getStore: () => storeMock, - setBlockHeight: jest.fn(), - modelProvider: mockStoreCache, - } as any; - - await testRunner.runTest(testMock, sandboxMock, indexBlock); + }); - expect((testRunner as any).passedTests).toBe(1); + const res = await testRunner.runTest(testMock, sandboxMock, indexBlock); + expect(res.passedTests).toBe(1); }); it('increments failedTests when expected and actual entity attributes do not match', async () => { @@ -117,18 +111,12 @@ describe('TestRunner', () => { }; const indexBlock = jest.fn().mockResolvedValue(undefined); - const storeMock = { + storeServiceMock.getStore = () => ({ get: jest.fn().mockResolvedValue(actualEntity), - }; - (testRunner as any).storeService = { - getStore: () => storeMock, - setBlockHeight: jest.fn(), - modelProvider: mockStoreCache, - } as any; - - await testRunner.runTest(testMock, sandboxMock, indexBlock); + }); - expect((testRunner as any).failedTests).toBe(1); + const res = await testRunner.runTest(testMock, sandboxMock, indexBlock); + expect(res.failedTests).toBe(1); }); it('increments failedTests when indexBlock throws an error', async () => { @@ -142,11 +130,11 @@ describe('TestRunner', () => { const indexBlock = jest.fn().mockRejectedValue(new Error('Test error')); - await testRunner.runTest(testMock, sandboxMock, indexBlock); + const res = await testRunner.runTest(testMock, sandboxMock, indexBlock); - expect((testRunner as any).failedTests).toBe(1); + expect(res.failedTests).toBe(1); - const summary = (testRunner as any).failedTestSummary; + const summary = res.failedTestSummary; expect(summary?.testName).toEqual(testMock.name); expect(summary?.entityId).toBeUndefined(); expect(summary?.entityName).toBeUndefined(); @@ -174,20 +162,16 @@ describe('TestRunner', () => { }; const indexBlock = jest.fn().mockResolvedValue(undefined); - const storeMock = { + + storeServiceMock.getStore = () => ({ get: jest.fn().mockResolvedValue(actualEntity), - }; - (testRunner as any).storeService = { - getStore: () => storeMock, - setBlockHeight: jest.fn(), - modelProvider: mockStoreCache, - } as any; + }); - await testRunner.runTest(testMock, sandboxMock, indexBlock); + const res = await testRunner.runTest(testMock, sandboxMock, indexBlock); - expect((testRunner as any).failedTests).toBe(1); + expect(res.failedTests).toBe(1); - const summary = (testRunner as any).failedTestSummary; + const summary = res.failedTestSummary; expect(summary?.testName).toEqual(testMock.name); expect(summary?.entityId).toEqual(expectedEntity.id); expect(summary?.entityName).toEqual(expectedEntity._name); diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 37df47af63..001c4de94d 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -1,18 +1,18 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Inject, Injectable} from '@nestjs/common'; -import {SubqlTest} from '@subql/testing'; -import {Sequelize} from '@subql/x-sequelize'; +import { Inject, Injectable } from '@nestjs/common'; +import { SubqlTest } from '@subql/testing'; +import { Sequelize } from '@subql/x-sequelize'; import chalk from 'chalk'; -import {isEqual} from 'lodash'; -import {IApi} from '../api.service'; -import {NodeConfig} from '../configure/NodeConfig'; -import {getLogger} from '../logger'; -import {TestSandbox} from './sandbox'; -import {StoreService} from './store.service'; -import {cacheProviderFlushData} from './storeModelProvider'; -import {IBlock, IIndexerManager} from './types'; +import { isEqual } from 'lodash'; +import { IApi } from '../api.service'; +import { NodeConfig } from '../configure/NodeConfig'; +import { getLogger } from '../logger'; +import { TestSandbox } from './sandbox'; +import { StoreService } from './store.service'; +import { cacheProviderFlushData } from './storeModelProvider'; +import { IBlock, IIndexerManager } from './types'; const logger = getLogger('test-runner'); @@ -57,7 +57,7 @@ export class TestRunner { logger.debug('Fetching block'); const [block] = await this.apiService.fetchBlocks([test.blockHeight]); - await this.storeService.setBlockHeight(test.blockHeight); + await this.storeService.setBlockHeader(block.getHeader()); // Ensure a block height is set so that data is flushed correctly await this.storeService.modelProvider.metadata.set('lastProcessedHeight', test.blockHeight - 1); const store = this.storeService.getStore(); diff --git a/packages/node-core/src/indexer/types.ts b/packages/node-core/src/indexer/types.ts index 3b22684897..b5ef954888 100644 --- a/packages/node-core/src/indexer/types.ts +++ b/packages/node-core/src/indexer/types.ts @@ -12,6 +12,14 @@ import {GraphQLSchema} from 'graphql'; import {BlockHeightMap} from '../utils/blockHeightMap'; import {ProcessBlockResponse} from './blockDispatcher'; +/** + * Indicates the mode of historical indexing + * false: No historical indexing + * height: Indexing is based on block height + * timestamp: Indexing is based on the unix timestamp in MS of the block + */ +export type HistoricalMode = false | 'height' | 'timestamp'; + export interface ISubqueryProject< N extends IProjectNetworkConfig = IProjectNetworkConfig, DS extends BaseDataSource = BaseDataSource, @@ -44,7 +52,7 @@ export interface IProjectService { blockOffset: number | undefined; startHeight: number; bypassBlocks: BypassBlocks; - reindex(lastCorrectHeight: number): Promise; + reindex(lastCorrectHeader: Header): Promise; /** * This is used everywhere but within indexing blocks, see comment on getDataSources for more info * */ @@ -68,6 +76,7 @@ export type Header = { blockHeight: number; blockHash: string; parentHash: string | undefined; + timestamp?: Date; }; export type BypassBlocks = (number | `${number}-${number}`)[]; diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts index a8fc670fe4..757acdf03d 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.spec.ts @@ -2,10 +2,10 @@ // SPDX-License-Identifier: GPL-3.0 // import { Header } from '@polkadot/types/interfaces'; -import {EventEmitter2} from '@nestjs/event-emitter'; -import {SchedulerRegistry} from '@nestjs/schedule'; -import {Header, IBlock} from '../indexer'; -import {StoreCacheService, CacheMetadataModel} from './storeModelProvider'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { Header, IBlock } from '../indexer'; +import { StoreCacheService, CacheMetadataModel } from './storeModelProvider'; import { METADATA_LAST_FINALIZED_PROCESSED_KEY, METADATA_UNFINALIZED_BLOCKS_KEY, @@ -22,6 +22,7 @@ class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService> blockHeight: 91, blockHash: `0xabc91f`, parentHash: `0xabc90f`, + timestamp: new Date(), }); } @@ -31,14 +32,16 @@ class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService> blockHeight: num, blockHash: hash, parentHash: `0xabc${num - 1}f`, + timestamp: new Date(), }); } - protected async getHeaderForHeight(height: number): Promise
{ + async getHeaderForHeight(height: number): Promise
{ return Promise.resolve({ blockHeight: height, blockHash: `0xabc${height}f`, parentHash: `0xabc${height - 1}f`, + timestamp: new Date(), }); } } @@ -46,8 +49,8 @@ class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService> function getMockMetadata(): any { const data: Record = {}; return { - upsert: ({key, value}: any) => (data[key] = value), - findOne: ({where: {key}}: any) => ({value: data[key]}), + upsert: ({ key, value }: any) => (data[key] = value), + findOne: ({ where: { key } }: any) => ({ value: data[key] }), findByPk: (key: string) => data[key], find: (key: string) => data[key], } as any; @@ -55,14 +58,14 @@ function getMockMetadata(): any { function mockStoreCache(): StoreCacheService { return { - metadata: new CacheMetadataModel(getMockMetadata()), + metadata: new CacheMetadataModel(getMockMetadata(), 'height'), } as StoreCacheService; } function mockBlock(height: number, hash: string, parentHash?: string): IBlock { return { getHeader: () => { - return {blockHeight: height, parentHash: parentHash ?? '', blockHash: hash}; + return { blockHeight: height, parentHash: parentHash ?? '', blockHash: hash, timestamp: new Date() }; }, block: { header: { @@ -78,7 +81,7 @@ describe('UnfinalizedBlocksService', () => { let unfinalizedBlocksService: UnfinalizedBlocksService; beforeEach(async () => { - unfinalizedBlocksService = new UnfinalizedBlocksService({unfinalizedBlocks: true} as any, mockStoreCache()); + unfinalizedBlocksService = new UnfinalizedBlocksService({ unfinalizedBlocks: true } as any, mockStoreCache()); await unfinalizedBlocksService.init(() => Promise.resolve()); }); @@ -107,7 +110,7 @@ describe('UnfinalizedBlocksService', () => { await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(111, '0xabc111')); await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(112, '0xabc112')); - expect((unfinalizedBlocksService as any).unfinalizedBlocks).toEqual([ + expect((unfinalizedBlocksService as any).unfinalizedBlocks).toMatchObject([ mockBlock(111, '0xabc111').block.header, mockBlock(112, '0xabc112').block.header, ]); @@ -132,7 +135,9 @@ describe('UnfinalizedBlocksService', () => { await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); - expect((unfinalizedBlocksService as any).unfinalizedBlocks).toEqual([mockBlock(113, '0xabc113').block.header]); + expect((unfinalizedBlocksService as any).unfinalizedBlocks).toMatchObject([ + mockBlock(113, '0xabc113').block.header, + ]); }); it('can handle a fork and rewind to the last finalized height', async () => { @@ -147,7 +152,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toBe(111); + expect(res).toMatchObject({blockHash: '0xabc111', blockHeight: 111, parentHash: ''}); // After this the call stack is something like: // indexerManager -> blockDispatcher -> project -> project -> reindex -> blockDispatcher.resetUnfinalizedBlocks @@ -172,7 +177,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(117, '0xabc117')); // Last valid block - expect(res).toBe(112); + expect(res).toMatchObject({blockHash: '0xabc112', blockHeight: 112, parentHash: ''}); }); it('can handle a fork when all unfinalized blocks are invalid', async () => { @@ -187,7 +192,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toBe(110); + expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); }); it('can handle a fork and when unfinalized blocks < finalized head', async () => { @@ -202,7 +207,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toBe(110); + expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); }); it('can handle a fork and when unfinalized blocks < finalized head 2', async () => { @@ -223,7 +228,7 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toBe(110); + expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); }); it('can handle a fork and when unfinalized blocks < finalized head with a large difference', async () => { @@ -238,35 +243,37 @@ describe('UnfinalizedBlocksService', () => { const res = await unfinalizedBlocksService.processUnfinalizedBlocks(mockBlock(113, '0xabc113')); // Last valid block - expect(res).toBe(110); + expect(res).toMatchObject({blockHash: '0xabc110f', blockHeight: 110, parentHash: '0xabc109f'}); }); it('can rewind any unfinalized blocks when restarted and unfinalized blocks is disabled', async () => { const storeCache = new StoreCacheService( null as any, - {storeCacheThreshold: 300} as any, + { storeCacheThreshold: 300 } as any, new EventEmitter2(), new SchedulerRegistry() ); - storeCache.init(true, false, {} as any, undefined); + storeCache.init('height', {} as any, undefined); await storeCache.metadata.set( METADATA_UNFINALIZED_BLOCKS_KEY, JSON.stringify([ - {blockHeight: 90, blockHash: '0xabcd'}, - {blockHeight: 91, blockHash: '0xabc91'}, - {blockHeight: 92, blockHash: '0xabc92'}, + { blockHeight: 90, blockHash: '0xabcd' }, + { blockHeight: 91, blockHash: '0xabc91' }, + { blockHeight: 92, blockHash: '0xabc92' }, ]) ); await storeCache.metadata.set(METADATA_LAST_FINALIZED_PROCESSED_KEY, 90); - const unfinalizedBlocksService2 = new UnfinalizedBlocksService({unfinalizedBlocks: false} as any, storeCache); + const unfinalizedBlocksService2 = new UnfinalizedBlocksService({ unfinalizedBlocks: false } as any, storeCache); const reindex = jest.fn().mockReturnValue(Promise.resolve()); await unfinalizedBlocksService2.init(reindex); - expect(reindex).toHaveBeenCalledWith(90); + expect(reindex).toHaveBeenCalledWith( + expect.objectContaining({blockHash: '0xabc90f', blockHeight: 90, parentHash: '0xabc89f'}) + ); expect((unfinalizedBlocksService2 as any).lastCheckedBlockHeight).toBe(90); }); }); diff --git a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts index 51d20f3ecc..b1724b2653 100644 --- a/packages/node-core/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node-core/src/indexer/unfinalizedBlocks.service.ts @@ -2,16 +2,16 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Transaction} from '@subql/x-sequelize'; -import {isEqual, last} from 'lodash'; -import {NodeConfig} from '../configure'; -import {Header, IBlock} from '../indexer/types'; -import {getLogger} from '../logger'; -import {exitWithError} from '../process'; -import {mainThreadOnly} from '../utils'; -import {ProofOfIndex} from './entities'; -import {PoiBlock} from './poi'; -import {IStoreModelProvider} from './storeModelProvider'; +import { Transaction } from '@subql/x-sequelize'; +import { isEqual, last } from 'lodash'; +import { NodeConfig } from '../configure'; +import { Header, IBlock } from '../indexer/types'; +import { getLogger } from '../logger'; +import { exitWithError } from '../process'; +import { mainThreadOnly } from '../utils'; +import { ProofOfIndex } from './entities'; +import { PoiBlock } from './poi'; +import { IStoreModelProvider } from './storeModelProvider'; const logger = getLogger('UnfinalizedBlocks'); @@ -25,12 +25,15 @@ const UNFINALIZED_THRESHOLD = 200; type UnfinalizedBlocks = Header[]; export interface IUnfinalizedBlocksService extends IUnfinalizedBlocksServiceUtil { - init(reindex: (targetHeight: number) => Promise): Promise; - processUnfinalizedBlocks(block: IBlock | undefined): Promise; - processUnfinalizedBlockHeader(header: Header | undefined): Promise; - resetUnfinalizedBlocks(tx?: Transaction): Promise; - resetLastFinalizedVerifiedHeight(tx?: Transaction): Promise; + init(reindex: (targetHeader: Header) => Promise): Promise
; + processUnfinalizedBlocks(block: IBlock | undefined): Promise
; + processUnfinalizedBlockHeader(header: Header | undefined): Promise
; + resetUnfinalizedBlocks(tx?: Transaction): void; + resetLastFinalizedVerifiedHeight(tx?: Transaction): void; getMetadataUnfinalizedBlocks(): Promise; + + // Used by reindex service + getHeaderForHeight(height: number): Promise
; } export interface IUnfinalizedBlocksServiceUtil { @@ -45,7 +48,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo // protected abstract blockToHeader(block: B): Header; protected abstract getFinalizedHead(): Promise
; protected abstract getHeaderForHash(hash: string): Promise
; - protected abstract getHeaderForHeight(height: number): Promise
; + abstract getHeaderForHeight(height: number): Promise
; @mainThreadOnly() protected blockToHeader(block: IBlock): Header { @@ -75,7 +78,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo protected readonly storeModelProvider: IStoreModelProvider ) {} - async init(reindex: (targetHeight: number) => Promise): Promise { + async init(reindex: (tagetHeader: Header) => Promise): Promise
{ logger.info(`Unfinalized blocks is ${this.nodeConfig.unfinalizedBlocks ? 'enabled' : 'disabled'}`); this.unfinalizedBlocks = await this.getMetadataUnfinalizedBlocks(); @@ -105,7 +108,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo return this.finalizedHeader.blockHeight; } - async processUnfinalizedBlockHeader(header?: Header): Promise { + async processUnfinalizedBlockHeader(header?: Header): Promise
{ if (header) { await this.registerUnfinalizedBlock(header); } @@ -123,7 +126,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo return; } - async processUnfinalizedBlocks(block?: IBlock): Promise { + async processUnfinalizedBlocks(block?: IBlock): Promise
{ return this.processUnfinalizedBlockHeader(block ? this.blockToHeader(block) : undefined); } @@ -161,7 +164,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo // remove any records less and equal than input finalized blockHeight private removeFinalized(blockHeight: number): void { - this.unfinalizedBlocks = this.unfinalizedBlocks.filter(({blockHeight: height}) => height > blockHeight); + this.unfinalizedBlocks = this.unfinalizedBlocks.filter(({ blockHeight: height }) => height > blockHeight); } // find closest record from block heights @@ -169,7 +172,7 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo // Have the block in the best block, can be verified return [...this.unfinalizedBlocks] // Copy so we can reverse .reverse() // Reverse the list to find the largest block - .find(({blockHeight: height}) => height <= blockHeight); + .find(({ blockHeight: height }) => height <= blockHeight); } // check unfinalized blocks for a fork, returns the header where a fork happened @@ -220,17 +223,17 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo return; } - protected async getLastCorrectFinalizedBlock(forkedHeader: Header): Promise { + protected async getLastCorrectFinalizedBlock(forkedHeader: Header): Promise
{ const bestVerifiableBlocks = this.unfinalizedBlocks.filter( - ({blockHeight}) => blockHeight <= this.finalizedBlockNumber + ({ blockHeight }) => blockHeight <= this.finalizedBlockNumber ); let checkingHeader = forkedHeader; // Work backwards through the blocks until we find a matching hash - for (const {blockHash, blockHeight} of bestVerifiableBlocks.reverse()) { - if (blockHash === checkingHeader.blockHash || blockHash === checkingHeader.parentHash) { - return blockHeight; + for (const bestHeader of bestVerifiableBlocks.reverse()) { + if (bestHeader.blockHash === checkingHeader.blockHash || bestHeader.blockHash === checkingHeader.parentHash) { + return bestHeader; } // Get the new parent @@ -238,7 +241,11 @@ export abstract class BaseUnfinalizedBlocksService implements IUnfinalizedBlo checkingHeader = await this.getHeaderForHash(checkingHeader.parentHash); } - return this.lastCheckedBlockHeight; + if (!this.lastCheckedBlockHeight) { + return undefined; + } + + return this.getHeaderForHeight(this.lastCheckedBlockHeight); } // Finds the last POI that had a correct block hash, this is used with the Eth sdk diff --git a/packages/node-core/src/indexer/worker/worker.service.ts b/packages/node-core/src/indexer/worker/worker.service.ts index 129645d543..9e8581c443 100644 --- a/packages/node-core/src/indexer/worker/worker.service.ts +++ b/packages/node-core/src/indexer/worker/worker.service.ts @@ -7,7 +7,7 @@ import {getLogger} from '../../logger'; import {monitorWrite} from '../../process'; import {AutoQueue, isTaskFlushedError, memoryLock} from '../../utils'; import {ProcessBlockResponse} from '../blockDispatcher'; -import {IBlock, IProjectService} from '../types'; +import {Header, IBlock, IProjectService} from '../types'; import {isBlockUnavailableError} from './utils'; export type FetchBlockResponse = {specVersion: number; parentHash: string} | undefined; @@ -23,9 +23,9 @@ const logger = getLogger(`WorkerService`); export abstract class BaseWorkerService< B /* BlockContent */, - R /* FetchBlockResponse */, + R extends Header /* FetchBlockResponse */, DS extends BaseDataSource = BaseDataSource, - E = {} /* Extra params for fetching blocks. Substrate uses specVersion in here*/ + E = {} /* Extra params for fetching blocks. Substrate uses specVersion in here*/, > { private fetchedBlocks: Record> = {}; private _isIndexing = false; @@ -44,7 +44,7 @@ export abstract class BaseWorkerService< this.queue = new AutoQueue(undefined, nodeConfig.batchSize, nodeConfig.timeout, 'Worker Service'); } - async fetchBlock(height: number, extra: E): Promise { + async fetchBlock(height: number, extra: E): Promise { try { return await this.queue.put(async () => { // If a dynamic ds is created we might be asked to fetch blocks again, use existing result @@ -65,10 +65,9 @@ export abstract class BaseWorkerService< return this.toBlockResponse(block.block); }); } catch (e: any) { - if (isTaskFlushedError(e)) { - return; + if (!isTaskFlushedError(e)) { + logger.error(e, `Failed to fetch block ${height}`); } - logger.error(e, `Failed to fetch block ${height}`); throw e; } } diff --git a/packages/node-core/src/indexer/worker/worker.ts b/packages/node-core/src/indexer/worker/worker.ts index 7384d8f65c..7d3854ec3e 100644 --- a/packages/node-core/src/indexer/worker/worker.ts +++ b/packages/node-core/src/indexer/worker/worker.ts @@ -13,6 +13,7 @@ import {ProcessBlockResponse} from '../blockDispatcher'; import {ConnectionPoolStateManager} from '../connectionPoolState.manager'; import {IDynamicDsService} from '../dynamic-ds.service'; import {MonitorServiceInterface} from '../monitor.service'; +import {Header} from '../types'; import {IUnfinalizedBlocksService} from '../unfinalizedBlocks.service'; import {hostMonitorKeys, monitorHostFunctions} from '../worker'; import {WorkerHost, Worker, AsyncMethods} from './worker.builder'; @@ -29,15 +30,15 @@ import {HostUnfinalizedBlocks, hostUnfinalizedBlocksKeys} from './worker.unfinal export type DefaultWorkerFunctions< ApiConnection /* ApiPromiseConnection*/, - DS extends BaseDataSource = BaseDataSource + DS extends BaseDataSource = BaseDataSource, > = HostCache & HostStore & HostDynamicDS & HostUnfinalizedBlocks & HostConnectionPoolState; let workerApp: INestApplication; -let workerService: BaseWorkerService; +let workerService: BaseWorkerService; const logger = getLogger(`worker #${threadId}`); -export function initWorkerServices(worker: INestApplication, service: BaseWorkerService): void { +export function initWorkerServices(worker: INestApplication, service: typeof workerService): void { if (workerApp) { logger.warn('Worker already initialised'); return; @@ -51,7 +52,7 @@ export function getWorkerApp(): INestApplication { return workerApp; } -export function getWorkerService>(): S { +export function getWorkerService(): S { assert(workerService, 'Worker Not initialised'); return workerService as S; } @@ -79,7 +80,7 @@ async function getStatus(): Promise { }; } -async function fetchBlock(height: number, specVersion: number): Promise { +async function fetchBlock(height: number, specVersion: number): ReturnType<(typeof workerService)['fetchBlock']> { assert(workerService, 'Worker Not initialised'); return workerService.fetchBlock(height, {specVersion}); } @@ -142,7 +143,7 @@ export function createWorkerHost< T extends AsyncMethods, H extends AsyncMethods & {initWorker: (height?: number) => Promise}, ApiConnection /* ApiPromiseConnection*/, - DS extends BaseDataSource = BaseDataSource + DS extends BaseDataSource = BaseDataSource, >(extraWorkerFns: (keyof T)[], extraHostFns: H): WorkerHost & T> { // Register these functions to be exposed to worker host return WorkerHost.create & T, IBaseIndexerWorker & H>( @@ -176,7 +177,7 @@ export async function createIndexerWorker< T extends IBaseIndexerWorker, ApiConnection extends IApiConnectionSpecific /*ApiPromiseConnection*/ /*ApiPromiseConnection*/, B, - DS extends BaseDataSource = BaseDataSource + DS extends BaseDataSource = BaseDataSource, >( workerPath: string, workerFns: (keyof Omit)[], diff --git a/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts b/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts index adfd3262de..884f05c5e8 100644 --- a/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts +++ b/packages/node-core/src/indexer/worker/worker.unfinalizedBlocks.service.ts @@ -6,7 +6,7 @@ import {Injectable} from '@nestjs/common'; import {Header, IBlock, IUnfinalizedBlocksService} from '../../indexer'; export type HostUnfinalizedBlocks = { - unfinalizedBlocksProcess: (header: Header | undefined) => Promise; + unfinalizedBlocksProcess: (header: Header | undefined) => Promise
; }; export const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[] = ['unfinalizedBlocksProcess']; @@ -19,16 +19,16 @@ export class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksServ } } - async processUnfinalizedBlocks(block: IBlock): Promise { + async processUnfinalizedBlocks(block: IBlock): Promise
{ return this.host.unfinalizedBlocksProcess(block.getHeader()); } - async processUnfinalizedBlockHeader(header: Header | undefined): Promise { + async processUnfinalizedBlockHeader(header: Header | undefined): Promise
{ return this.host.unfinalizedBlocksProcess(header); } // eslint-disable-next-line @typescript-eslint/promise-function-async - init(reindex: (targetHeight: number) => Promise): Promise { + init(reindex: (targetHeader: Header) => Promise): Promise
{ throw new Error('This method should not be called from a worker'); } // eslint-disable-next-line @typescript-eslint/require-await @@ -48,4 +48,9 @@ export class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksServ registerFinalizedBlock(header: Header): void { throw new Error('This method should not be called from a worker'); } + + // eslint-disable-next-line @typescript-eslint/promise-function-async + getHeaderForHeight(height: number): Promise
{ + throw new Error('This method should not be called from a worker'); + } } diff --git a/packages/node-core/src/subcommands/reindex.init.ts b/packages/node-core/src/subcommands/reindex.init.ts index ad8ea80eca..949c81fda6 100644 --- a/packages/node-core/src/subcommands/reindex.init.ts +++ b/packages/node-core/src/subcommands/reindex.init.ts @@ -15,13 +15,14 @@ export async function reindexInit(reindexModule: any, targetHeight: number): Pro const reindexService = app.get(ReindexService); await reindexService.init(); - const actualReindexHeight = await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight); - if (actualReindexHeight !== targetHeight) { + const actualReindex = await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight); + if (actualReindex.blockHeight !== targetHeight) { logger.info( - `Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}` + `Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindex.blockHeight}, will index to ${actualReindex.blockHeight}` ); } - await reindexService.reindex(actualReindexHeight); + + await reindexService.reindex(actualReindex); } catch (e: any) { exitWithError(new Error(`Reindex failed to execute`, {cause: e}), logger); } diff --git a/packages/node-core/src/subcommands/reindex.service.ts b/packages/node-core/src/subcommands/reindex.service.ts index d35cf42af5..f117172155 100644 --- a/packages/node-core/src/subcommands/reindex.service.ts +++ b/packages/node-core/src/subcommands/reindex.service.ts @@ -2,10 +2,10 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import {Inject, Injectable} from '@nestjs/common'; -import {BaseDataSource} from '@subql/types-core'; -import {Sequelize} from '@subql/x-sequelize'; -import {NodeConfig, ProjectUpgradeService} from '../configure'; +import { Inject, Injectable } from '@nestjs/common'; +import { BaseDataSource } from '@subql/types-core'; +import { Sequelize } from '@subql/x-sequelize'; +import { NodeConfig, ProjectUpgradeService } from '../configure'; import { IUnfinalizedBlocksService, StoreService, @@ -13,19 +13,20 @@ import { PoiService, IMetadata, cacheProviderFlushData, + Header, } from '../indexer'; -import {DynamicDsService} from '../indexer/dynamic-ds.service'; -import {getLogger} from '../logger'; -import {exitWithError, monitorWrite} from '../process'; -import {getExistingProjectSchema, initDbSchema, reindex} from '../utils'; -import {ForceCleanService} from './forceClean.service'; +import { DynamicDsService } from '../indexer/dynamic-ds.service'; +import { getLogger } from '../logger'; +import { exitWithError, monitorWrite } from '../process'; +import { getExistingProjectSchema, initDbSchema, reindex } from '../utils'; +import { ForceCleanService } from './forceClean.service'; const logger = getLogger('Reindex'); @Injectable() export class ReindexService

{ private _metadataRepo?: IMetadata; - private _lastProcessedHeight?: number; + private _lastProcessedHeader?: { height: number; timestamp?: number }; constructor( private readonly sequelize: Sequelize, @@ -62,38 +63,47 @@ export class ReindexService

{ + async getTargetHeightWithUnfinalizedBlocks(inputHeight: number): Promise

{ + const inputHeader = await this.unfinalizedBlocksService.getHeaderForHeight(inputHeight); + const unfinalizedBlocks = await this.unfinalizedBlocksService.getMetadataUnfinalizedBlocks(); - const bestBlocks = unfinalizedBlocks.filter(({blockHeight}) => blockHeight <= inputHeight); - if (bestBlocks.length === 0) { - return inputHeight; + const bestBlocks = unfinalizedBlocks.filter(({ blockHeight }) => blockHeight <= inputHeight); + if (bestBlocks.length && inputHeight >= bestBlocks[0].blockHeight) { + return bestBlocks[0]; } - const {blockHeight: firstBestBlock} = bestBlocks[0]; - return Math.min(inputHeight, firstBestBlock); + return inputHeader; } private async getExistingProjectSchema(): Promise { return getExistingProjectSchema(this.nodeConfig, this.sequelize); } - get lastProcessedHeight(): number { - assert(this._lastProcessedHeight !== undefined, 'Cannot reindex without lastProcessedHeight been initialized'); - return this._lastProcessedHeight; - } - - private async getLastProcessedHeight(): Promise { - return this.metadataRepo.find('lastProcessedHeight'); + get lastProcessedHeader(): { height: number; timestamp?: number } { + assert(this._lastProcessedHeader !== undefined, 'Cannot reindex without lastProcessedHeight been initialized'); + return this._lastProcessedHeader; } private getStartBlockFromDataSources(): number { @@ -110,14 +120,16 @@ export class ReindexService

{ + async reindex(targetBlockHeader: Header): Promise { const startHeight = this.getStartBlockFromDataSources(); - monitorWrite(`- Reindex when last processed is ${this.lastProcessedHeight}, to block ${targetBlockHeight}`); + monitorWrite( + `- Reindex when last processed is ${this.lastProcessedHeader.height}, to block ${targetBlockHeader.blockHeight}` + ); await reindex( startHeight, - targetBlockHeight, - this.lastProcessedHeight, + targetBlockHeader, + this.lastProcessedHeader, this.storeService, this.unfinalizedBlocksService, this.dynamicDsService, diff --git a/packages/node-core/src/utils/autoQueue.ts b/packages/node-core/src/utils/autoQueue.ts index bf826fdca3..10a8846fdd 100644 --- a/packages/node-core/src/utils/autoQueue.ts +++ b/packages/node-core/src/utils/autoQueue.ts @@ -13,13 +13,15 @@ export interface IQueue { } export class TaskFlushedError extends Error { + readonly name = 'TaskFlushedError'; + constructor(queueName = 'Auto') { super(`This task was flushed from the ${queueName} queue before completing`); } } export function isTaskFlushedError(e: any): e is TaskFlushedError { - return e instanceof TaskFlushedError; + return (e as TaskFlushedError)?.name === 'TaskFlushedError'; } export class Queue implements IQueue { @@ -121,7 +123,12 @@ export class AutoQueue implements IQueue { * @param {number} [taskTimeoutSec=900] - A timeout for tasks to complete in. Units are seconds. Align with nodeConfig process timeout. * @param {string} [name] - A name for the queue to help with debugging * */ - constructor(capacity?: number, public concurrency = 1, private taskTimeoutSec = 900, private name = 'Auto') { + constructor( + capacity?: number, + public concurrency = 1, + private taskTimeoutSec = 900, + private name = 'Auto' + ) { this.queue = new Queue>(capacity); } diff --git a/packages/node-core/src/utils/blocks.ts b/packages/node-core/src/utils/blocks.ts index 46e783e8fc..7bec3c566d 100644 --- a/packages/node-core/src/utils/blocks.ts +++ b/packages/node-core/src/utils/blocks.ts @@ -3,7 +3,7 @@ import {Schedule} from 'cron-converter'; import {getBlockHeight} from '../indexer/dictionary'; -import {BypassBlocks, IBlock} from '../indexer/types'; +import {BypassBlocks, Header, HistoricalMode, IBlock} from '../indexer/types'; import {getLogger} from '../logger'; const logger = getLogger('timestamp-filter'); @@ -53,3 +53,14 @@ function inBypassBlocks(bypassBlocks: BypassBlocks, blockNum: number): boolean { return blockNum >= start && blockNum <= end; }); } + +export function getHistoricalUnit(mode: HistoricalMode, header: Header): number { + if (mode === 'timestamp') { + if (!header.timestamp) { + throw new Error('Timestamp missing on current block header'); + } + return header.timestamp.getTime(); + } + + return header.blockHeight; +} diff --git a/packages/node-core/src/utils/configure.spec.ts b/packages/node-core/src/utils/configure.spec.ts index 795f13ccae..212aa4238f 100644 --- a/packages/node-core/src/utils/configure.spec.ts +++ b/packages/node-core/src/utils/configure.spec.ts @@ -22,10 +22,31 @@ describe('configure utils', () => { rebaseArgsWithManifest(mockArgv, rawManifest); // Fill the missing args, in manifest runner historical is enabled - expect(mockArgv.disableHistorical).toBeFalsy(); + expect(mockArgv.disableHistorical).toBeUndefined(); + expect(mockArgv.historical).toBeTruthy(); // Keep the original value from argv expect(mockArgv.batchSize).toBe(500); // Args could override manifest options expect(mockArgv.unsafe).toBeFalsy(); }); + + it('should rebase historical manfiest options', () => { + const mock = { + historical: 'height', + } as any; + + rebaseArgsWithManifest(mock, {runner: {node: {options: {historical: true}}}}); + expect(mock.historical).toBe('height'); + + rebaseArgsWithManifest(mock, {runner: {node: {options: {historical: false}}}}); + expect(mock.historical).toBe(false); + + rebaseArgsWithManifest(mock, {runner: {node: {options: {historical: 'height'}}}}); + expect(mock.historical).toBe('height'); + + rebaseArgsWithManifest(mock, {runner: {node: {options: {historical: 'timestamp'}}}}); + expect(mock.historical).toBe('timestamp'); + + expect(mock.disableHistorical).toBeUndefined(); + }); }); diff --git a/packages/node-core/src/utils/configure.ts b/packages/node-core/src/utils/configure.ts index 45f019b526..fea6a0e10d 100644 --- a/packages/node-core/src/utils/configure.ts +++ b/packages/node-core/src/utils/configure.ts @@ -6,10 +6,12 @@ import {getProjectRootAndManifest, IPFS_REGEX, RunnerNodeOptionsModel} from '@su import {plainToClass} from 'class-transformer'; import {last} from 'lodash'; import {IConfig, MinConfig} from '../configure/NodeConfig'; +import {HistoricalMode} from '../indexer'; // These are overridable types from node argv export interface ArgvOverrideOptions { unsafe?: boolean; + historical?: HistoricalMode; disableHistorical?: boolean; unfinalizedBlocks?: boolean; skipTransactions?: boolean; @@ -43,7 +45,7 @@ export function rebaseArgsWithManifest(argvs: ArgvOverrideOptions, rawManifest: if (key === 'historical') { if (value !== undefined) { // THIS IS OPPOSITE - argvs.disableHistorical = !value; + argvs.historical = value === true ? 'height' : value; } return; } diff --git a/packages/node-core/src/utils/reindex.ts b/packages/node-core/src/utils/reindex.ts index 88d31ceaf3..9e03844b92 100644 --- a/packages/node-core/src/utils/reindex.ts +++ b/packages/node-core/src/utils/reindex.ts @@ -1,8 +1,8 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Sequelize} from '@subql/x-sequelize'; -import {IProjectUpgradeService} from '../configure'; +import { Sequelize } from '@subql/x-sequelize'; +import { IProjectUpgradeService } from '../configure'; import { DynamicDsService, IUnfinalizedBlocksService, @@ -12,10 +12,12 @@ import { StoreCacheService, cacheProviderFlushData, cacheProviderResetData, + Header, } from '../indexer'; -import {getLogger} from '../logger'; -import {exitWithError} from '../process'; -import {ForceCleanService} from '../subcommands/forceClean.service'; +import { getLogger } from '../logger'; +import { exitWithError } from '../process'; +import { ForceCleanService } from '../subcommands/forceClean.service'; +import { getHistoricalUnit } from './blocks'; const logger = getLogger('Reindex'); @@ -40,8 +42,8 @@ const logger = getLogger('Reindex'); */ export async function reindex( startHeight: number, - targetBlockHeight: number, - lastProcessedHeight: number, + targetBlockHeader: Header, + lastProcessed: { height: number; timestamp?: number }, storeService: StoreService, unfinalizedBlockService: IUnfinalizedBlocksService, dynamicDsService: DynamicDsService, @@ -50,17 +52,20 @@ export async function reindex( poiService?: PoiService, forceCleanService?: ForceCleanService ): Promise { - if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) { + const lastUnit = storeService.historical === 'timestamp' ? lastProcessed.timestamp : lastProcessed.height; + const targetUnit = getHistoricalUnit(storeService.historical, targetBlockHeader); + + if (!lastUnit || lastUnit < targetUnit) { logger.warn( - `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block` + `Skipping reindexing to ${storeService.historical} ${targetUnit}: current indexing height ${lastUnit} is behind requested ${storeService.historical}` ); return; } // if startHeight is greater than the targetHeight, just force clean - if (targetBlockHeight < startHeight) { + if (targetBlockHeader.blockHeight < startHeight) { logger.info( - `targetHeight: ${targetBlockHeight} is less than startHeight: ${startHeight}. Hence executing force-clean` + `targetHeight: ${targetBlockHeader.blockHeight} is less than startHeight: ${startHeight}. Hence executing force-clean` ); if (!forceCleanService) { exitWithError(`ForceCleanService not provided, cannot force clean`, logger); @@ -69,7 +74,7 @@ export async function reindex( await cacheProviderResetData(storeService.modelProvider); await forceCleanService?.forceClean(); } else { - logger.info(`Reindexing to block: ${targetBlockHeight}`); + logger.info(`Reindexing to ${storeService.historical}: ${targetUnit}`); await cacheProviderFlushData(storeService.modelProvider, true); await cacheProviderResetData(storeService.modelProvider); if (storeService.modelProvider instanceof StoreCacheService) { @@ -84,17 +89,22 @@ export async function reindex( 2.1 On start, projectUpgrade rewind will sync the sequelize models 2.2 On start, without projectUpgrade or upgradablePoint, sequelize will sync models through project.service */ - await projectUpgradeService.rewind(targetBlockHeight, lastProcessedHeight, transaction, storeService); + await projectUpgradeService.rewind( + targetBlockHeader.blockHeight, + lastProcessed.height, + transaction, + storeService + ); await Promise.all([ - storeService.rewind(targetBlockHeight, transaction), - unfinalizedBlockService.resetUnfinalizedBlocks(transaction), // TODO: may not needed for nonfinalized chains - unfinalizedBlockService.resetLastFinalizedVerifiedHeight(transaction), // TODO: may not needed for nonfinalized chains - dynamicDsService.resetDynamicDatasource(targetBlockHeight, transaction), - poiService?.rewind(targetBlockHeight, transaction), + storeService.rewind(targetBlockHeader, transaction), + unfinalizedBlockService.resetUnfinalizedBlocks(), // TODO: may not needed for nonfinalized chains + unfinalizedBlockService.resetLastFinalizedVerifiedHeight(), // TODO: may not needed for nonfinalized chains + dynamicDsService.resetDynamicDatasource(targetBlockHeader.blockHeight, transaction), + poiService?.rewind(targetBlockHeader.blockHeight, transaction), ]); // Flush metadata changes from above Promise.all - await storeService.modelProvider.metadata.flush?.(transaction, targetBlockHeight); + await storeService.modelProvider.metadata.flush?.(transaction, targetUnit); await transaction.commit(); logger.info('Reindex Success'); diff --git a/packages/node-core/src/yargs.ts b/packages/node-core/src/yargs.ts index 404995c048..79eb12718a 100644 --- a/packages/node-core/src/yargs.ts +++ b/packages/node-core/src/yargs.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import yargs, {example} from 'yargs'; +import yargs from 'yargs'; import {hideBin} from 'yargs/helpers'; import {initLogger} from './logger'; @@ -60,7 +60,7 @@ export function yargsBuilder< .command({ command: 'reindex', describe: - 'Reindex to specified block height. Historical must be enabled for the targeted project (--disable-historical=false). Once the command is executed, the application would exit upon completion.', + 'Reindex to specified block height. Historical must be enabled for the targeted project (--historical). The application will exit upon completion.', builder: (yargs) => yargs.options('targetHeight', { type: 'number', @@ -103,9 +103,18 @@ export function yargsBuilder< }, 'disable-historical': { demandOption: false, - describe: 'Disable storing historical state entities', + describe: 'Disable storing historical state entities, please use `historical` flag instead', type: 'boolean', + deprecated: true, + conflicts: 'historical', + // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig + }, + historical: { + describe: 'Enable historical state entities, ', + type: 'string', + choices: ['false', 'height', 'timestamp'], // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig + conflicts: 'disable-historical', }, 'log-level': { demandOption: false, @@ -114,6 +123,7 @@ export function yargsBuilder< choices: ['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent'], }, 'multi-chain': { + alias: 'multichain', demandOption: false, default: false, describe: 'Enables indexing multiple subquery projects into the same database schema', diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index a29a3da388..0967e21621 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Support for historical indexing by timestamp as well as block height (#2584) ### Added - Add an `--enable-cache` flag, allowing you to choose between DB or cache for IO operations. diff --git a/packages/node/src/configure/SchemaMigration.service.test.ts b/packages/node/src/configure/SchemaMigration.service.test.ts index 1a21b858d2..4ef4bb3c89 100644 --- a/packages/node/src/configure/SchemaMigration.service.test.ts +++ b/packages/node/src/configure/SchemaMigration.service.test.ts @@ -5,7 +5,6 @@ import { INestApplication } from '@nestjs/common'; import { DbOption, StoreCacheService } from '@subql/node-core'; import { QueryTypes, Sequelize } from '@subql/x-sequelize'; import { rimraf } from 'rimraf'; -import { ApiService } from '../indexer/api.service'; import { ProjectService } from '../indexer/project.service'; import { prepareApp } from '../utils/test.utils'; @@ -55,9 +54,6 @@ describe('SchemaMigration integration tests', () => { app = await prepareApp(schemaName, cid); projectService = app.get('IProjectService'); - const apiService = app.get(ApiService); - - await apiService.init(); await projectService.init(1); const dbResults = await sequelize.query( @@ -80,9 +76,6 @@ describe('SchemaMigration integration tests', () => { app = await prepareApp(schemaName, cid); projectService = app.get('IProjectService'); - - const apiService = app.get(ApiService); - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; @@ -101,9 +94,6 @@ describe('SchemaMigration integration tests', () => { const projectUpgradeService = app.get('IProjectUpgradeService'); const storeCache = app.get('IStoreModelProvider'); const cacheSpy = jest.spyOn(storeCache, 'updateModels'); - const apiService = app.get(ApiService); - - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; @@ -138,10 +128,8 @@ describe('SchemaMigration integration tests', () => { projectService = app.get('IProjectService'); const projectUpgradeService = app.get('IProjectUpgradeService'); - const storeCache = app.get('IStoreModelProvider'); - const apiService = app.get(ApiService); + const storeCache = app.get('IStoreModelProvider'); - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; diff --git a/packages/node/src/indexer/api.service.spec.ts b/packages/node/src/indexer/api.service.spec.ts index 2f80b7c6c7..564c36f1ee 100644 --- a/packages/node/src/indexer/api.service.spec.ts +++ b/packages/node/src/indexer/api.service.spec.ts @@ -98,7 +98,7 @@ describe('ApiService', () => { it('read custom types from project manifest', async () => { const createSpy = jest.spyOn(ApiPromise, 'create'); - apiService = new ApiService( + apiService = await ApiService.init( project, new ConnectionPoolService( nodeConfig, @@ -107,7 +107,6 @@ describe('ApiService', () => { new EventEmitter2(), nodeConfig, ); - await apiService.init(); const { version } = require('../../package.json'); expect(WsProvider).toHaveBeenCalledWith( Object.keys(testNetwork.endpoint)[0], @@ -133,16 +132,16 @@ describe('ApiService', () => { subquery: 'example', }); - apiService = new ApiService( - project, - new ConnectionPoolService( + await expect( + ApiService.init( + project, + new ConnectionPoolService( + nodeConfig, + new ConnectionPoolStateManager(), + ), + new EventEmitter2(), nodeConfig, - new ConnectionPoolStateManager(), ), - new EventEmitter2(), - nodeConfig, - ); - - await expect(apiService.init()).rejects.toThrow(); + ).rejects.toThrow(); }); }); diff --git a/packages/node/src/indexer/api.service.test.ts b/packages/node/src/indexer/api.service.test.ts index 405756958c..68754936b7 100644 --- a/packages/node/src/indexer/api.service.test.ts +++ b/packages/node/src/indexer/api.service.test.ts @@ -72,7 +72,16 @@ describe('ApiService', () => { useFactory: () => ({}), }, EventEmitter2, - ApiService, + { + provide: ApiService, + useFactory: ApiService.init, + inject: [ + 'ISubqueryProject', + ConnectionPoolService, + EventEmitter2, + NodeConfig, + ], + }, ], imports: [EventEmitterModule.forRoot()], }).compile(); @@ -80,7 +89,6 @@ describe('ApiService', () => { app = module.createNestApplication(); await app.init(); const apiService = app.get(ApiService); - await apiService.init(); return apiService; }; @@ -481,7 +489,16 @@ describe('Load chain type hasher', () => { useFactory: () => ({}), }, EventEmitter2, - ApiService, + { + provide: ApiService, + useFactory: ApiService.init, + inject: [ + 'ISubqueryProject', + ConnectionPoolService, + EventEmitter2, + NodeConfig, + ], + }, ], imports: [EventEmitterModule.forRoot()], }).compile(); @@ -489,7 +506,6 @@ describe('Load chain type hasher', () => { app = module.createNestApplication(); await app.init(); const apiService = app.get(ApiService); - await apiService.init(); return apiService; }; diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index a0d29ba653..0df7ccc894 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -122,7 +122,7 @@ export class ApiService private nodeConfig: SubstrateNodeConfig; - constructor( + private constructor( @Inject('ISubqueryProject') private project: SubqueryProject, connectionPoolService: ConnectionPoolService, eventEmitter: EventEmitter2, @@ -161,16 +161,28 @@ export class ApiService await this.connectionPoolService.onApplicationShutdown(); } - async init(): Promise { + static async init( + project: SubqueryProject, + connectionPoolService: ConnectionPoolService, + eventEmitter: EventEmitter2, + nodeConfig: NodeConfig, + ): Promise { + const apiService = new ApiService( + project, + connectionPoolService, + eventEmitter, + nodeConfig, + ); + overrideConsoleWarn(); let chainTypes: RegisteredTypes | undefined; let network: SubstrateNetworkConfig; try { - chainTypes = await updateChainTypesHasher(this.project.chainTypes); - network = this.project.network; + chainTypes = await updateChainTypesHasher(project.chainTypes); + network = project.network; - if (this.nodeConfig.primaryNetworkEndpoint) { - const [endpoint, config] = this.nodeConfig.primaryNetworkEndpoint; + if (apiService.nodeConfig.primaryNetworkEndpoint) { + const [endpoint, config] = apiService.nodeConfig.primaryNetworkEndpoint; (network.endpoint as Record)[endpoint] = config; } @@ -182,13 +194,13 @@ export class ApiService logger.info('Using provided chain types'); } - await this.createConnections( + await apiService.createConnections( network, //createConnection (endpoint, config) => ApiPromiseConnection.create( endpoint, - this.fetchBlocksBatches, + apiService.fetchBlocksBatches, { chainTypes, }, @@ -198,14 +210,14 @@ export class ApiService (connection: ApiPromiseConnection, endpoint: string, index: number) => { const api = connection.unsafeApi; api.on('connected', () => { - this.eventEmitter.emit(IndexerEvent.ApiConnected, { + eventEmitter.emit(IndexerEvent.ApiConnected, { value: 1, apiIndex: index, endpoint: endpoint, }); }); api.on('disconnected', () => { - this.eventEmitter.emit(IndexerEvent.ApiConnected, { + eventEmitter.emit(IndexerEvent.ApiConnected, { value: 0, apiIndex: index, endpoint: endpoint, @@ -213,7 +225,7 @@ export class ApiService }); }, ); - return this; + return apiService; } async updateChainTypes(): Promise { diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index bf9be4346f..fc92ebca9e 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -26,6 +26,7 @@ import { RuntimeService } from '../runtime/runtimeService'; import { BlockContent } from '../types'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; import { IIndexerWorker } from '../worker/worker'; +import { FetchBlockResponse } from '../worker/worker.service'; type IndexerWorker = IIndexerWorker & { terminate: () => Promise; @@ -121,7 +122,7 @@ export class WorkerBlockDispatcherService protected async fetchBlock( worker: IndexerWorker, height: number, - ): Promise { + ): Promise { // get SpecVersion from main runtime service const { blockSpecVersion, syncedDictionary } = await this.runtimeService.getSpecVersion(height); @@ -131,7 +132,7 @@ export class WorkerBlockDispatcherService } // const start = new Date(); - await worker.fetchBlock(height, blockSpecVersion); + return worker.fetchBlock(height, blockSpecVersion); // const end = new Date(); // const waitTime = end.getTime() - start.getTime(); diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index acf65ad23b..3d4e411961 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -6,7 +6,6 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { StoreService, NodeConfig, - StoreCacheService, ConnectionPoolStateManager, IProjectUpgradeService, PoiSyncService, @@ -14,6 +13,7 @@ import { MonitorService, CoreModule, IStoreModelProvider, + ConnectionPoolService, } from '@subql/node-core'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; @@ -34,7 +34,16 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @Module({ imports: [CoreModule], providers: [ - ApiService, + { + provide: ApiService, + useFactory: ApiService.init, + inject: [ + 'ISubqueryProject', + ConnectionPoolService, + EventEmitter2, + NodeConfig, + ], + }, IndexerManager, { provide: 'IBlockDispatcher', @@ -57,32 +66,32 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; ) => nodeConfig.workers ? new WorkerBlockDispatcherService( - nodeConfig, - eventEmitter, - projectService, - projectUpgradeService, - cacheService, - storeService, - storeModelProvider, - poiSyncService, - project, - dynamicDsService, - unfinalizedBlocks, - connectionPoolState, - monitorService, - ) + nodeConfig, + eventEmitter, + projectService, + projectUpgradeService, + cacheService, + storeService, + storeModelProvider, + poiSyncService, + project, + dynamicDsService, + unfinalizedBlocks, + connectionPoolState, + monitorService, + ) : new BlockDispatcherService( - apiService, - nodeConfig, - indexerManager, - eventEmitter, - projectService, - projectUpgradeService, - storeService, - storeModelProvider, - poiSyncService, - project, - ), + apiService, + nodeConfig, + indexerManager, + eventEmitter, + projectService, + projectUpgradeService, + storeService, + storeModelProvider, + poiSyncService, + project, + ), inject: [ NodeConfig, EventEmitter2, diff --git a/packages/node/src/indexer/indexer.manager.spec.ts b/packages/node/src/indexer/indexer.manager.spec.ts index 5c02cf7b0a..2662b939ab 100644 --- a/packages/node/src/indexer/indexer.manager.spec.ts +++ b/packages/node/src/indexer/indexer.manager.spec.ts @@ -150,14 +150,14 @@ export function mockProjectUpgradeService( }; } -function createIndexerManager( +async function createIndexerManager( project: SubqueryProject, connectionPoolService: ConnectionPoolService, nodeConfig: NodeConfig, -): IndexerManager { +): Promise { const sequelize = new Sequelize(); const eventEmitter = new EventEmitter2(); - const apiService = new ApiService( + const apiService = await ApiService.init( project, connectionPoolService, eventEmitter, @@ -231,7 +231,7 @@ describe('IndexerManager', () => { }); it.skip('should be able to start the manager (v0.0.1)', async () => { - // indexerManager = createIndexerManager( + // indexerManager = await createIndexerManager( // testSubqueryProject_1(), // new ConnectionPoolService( // nodeConfig, @@ -244,7 +244,7 @@ describe('IndexerManager', () => { }); it.skip('should be able to start the manager (v0.2.0)', async () => { - // indexerManager = createIndexerManager( + // indexerManager = await createIndexerManager( // testSubqueryProject_2(), // new ConnectionPoolService( // nodeConfig, diff --git a/packages/node/src/indexer/project.service.spec.ts b/packages/node/src/indexer/project.service.spec.ts index 16960e499c..d148a804f1 100644 --- a/packages/node/src/indexer/project.service.spec.ts +++ b/packages/node/src/indexer/project.service.spec.ts @@ -170,7 +170,16 @@ describe('ProjectService', () => { inject: [ApiService, 'ISubqueryProject'], }, EventEmitter2, - ApiService, + { + provide: ApiService, + useFactory: ApiService.init, + inject: [ + 'ISubqueryProject', + ConnectionPoolService, + EventEmitter2, + NodeConfig, + ], + }, { provide: ProjectUpgradeService, useValue: projectUpgrade, @@ -182,7 +191,6 @@ describe('ProjectService', () => { const app = module.createNestApplication(); await app.init(); apiService = app.get(ApiService); - await apiService.init(); projectUpgradeService = app.get( ProjectUpgradeService, ) as ProjectUpgradeService; diff --git a/packages/node/src/indexer/store.service.test.ts b/packages/node/src/indexer/store.service.test.ts index 46c8033511..3278789dda 100644 --- a/packages/node/src/indexer/store.service.test.ts +++ b/packages/node/src/indexer/store.service.test.ts @@ -12,7 +12,6 @@ import { import { QueryTypes, Sequelize } from '@subql/x-sequelize'; import { rimraf } from 'rimraf'; import { prepareApp } from '../utils/test.utils'; -import { ApiService } from './api.service'; import { ProjectService } from './project.service'; const option: DbOption = { @@ -56,9 +55,6 @@ describe('Store service integration test', () => { app = await prepareApp(schemaName, cid, false); projectService = app.get('IProjectService'); - const apiService = app.get(ApiService); - - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; @@ -148,6 +144,7 @@ AND table_name = 'positions'; is_nullable: 'NO', }); }); + it('Correct db sync on non-historical', async () => { const cid = 'QmNUNBiVC1BDDNbXCbTxzPexodbSmTqZUaohbeBae6b6r8'; schemaName = 'sync-schema-2'; @@ -155,9 +152,6 @@ AND table_name = 'positions'; app = await prepareApp(schemaName, cid, true); projectService = app.get('IProjectService'); - const apiService = app.get(ApiService); - - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; @@ -216,7 +210,6 @@ ORDER BY foreign_column: 'id', }); }); - // it('Cyclic relations on non-historical', async () => { const cid = 'QmTLwdpfE7xsmAtPj3Bep9KKgAPbt2tvXisUHcHys6anSG'; @@ -225,9 +218,6 @@ ORDER BY app = await prepareApp(schemaName, cid, true); projectService = app.get('IProjectService'); - const apiService = app.get(ApiService); - - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; @@ -262,6 +252,7 @@ WHERE foreign_column_name: 'id', }); }); + it('Init with enums', async () => { const cid = 'QmVDDxVgmkKzXKcK5YBkEu3Wvzao7uQxear2SVLTUg2bQ1'; schemaName = 'sync-schema-4'; @@ -269,9 +260,6 @@ WHERE app = await prepareApp(schemaName, cid, true); projectService = app.get('IProjectService'); - const apiService = app.get(ApiService); - - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; @@ -295,6 +283,7 @@ ORDER BY t.typname, e.enumsortorder;`, '65c7fd4e5d', ]); }); + it('Able to drop notification triggers and functions', async () => { // if subscription is no longer enabled should be able to drop all prior triggers and functions related to subscription const cid = 'Qma3HraGKnH5Gte2WVs4sAAY6z5nBSqVuVq7Ef3eVQQPvz'; @@ -328,9 +317,6 @@ ORDER BY t.typname, e.enumsortorder;`, app = await prepareApp(schemaName, cid, false, false); projectService = app.get('IProjectService'); - const apiService = app.get(ApiService); - - await apiService.init(); await projectService.init(1); tempDir = (projectService as any).project.root; diff --git a/packages/node/src/indexer/unfinalizedBlocks.service.ts b/packages/node/src/indexer/unfinalizedBlocks.service.ts index 6aff82c68a..67c7455058 100644 --- a/packages/node/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node/src/indexer/unfinalizedBlocks.service.ts @@ -9,7 +9,11 @@ import { mainThreadOnly, NodeConfig, } from '@subql/node-core'; -import { substrateHeaderToHeader } from '../utils/substrate'; +import { + getBlockByHeight, + substrateBlockToHeader, + substrateHeaderToHeader, +} from '../utils/substrate'; import { ApiService } from './api.service'; import { BlockContent, LightBlockContent } from './types'; @@ -37,14 +41,15 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService< // TODO: add cache here @mainThreadOnly() protected async getHeaderForHash(hash: string): Promise

{ - return substrateHeaderToHeader( - await this.apiService.api.rpc.chain.getHeader(hash), + return substrateBlockToHeader( + await this.apiService.api.rpc.chain.getBlock(hash), ); } @mainThreadOnly() - protected async getHeaderForHeight(height: number): Promise
{ - const hash = await this.apiService.api.rpc.chain.getBlockHash(height); - return this.getHeaderForHash(hash.toHex()); + async getHeaderForHeight(height: number): Promise
{ + return substrateBlockToHeader( + await getBlockByHeight(this.apiService.api, height), + ); } } diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index eb150e16f2..8455afae1f 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -10,9 +10,7 @@ import { WorkerUnfinalizedBlocksService, WorkerCoreModule, } from '@subql/node-core'; -import { SubqueryProject } from '../../configure/SubqueryProject'; import { ApiService } from '../api.service'; -import { ApiPromiseConnection } from '../apiPromise.connection'; import { DsProcessorService } from '../ds-processor.service'; import { DynamicDsService } from '../dynamic-ds.service'; import { IndexerManager } from '../indexer.manager'; @@ -31,21 +29,7 @@ import { WorkerService } from './worker.service'; IndexerManager, { provide: ApiService, - useFactory: async ( - project: SubqueryProject, - connectionPoolService: ConnectionPoolService, - eventEmitter: EventEmitter2, - nodeConfig: NodeConfig, - ) => { - const apiService = new ApiService( - project, - connectionPoolService, - eventEmitter, - nodeConfig, - ); - await apiService.init(); - return apiService; - }, + useFactory: ApiService.init, inject: [ 'ISubqueryProject', ConnectionPoolService, diff --git a/packages/node/src/indexer/worker/worker.service.ts b/packages/node/src/indexer/worker/worker.service.ts index 8ecb6828bb..5ed1d0e69d 100644 --- a/packages/node/src/indexer/worker/worker.service.ts +++ b/packages/node/src/indexer/worker/worker.service.ts @@ -9,15 +9,17 @@ import { BaseWorkerService, IProjectUpgradeService, IBlock, + Header, } from '@subql/node-core'; import { SubstrateDatasource } from '@subql/types'; +import { substrateBlockToHeader } from '../../utils/substrate'; import { ApiService } from '../api.service'; import { SpecVersion } from '../dictionary'; import { IndexerManager } from '../indexer.manager'; import { WorkerRuntimeService } from '../runtime/workerRuntimeService'; import { BlockContent, isFullBlock, LightBlockContent } from '../types'; -export type FetchBlockResponse = { specVersion?: number; parentHash: string }; +export type FetchBlockResponse = Header & { specVersion?: number }; @Injectable() export class WorkerService extends BaseWorkerService< @@ -56,10 +58,13 @@ export class WorkerService extends BaseWorkerService< return block; } - protected toBlockResponse(block: BlockContent): FetchBlockResponse { + // TODO test this with LightBlockContent + protected toBlockResponse( + block: BlockContent /* | LightBlockContent*/, + ): FetchBlockResponse { return { + ...substrateBlockToHeader(block.block), specVersion: block.block.specVersion, - parentHash: block.block.block.header.parentHash.toHex(), }; } diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts index 83aa1f71cf..469f3056a0 100644 --- a/packages/node/src/init.ts +++ b/packages/node/src/init.ts @@ -10,7 +10,6 @@ import { NestLogger, } from '@subql/node-core'; import { AppModule } from './app.module'; -import { ApiService } from './indexer/api.service'; import { FetchService } from './indexer/fetch.service'; import { ProjectService } from './indexer/project.service'; import { yargsOptions } from './yargs'; @@ -34,10 +33,8 @@ export async function bootstrap(): Promise { const projectService: ProjectService = app.get('IProjectService'); const fetchService = app.get(FetchService); - const apiService = app.get(ApiService); // Initialise async services, we do this here rather than in factories, so we can capture one off events - await apiService.init(); await projectService.init(); await fetchService.init(projectService.startHeight); diff --git a/packages/node/src/subcommands/reindex.module.ts b/packages/node/src/subcommands/reindex.module.ts index 7d69d44eb7..965bfef559 100644 --- a/packages/node/src/subcommands/reindex.module.ts +++ b/packages/node/src/subcommands/reindex.module.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import { Module } from '@nestjs/common'; -import { EventEmitterModule } from '@nestjs/event-emitter'; +import { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter'; import { SchedulerRegistry } from '@nestjs/schedule'; import { DbModule, @@ -11,6 +11,9 @@ import { ReindexService, StoreService, PoiService, + ConnectionPoolService, + NodeConfig, + ConnectionPoolStateManager, } from '@subql/node-core'; import { ConfigureModule } from '../configure/configure.module'; import { ApiService } from '../indexer/api.service'; @@ -34,10 +37,18 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; useClass: DynamicDsService, }, DsProcessorService, + ConnectionPoolStateManager, + ConnectionPoolService, { // Used to work with DI for unfinalizedBlocksService but not used with reindex provide: ApiService, - useFactory: () => undefined, + useFactory: ApiService.init, + inject: [ + 'ISubqueryProject', + ConnectionPoolService, + EventEmitter2, + NodeConfig, + ], }, SchedulerRegistry, ], diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index 5e7eab2045..0f6d806e35 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -15,6 +15,7 @@ import { StoreService, TestRunner, SandboxService, + NodeConfig, } from '@subql/node-core'; import { ConfigureModule } from '../configure/configure.module'; import { ApiService } from '../indexer/api.service'; @@ -42,7 +43,16 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; provide: 'IProjectService', useClass: ProjectService, }, - ApiService, + { + provide: ApiService, + useFactory: ApiService.init, + inject: [ + 'ISubqueryProject', + ConnectionPoolService, + EventEmitter2, + NodeConfig, + ], + }, SchedulerRegistry, TestRunner, { diff --git a/packages/node/src/subcommands/testing.service.ts b/packages/node/src/subcommands/testing.service.ts index 1f9677eab4..1ae9933a48 100644 --- a/packages/node/src/subcommands/testing.service.ts +++ b/packages/node/src/subcommands/testing.service.ts @@ -49,10 +49,8 @@ export class TestingService extends BaseTestingService< await testContext.init(); const projectService: ProjectService = testContext.get('IProjectService'); - const apiService = testContext.get(ApiService); // Initialise async services, we do this here rather than in factories, so we can capture one off events - await apiService.init(); await projectService.init(); return [testContext.close.bind(testContext), testContext.get(TestRunner)]; diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts index 1b4dd9c9c6..86c4c15152 100644 --- a/packages/node/src/utils/substrate.ts +++ b/packages/node/src/utils/substrate.ts @@ -47,6 +47,13 @@ export function substrateHeaderToHeader(header: SubstrateHeader): Header { }; } +export function substrateBlockToHeader(block: SignedBlock): Header { + return { + ...substrateHeaderToHeader(block.block.header), + timestamp: getTimestamp(block), + }; +} + export function wrapBlock( signedBlock: SignedBlock, events: EventRecord[], @@ -76,6 +83,7 @@ export function getTimestamp({ } // For network that doesn't use timestamp-set, return undefined // See test `return undefined if no timestamp set extrinsic` + // E.g Shiden return undefined; } @@ -395,7 +403,7 @@ export async function fetchBlocksBatches( const wrappedEvents = wrapEvents(wrappedExtrinsics, events, wrappedBlock); return { - getHeader: () => substrateHeaderToHeader(wrappedBlock.block.header), + getHeader: () => substrateBlockToHeader(wrappedBlock), block: { block: wrappedBlock, extrinsics: wrappedExtrinsics, diff --git a/packages/node/src/utils/test.utils.ts b/packages/node/src/utils/test.utils.ts index 84867b9ca9..9916816590 100644 --- a/packages/node/src/utils/test.utils.ts +++ b/packages/node/src/utils/test.utils.ts @@ -10,6 +10,7 @@ import { Test } from '@nestjs/testing'; import { CoreModule, DbModule, + HistoricalMode, NodeConfig, registerApp, } from '@subql/node-core'; @@ -23,13 +24,13 @@ import { FetchModule } from '../indexer/fetch.module'; const mockInstance = async ( cid: string, schemaName: string, - disableHistorical: boolean, + historical: HistoricalMode, useSubscription: boolean, timestampField: boolean, ) => { const argv: Record = { _: [], - disableHistorical, + historical, subquery: `ipfs://${cid}`, dbSchema: schemaName, allowSchemaMigration: true, @@ -49,14 +50,14 @@ const mockInstance = async ( async function mockRegister( cid: string, schemaName: string, - disableHistorical: boolean, + historical: HistoricalMode, useSubscription: boolean, timestampField: boolean, ): Promise { const { nodeConfig, project } = await mockInstance( cid, schemaName, - disableHistorical, + historical, useSubscription, timestampField, ); @@ -99,7 +100,7 @@ export async function prepareApp( mockRegister( cid, schemaName, - disableHistorical, + disableHistorical ? false : 'height', useSubscription, timestampField, ), diff --git a/packages/query/CHANGELOG.md b/packages/query/CHANGELOG.md index 482a815c26..b17ce35ba0 100644 --- a/packages/query/CHANGELOG.md +++ b/packages/query/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed - Update the playground to the latest GraphiQL (#2588) +### Added +- Support for historical indexing by timestamp as well as block height (#2584) ## [2.15.2] - 2024-09-25 ### Changed diff --git a/packages/query/src/graphql/graphql.module.ts b/packages/query/src/graphql/graphql.module.ts index a50f82bdf0..e6be02e73c 100644 --- a/packages/query/src/graphql/graphql.module.ts +++ b/packages/query/src/graphql/graphql.module.ts @@ -6,7 +6,7 @@ import {setInterval} from 'timers'; import PgPubSub from '@graphile/pg-pubsub'; import {Module, OnModuleDestroy, OnModuleInit} from '@nestjs/common'; import {HttpAdapterHost} from '@nestjs/core'; -import {delay, getDbType, SUPPORT_DB} from '@subql/common'; +import {delay} from '@subql/common'; import {hashName} from '@subql/utils'; import {getPostGraphileBuilder, Plugin, PostGraphileCoreOptions} from '@subql/x-postgraphile-core'; import {ApolloServerPluginCacheControl, ApolloServerPluginLandingPageDisabled} from 'apollo-server-core'; @@ -47,7 +47,6 @@ class NoInitError extends Error { }) export class GraphqlModule implements OnModuleInit, OnModuleDestroy { private _apolloServer?: ApolloServer; - private _dbType?: SUPPORT_DB; constructor( private readonly httpAdapterHost: HttpAdapterHost, private readonly config: Config, @@ -60,26 +59,15 @@ export class GraphqlModule implements OnModuleInit, OnModuleDestroy { return this._apolloServer; } - private get dbType(): SUPPORT_DB { - assert(this._dbType, new NoInitError()); - return this._dbType; - } - async onModuleInit(): Promise { if (!this.httpAdapterHost) { return; } - this._dbType = await getDbType(this.pgPool); try { this._apolloServer = await this.createServer(); } catch (e: any) { throw new Error(`create apollo server failed, ${e.message}`); } - if (this.dbType === SUPPORT_DB.cockRoach) { - logger.info(`Using Cockroach database, subscription and hot-schema functions are not supported`); - argv.subscription = false; - argv['disable-hot-schema'] = true; - } } async schemaListener(dbSchema: string, options: PostGraphileCoreOptions): Promise { @@ -153,7 +141,7 @@ export class GraphqlModule implements OnModuleInit, OnModuleDestroy { connectionFilterRelations: false, // We use our own forked version with historical support // cockroach db does not support pgPartition - pgUsePartitionedParent: this.dbType !== SUPPORT_DB.cockRoach, + pgUsePartitionedParent: true, }, }; diff --git a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts index 707e8520f0..1411a3504c 100644 --- a/packages/query/src/graphql/plugins/GetMetadataPlugin.ts +++ b/packages/query/src/graphql/plugins/GetMetadataPlugin.ts @@ -17,6 +17,7 @@ const {version: packageVersion} = require('../../../package.json'); const META_JSON_FIELDS = ['deployments']; const METADATA_TYPES = { lastProcessedHeight: 'number', + lastProcessedBlockTimestamp: 'number', lastProcessedTimestamp: 'number', targetHeight: 'number', lastFinalizedVerifiedHeight: 'number', @@ -34,6 +35,7 @@ const METADATA_TYPES = { lastCreatedPoiHeight: 'number', latestSyncedPoiHeight: 'number', dbSize: 'string', + historicalStateEnabled: 'string', }; const METADATA_KEYS = Object.keys(METADATA_TYPES); @@ -132,7 +134,7 @@ async function fetchMetadataFromTable( // Store default metadata name in table avoid query system table let defaultMetadataName: string; -async function fetchFromTable( +export async function fetchFromTable( pgClient: Client, schemaName: string, chainId: string | undefined, @@ -239,8 +241,8 @@ export const GetMetadataPlugin = makeExtendSchemaPlugin((build: Build, options) _metadatas( after: Cursor before: Cursor # distinct: [_mmr_distinct_enum] = null # filter: _MetadataFilter # first: Int # offset: Int + # orderBy: [_MetadatasOrderBy!] = [PRIMARY_KEY_ASC] ): # last: Int - # orderBy: [_MetadatasOrderBy!] = [PRIMARY_KEY_ASC] _Metadatas } `, diff --git a/packages/query/src/graphql/plugins/historical/PgBlockHeightPlugin.ts b/packages/query/src/graphql/plugins/historical/PgBlockHeightPlugin.ts index 9724f6126e..7c2661db69 100644 --- a/packages/query/src/graphql/plugins/historical/PgBlockHeightPlugin.ts +++ b/packages/query/src/graphql/plugins/historical/PgBlockHeightPlugin.ts @@ -4,6 +4,7 @@ import {QueryBuilder} from '@subql/x-graphile-build-pg'; import {Plugin, Context} from 'graphile-build'; import {GraphQLString} from 'graphql'; +import {fetchFromTable} from '../GetMetadataPlugin'; import {makeRangeQuery, hasBlockRange} from './utils'; function addRangeQuery(queryBuilder: QueryBuilder, sql: any) { @@ -17,7 +18,18 @@ function addQueryContext(queryBuilder: QueryBuilder, sql: any, blockHeight: any) } } -export const PgBlockHeightPlugin: Plugin = (builder) => { +export const PgBlockHeightPlugin: Plugin = async (builder, options) => { + // Note this varies from node where true is allowed because of legacy support + let historicalMode: boolean | 'height' | 'timestamp' = 'height'; + const [schemaName] = options.pgSchemas; + + try { + const {historicalStateEnabled} = await fetchFromTable(options.pgConfig, schemaName, undefined, false); + historicalMode = historicalStateEnabled; + } catch (e) { + /* Do nothing, default value is already set */ + } + // Adds blockHeight condition to join clause when joining a table that has _block_range column builder.hook( 'GraphQLObjectType:fields:field', @@ -41,9 +53,10 @@ export const PgBlockHeightPlugin: Plugin = (builder) => { return field; } - addArgDataGenerator(({blockHeight}) => ({ + addArgDataGenerator(({blockHeight, timestamp}) => ({ pgQuery: (queryBuilder: QueryBuilder) => { - addQueryContext(queryBuilder, sql, blockHeight); + // If timestamp provided use that as the value + addQueryContext(queryBuilder, sql, blockHeight ?? timestamp); addRangeQuery(queryBuilder, sql); }, })); @@ -56,7 +69,10 @@ export const PgBlockHeightPlugin: Plugin = (builder) => { ( args, {extend, pgSql: sql}, - {addArgDataGenerator, scope: {isPgFieldConnection, isPgRowByUniqueConstraintField, pgFieldIntrospection}} + { + addArgDataGenerator, + scope: {isPgFieldConnection, isPgRowByUniqueConstraintField, pgFieldIntrospection}, + }: Context ) => { if (!isPgRowByUniqueConstraintField && !isPgFieldConnection) { return args; @@ -65,16 +81,27 @@ export const PgBlockHeightPlugin: Plugin = (builder) => { return args; } - addArgDataGenerator(({blockHeight}) => ({ + addArgDataGenerator(({blockHeight, timestamp}) => ({ pgQuery: (queryBuilder: QueryBuilder) => { - addQueryContext(queryBuilder, sql, blockHeight); + // If timestamp provided use that as the value + addQueryContext(queryBuilder, sql, blockHeight ?? timestamp); addRangeQuery(queryBuilder, sql); }, })); + if (historicalMode === 'timestamp') { + return extend(args, { + timestamp: { + description: 'When specified, the query will return results as of this timestamp. Unix timestamp in MS', + defaultValue: '9223372036854775807', + type: GraphQLString, // String because of int overflow + }, + }); + } + return extend(args, { blockHeight: { - description: 'Block height', + description: 'When specified, the query will return results as of this block height', defaultValue: '9223372036854775807', type: GraphQLString, // String because of int overflow }, diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.md index 78b09f444d..1ab4cb130d 100644 --- a/packages/utils/CHANGELOG.md +++ b/packages/utils/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Support for indexes on enums (#2586) +### Changed +- Update metadata types (#2584) ## [2.14.0] - 2024-08-05 ### Added diff --git a/packages/utils/src/query/types.ts b/packages/utils/src/query/types.ts index 36da19ca74..2c81292c76 100644 --- a/packages/utils/src/query/types.ts +++ b/packages/utils/src/query/types.ts @@ -8,6 +8,7 @@ export type TableEstimate = { export type MetaData = { lastProcessedHeight: number; + lastProcessedBlockTimestamp: number; lastProcessedTimestamp: number; targetHeight: number; chain: string; @@ -19,4 +20,5 @@ export type MetaData = { startHeight?: number; rowCountEstimate: TableEstimate[]; deployments: Record; + historicalStateEnabled: boolean | 'height' | 'timestamp'; };