From 69aac414b72faf20f184b177e5327584141780d6 Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 13 Jun 2024 15:37:40 +0800 Subject: [PATCH 01/25] enable strict node --- packages/node/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/node/tsconfig.json b/packages/node/tsconfig.json index 621baaff7a..4a94b628d0 100644 --- a/packages/node/tsconfig.json +++ b/packages/node/tsconfig.json @@ -4,7 +4,8 @@ "sourceMap": true, "tsBuildInfoFile": "dist/.tsbuildinfo", "rootDir": "src", - "outDir": "./dist" + "outDir": "./dist", + "strict": true }, "references": [ { "path": "../common" }, From b3c52dd2f2e296742c6202e7c7abd04e432d9418 Mon Sep 17 00:00:00 2001 From: Tate Date: Fri, 14 Jun 2024 16:16:28 +0800 Subject: [PATCH 02/25] ts strict change in node --- .../blockDispatcher/base-block-dispatcher.ts | 8 ++-- .../worker-block-dispatcher.ts | 2 +- packages/node-core/src/utils/project.ts | 2 +- .../configure/SchemaMigration.service.test.ts | 4 +- .../node/src/configure/SubqueryProject.ts | 7 +-- packages/node/src/indexer/api.service.test.ts | 5 +-- packages/node/src/indexer/api.service.ts | 44 ++++++++++++------- .../node/src/indexer/apiPromise.connection.ts | 2 +- .../block-dispatcher.service.ts | 15 ++++--- .../substrate-block-dispatcher.ts | 2 +- .../worker-block-dispatcher.service.ts | 6 +-- .../dictionary/substrateDictionary.service.ts | 2 + .../v1/substrateDictionaryV1.spec.ts | 14 +++--- .../dictionary/v1/substrateDictionaryV1.ts | 26 ++++++----- .../src/indexer/ds-processor.service.spec.ts | 2 +- .../node/src/indexer/dynamic-ds.service.ts | 4 +- .../node/src/indexer/fetch.service.spec.ts | 16 +++---- .../node/src/indexer/fetch.service.test.ts | 14 +++--- .../node/src/indexer/indexer.manager.spec.ts | 2 +- .../indexer/runtime/base-runtime.service.ts | 13 +++--- .../indexer/runtime/runtime.service.spec.ts | 12 ++--- .../src/indexer/runtime/runtimeService.ts | 6 +-- .../indexer/runtime/workerRuntimeService.ts | 4 +- .../node/src/indexer/store.service.test.ts | 14 +++--- .../node/src/indexer/worker/worker.service.ts | 8 ++-- packages/node/src/indexer/worker/worker.ts | 4 +- packages/node/src/utils/project.ts | 5 ++- packages/node/src/utils/substrate.ts | 20 ++++++--- packages/node/tsconfig.json | 3 +- 29 files changed, 151 insertions(+), 115 deletions(-) diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts index f6c3706227..5ddb106d14 100644 --- a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts @@ -50,7 +50,7 @@ export abstract class BaseBlockDispatcher implements IB protected _processedBlockCount = 0; protected _latestProcessedHeight = 0; protected currentProcessingHeight = 0; - private _onDynamicDsCreated?: (height: number) => Promise; + private _onDynamicDsCreated?: (height: number) => void; private _pendingRewindHeight?: number; protected smartBatchService: SmartBatchService; @@ -72,7 +72,7 @@ export abstract class BaseBlockDispatcher implements IB abstract enqueueBlocks(heights: (IBlock | number)[], latestBufferHeight?: number): void | Promise; - async init(onDynamicDsCreated: (height: number) => Promise): Promise { + async init(onDynamicDsCreated: (height: number) => void): Promise { this._onDynamicDsCreated = onDynamicDsCreated; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion this.setProcessedBlockCount((await this.storeCacheService.metadata.find('processedBlockCount', 0))!); @@ -103,7 +103,7 @@ export abstract class BaseBlockDispatcher implements IB this._latestProcessedHeight = height; } - protected get onDynamicDsCreated(): (height: number) => Promise { + protected get onDynamicDsCreated(): (height: number) => void { if (!this._onDynamicDsCreated) { throw new Error('BaseBlockDispatcher has not been initialized'); } @@ -204,7 +204,7 @@ export abstract class BaseBlockDispatcher implements IB this.createPOI(height, blockHash, operationHash); if (dynamicDsCreated) { - await this.onDynamicDsCreated(height); + this.onDynamicDsCreated(height); } assert( !this.latestProcessedHeight || height > this.latestProcessedHeight, diff --git a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts index 1e66359d69..00b1e86c71 100644 --- a/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts +++ b/packages/node-core/src/indexer/blockDispatcher/worker-block-dispatcher.ts @@ -81,7 +81,7 @@ export abstract class WorkerBlockDispatcher this.numWorkers = nodeConfig.workers!; } - async init(onDynamicDsCreated: (height: number) => Promise): Promise { + async init(onDynamicDsCreated: (height: number) => void): Promise { this.workers = await Promise.all(new Array(this.numWorkers).fill(0).map(() => this.createIndexerWorker())); return super.init(onDynamicDsCreated); } diff --git a/packages/node-core/src/utils/project.ts b/packages/node-core/src/utils/project.ts index fa2645d86f..f9941fbcc2 100644 --- a/packages/node-core/src/utils/project.ts +++ b/packages/node-core/src/utils/project.ts @@ -17,7 +17,7 @@ import {exitWithError} from '../process'; const logger = getLogger('Project-Utils'); -export async function getValidPort(argvPort: number): Promise { +export async function getValidPort(argvPort: number | undefined): Promise { const validate = (x: any) => { const p = parseInt(x); return isNaN(p) ? null : p; diff --git a/packages/node/src/configure/SchemaMigration.service.test.ts b/packages/node/src/configure/SchemaMigration.service.test.ts index 9f35ad7548..a4a428bd14 100644 --- a/packages/node/src/configure/SchemaMigration.service.test.ts +++ b/packages/node/src/configure/SchemaMigration.service.test.ts @@ -58,11 +58,11 @@ describe('SchemaMigration integration tests', () => { await apiService.init(); await projectService.init(1); - const dbResults = await sequelize.query( + const dbResults: string[][] = await sequelize.query( `SELECT table_name FROM information_schema.tables WHERE table_schema= :schema;`, { type: QueryTypes.SELECT, replacements: { schema: schemaName } }, ); - const tableNames: string[] = dbResults.map((row: string[]) => { + const tableNames: string[] = dbResults.map((row) => { return row[0]; }); diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index 701c57a49c..2be5557d59 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -72,7 +72,7 @@ export class SubqueryProject implements ISubqueryProject { this.#dataSources = await insertBlockFiltersCronSchedules( this.dataSources, getTimestamp, - isRuntimeDs, + isRuntimeDs, // Note: ts-strict change --- Not sure it should be modified SubstrateHandlerKind.Block, ); } @@ -146,7 +146,7 @@ async function loadProjectFromManifestBase( ), ); - let schemaString: string; + let schemaString: string | undefined; try { schemaString = await reader.getFile(projectManifest.schema.file); } catch (e) { @@ -154,6 +154,7 @@ async function loadProjectFromManifestBase( `unable to fetch the schema from ${projectManifest.schema.file}`, ); } + assert(schemaString, 'Schema file is empty'); const schema = buildSchemaFromString(schemaString); const chainTypes = projectManifest.network.chaintypes @@ -171,7 +172,7 @@ async function loadProjectFromManifestBase( projectManifest.templates, root, reader, - isCustomDs, + isCustomDs, // Note: ts-strict change --- Not sure it should be modified ); const runner = projectManifest.runner; assert( diff --git a/packages/node/src/indexer/api.service.test.ts b/packages/node/src/indexer/api.service.test.ts index d3f6893c3b..bc7a27d329 100644 --- a/packages/node/src/indexer/api.service.test.ts +++ b/packages/node/src/indexer/api.service.test.ts @@ -157,9 +157,8 @@ describe('ApiService', () => { mockBlock.block.header, runtimeVersion, ); - const patchedResult = await patchedApi.rpc.state.getRuntimeVersion( - earlyBlockhash, - ); + const patchedResult = + await patchedApi.rpc.state.getRuntimeVersion(earlyBlockhash); expect(apiResults).toEqual(patchedResult); // patchedApi without input blockHash, will return runtimeVersion at 6721195 const patchedResult2 = await patchedApi.rpc.state.getRuntimeVersion(); diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 4cc9f4ec63..be12c4c980 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -22,6 +22,7 @@ import { IBlock, MetadataMismatchError, exitWithError, + IApiConnectionSpecific, } from '@subql/node-core'; import { SubstrateNodeConfig } from '../configure/NodeConfig'; import { SubqueryProject } from '../configure/SubqueryProject'; @@ -105,10 +106,10 @@ export class ApiService > implements OnApplicationShutdown { - private fetchBlocksFunction: FetchFunc; + private fetchBlocksFunction!: FetchFunc; private fetchBlocksBatches: GetFetchFunc = () => this.fetchBlocksFunction; - private currentBlockHash: string; - private currentBlockNumber: number; + private currentBlockHash!: string; + private currentBlockNumber!: number; private nodeConfig: SubstrateNodeConfig; @@ -154,7 +155,15 @@ export class ApiService chainTypes, }), //postConnectedHook - (connection: ApiPromiseConnection, endpoint: string, index: number) => { + ( + connection: IApiConnectionSpecific< + ApiPromise, + ApiAt, + IBlock[] | IBlock[] + >, + endpoint: string, + index: number, + ) => { const api = connection.unsafeApi; api.on('connected', () => { this.eventEmitter.emit(IndexerEvent.ApiConnected, { @@ -295,18 +304,21 @@ export class ApiService } private patchApiRpc(api: ApiPromise, apiAt: ApiAt): void { - apiAt.rpc = Object.entries(api.rpc).reduce((acc, [module, rpcMethods]) => { - acc[module] = Object.entries(rpcMethods).reduce( - (accInner, [name, rpcPromiseResult]) => { - accInner[name] = this.redecorateRpcFunction( - rpcPromiseResult as RpcMethodResult, - ); - return accInner; - }, - {}, - ); - return acc; - }, {} as ApiPromise['rpc']); + apiAt.rpc = Object.entries(api.rpc).reduce( + (acc, [module, rpcMethods]) => { + acc[module] = Object.entries(rpcMethods).reduce( + (accInner, [name, rpcPromiseResult]) => { + accInner[name] = this.redecorateRpcFunction( + rpcPromiseResult as RpcMethodResult, + ); + return accInner; + }, + {}, + ); + return acc; + }, + {} as ApiPromise['rpc'], + ); } private getRPCFunctionName( diff --git a/packages/node/src/indexer/apiPromise.connection.ts b/packages/node/src/indexer/apiPromise.connection.ts index 389c19e5e8..486dcafe7e 100644 --- a/packages/node/src/indexer/apiPromise.connection.ts +++ b/packages/node/src/indexer/apiPromise.connection.ts @@ -58,7 +58,7 @@ export class ApiPromiseConnection fetchBlocksBatches: GetFetchFunc, args: { chainTypes: RegisteredTypes }, ): Promise { - let provider: ProviderInterface; + let provider: ProviderInterface | undefined; let throwOnConnect = false; const headers = { diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index cad8d8f117..60a3bb5da8 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { @@ -29,7 +30,7 @@ export class BlockDispatcherService extends BlockDispatcher implements OnApplicationShutdown { - private runtimeService: RuntimeService; + private runtimeService!: RuntimeService; constructor( private apiService: ApiService, @@ -72,19 +73,21 @@ export class BlockDispatcherService } async init( - onDynamicDsCreated: (height: number) => Promise, + onDynamicDsCreated: (height: number) => void, runtimeService?: RuntimeService, ): Promise { await super.init(onDynamicDsCreated); - this.runtimeService = runtimeService; + if (runtimeService) this.runtimeService = runtimeService; } protected async indexBlock( block: IBlock | IBlock, ): Promise { - const runtimeVersion = !isFullBlock(block.block) - ? undefined - : await this.runtimeService.getRuntimeVersion(block.block.block); + assert(isFullBlock(block.block), 'Block must be a full block'); + const runtimeVersion = await this.runtimeService.getRuntimeVersion( + block.block.block, + ); + return this.indexerManager.indexBlock( block, await this.projectService.getDataSources(block.getHeader().blockHeight), diff --git a/packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts b/packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts index 699818bf5c..a81b4e6b03 100644 --- a/packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts +++ b/packages/node/src/indexer/blockDispatcher/substrate-block-dispatcher.ts @@ -8,7 +8,7 @@ import { RuntimeService } from '../runtime/runtimeService'; export interface ISubstrateBlockDispatcher extends IBlockDispatcher { init( - onDynamicDsCreated: (height: number) => Promise, + onDynamicDsCreated: (height: number) => void, runtimeService?: RuntimeService, ): Promise; } diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 43a396622f..da3dfc8696 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -39,7 +39,7 @@ export class WorkerBlockDispatcherService > implements OnApplicationShutdown { - private runtimeService: RuntimeService; + private runtimeService!: RuntimeService; constructor( nodeConfig: NodeConfig, @@ -90,12 +90,12 @@ export class WorkerBlockDispatcherService } async init( - onDynamicDsCreated: (height: number) => Promise, + onDynamicDsCreated: (height: number) => void, runtimeService?: RuntimeService, ): Promise { await super.init(onDynamicDsCreated); // Sync workers runtime from main - this.runtimeService = runtimeService; + if (runtimeService) this.runtimeService = runtimeService; this.syncWorkerRuntimes(); } diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts index ec9c04215c..a3a38a42dd 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts @@ -58,6 +58,7 @@ export class SubstrateDictionaryService extends DictionaryService< this.project, this.nodeConfig, this.dsProcessorService.getDsProcessor.bind( + // Note: ts-strict change this.dsProcessorService, ), endpoint, @@ -86,6 +87,7 @@ export class SubstrateDictionaryService extends DictionaryService< private getV1Dictionary(): SubstrateDictionaryV1 | undefined { // TODO this needs to be removed once Substrate supports V2 dictionaries + if (!this._currentDictionaryIndex) return undefined; const dict = this._dictionaries[this._currentDictionaryIndex]; if (!dict) return undefined; assert( diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts index ae76b00b3e..5d33f9aab2 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts @@ -54,7 +54,7 @@ describe('Substrate DictionaryService', () => { const specVersions = await dictionaryService.getSpecVersions(); - expect(specVersions.length).toBeGreaterThan(0); + expect(specVersions?.length).toBeGreaterThan(0); dictionaryService.onApplicationShutdown(); }, 50000); }); @@ -89,19 +89,19 @@ describe('Building dictionary query entries', () => { /* If there are any blockhandlers without a modulo or timestamp filter we expect no query entries */ const result1 = buildDictionaryV1QueryEntries( [makeDs([blockHandler])], - () => undefined, + () => undefined as any, ); expect(result1).toEqual([]); const result2 = buildDictionaryV1QueryEntries( [makeDs([blockHandler, callHandler, eventHandler])], - () => undefined, + () => undefined as any, ); expect(result2).toEqual([]); const result3 = buildDictionaryV1QueryEntries( [makeDs([blockHandler]), makeDs([callHandler]), makeDs([eventHandler])], - () => undefined, + () => undefined as any, ); expect(result3).toEqual([]); }); @@ -115,7 +115,7 @@ describe('Building dictionary query entries', () => { eventHandler, ]), ], - () => undefined, + () => undefined as any, ); expect(result1).toEqual([ { @@ -138,13 +138,13 @@ describe('Building dictionary query entries', () => { it('supports any handler with no filters', () => { const result1 = buildDictionaryV1QueryEntries( [makeDs([{ kind: SubstrateHandlerKind.Call, handler: 'handleCall' }])], - () => undefined, + () => undefined as any, ); expect(result1).toEqual([]); const result2 = buildDictionaryV1QueryEntries( [makeDs([{ kind: SubstrateHandlerKind.Event, handler: 'handleEvent' }])], - () => undefined, + () => undefined as any, ); expect(result2).toEqual([]); }); diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts index 8cf52ffb52..51a4881c41 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { gql } from '@apollo/client/core'; import { isCustomDs, @@ -32,10 +33,10 @@ function eventFilterToQueryEntry( return { entity: 'events', conditions: [ - { field: 'module', value: filter.module }, + { field: 'module', value: filter.module }, // Note: ts-strict change -- There are pre-judgments at runtime, but they don't cover all of them { field: 'event', - value: filter.method, + value: filter.method, // Note: ts-strict change -- There are pre-judgments at runtime, but they don't cover all of them }, ], }; @@ -51,7 +52,7 @@ function callFilterToQueryEntry( ({ field: key === 'method' ? 'call' : key, value: filter[key], - } as DictionaryQueryCondition), + }) as DictionaryQueryCondition, ), }; } @@ -62,7 +63,7 @@ function getBaseHandlerKind< ds: SubstrateDataSource, handler: SubstrateHandler, getDsProcessor: (ds: SubstrateDatasource) => P, -): SubstrateHandlerKind { +): SubstrateHandlerKind | undefined { if (isRuntimeDs(ds) && isBaseHandler(handler)) { return handler.kind; } else if (isCustomDs(ds) && isCustomHandler(handler)) { @@ -109,8 +110,9 @@ export function buildDictionaryV1QueryEntries< const plugin = isCustomDs(ds) ? getDsProcessor(ds) : undefined; for (const handler of ds.mapping.handlers) { const baseHandlerKind = getBaseHandlerKind(ds, handler, getDsProcessor); - let filterList: SubstrateRuntimeHandlerFilter[]; + let filterList: SubstrateRuntimeHandlerFilter[] = []; if (isCustomDs(ds)) { + assert(plugin, 'plugin should be defined'); const processor = plugin.handlerProcessors[handler.kind]; if (processor.dictionaryQuery) { const queryEntry = processor.dictionaryQuery( @@ -128,7 +130,7 @@ export function buildDictionaryV1QueryEntries< getDsProcessor, ); } else { - filterList = [handler.filter]; + if (handler.filter) filterList = [handler.filter]; } // Filter out any undefined filterList = filterList.filter(Boolean); @@ -189,7 +191,7 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { protected getDsProcessor: ( ds: SubstrateDatasource, ) => DsProcessor, - dictionaryUrl?: string, + dictionaryUrl: string, chainId?: string, ) { super(dictionaryUrl, chainId ?? project.network.chainId, nodeConfig); @@ -201,7 +203,7 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { getDsProcessor: ( ds: SubstrateDatasource, ) => DsProcessor, - dictionaryUrl?: string, + dictionaryUrl: string, chainId?: string, ): Promise { const dictionary = new SubstrateDictionaryV1( @@ -221,7 +223,7 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { return buildDictionaryV1QueryEntries(dataSources, this.getDsProcessor); } - parseSpecVersions(raw: SpecVersionDictionary): SpecVersion[] { + parseSpecVersions(raw: SpecVersionDictionary | undefined): SpecVersion[] { if (raw === undefined) { return []; } @@ -250,7 +252,7 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { return Array.from(specVersionBlockHeightSet); } - async getSpecVersionsRaw(): Promise { + async getSpecVersionsRaw(): Promise { const { query } = this.specVersionQuery(); try { const resp = await timeout( @@ -264,12 +266,12 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { const specVersions = resp.data.specVersions; return { _metadata, specVersions }; } catch (err) { - logger.warn(err, `failed to fetch specVersion result`); + logger.warn(err as Error, `failed to fetch specVersion result`); return undefined; } } - async getSpecVersions(): Promise { + async getSpecVersions(): Promise { try { return this.parseSpecVersions(await this.getSpecVersionsRaw()); } catch { diff --git a/packages/node/src/indexer/ds-processor.service.spec.ts b/packages/node/src/indexer/ds-processor.service.spec.ts index 3d37c519c0..8ad65a5fea 100644 --- a/packages/node/src/indexer/ds-processor.service.spec.ts +++ b/packages/node/src/indexer/ds-processor.service.spec.ts @@ -10,7 +10,7 @@ import { SubqueryProject } from '../configure/SubqueryProject'; import { DsProcessorService } from './ds-processor.service'; function getTestProject( - extraDataSources?: SubstrateCustomDatasource[], + extraDataSources: SubstrateCustomDatasource[], ): SubqueryProject { return new SubqueryProject( 'test', diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts index a0a1545a95..0db1c87b27 100644 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ b/packages/node/src/indexer/dynamic-ds.service.ts @@ -46,7 +46,9 @@ export class DynamicDsService extends BaseDynamicDsService< return dsObj; } catch (e) { - throw new Error(`Unable to create dynamic datasource.\n ${e.message}`); + throw new Error( + `Unable to create dynamic datasource.\n ${(e as any).message}`, + ); } } } diff --git a/packages/node/src/indexer/fetch.service.spec.ts b/packages/node/src/indexer/fetch.service.spec.ts index f373315dd3..d9d2d48093 100644 --- a/packages/node/src/indexer/fetch.service.spec.ts +++ b/packages/node/src/indexer/fetch.service.spec.ts @@ -60,16 +60,16 @@ describe('FetchSevice', () => { beforeEach(() => { fetchService = new FetchService( - null, // ApiService - null, // NodeConfig + null as any, // ApiService + null as any, // NodeConfig projectService, // ProjectService {} as any, // Project - null, // BlockDispatcher, - null, // DictionaryService - null, // UnfinalizedBlocks - null, // EventEmitter - null, // SchedulerRegistry - null, // RuntimeService + null as any, // BlockDispatcher, + null as any, // DictionaryService + null as any, // UnfinalizedBlocks + null as any, // EventEmitter + null as any, // SchedulerRegistry + null as any, // RuntimeService ) as any; }); diff --git a/packages/node/src/indexer/fetch.service.test.ts b/packages/node/src/indexer/fetch.service.test.ts index 296818899b..a9127ae487 100644 --- a/packages/node/src/indexer/fetch.service.test.ts +++ b/packages/node/src/indexer/fetch.service.test.ts @@ -23,19 +23,19 @@ describe('FetchService', () => { fetchService = new FetchService( apiService, // ApiService - null, // NodeConfig - null, // ProjectService + null as any, // NodeConfig + null as any, // ProjectService {} as any, // Project - null, // BlockDispatcher, - null, // DictionaryService + null as any, // BlockDispatcher, + null as any, // DictionaryService { registerFinalizedBlock: () => { /* Nothing */ }, } as any, // UnfinalizedBlocks - null, // EventEmitter - null, // SchedulerRegistry - null, // RuntimeService + null as any, // EventEmitter + null as any, // SchedulerRegistry + null as any, // RuntimeService ) as any; }, 10000); diff --git a/packages/node/src/indexer/indexer.manager.spec.ts b/packages/node/src/indexer/indexer.manager.spec.ts index 4ff11ebbca..e8f9ef5b26 100644 --- a/packages/node/src/indexer/indexer.manager.spec.ts +++ b/packages/node/src/indexer/indexer.manager.spec.ts @@ -128,7 +128,7 @@ export function mockProjectUpgradeService( project: SubqueryProject, ): IProjectUpgradeService { const startBlock = Math.min( - ...project.dataSources.map((ds) => ds.startBlock), + ...project.dataSources.map((ds) => ds.startBlock || 1), ); let currentHeight = startBlock; diff --git a/packages/node/src/indexer/runtime/base-runtime.service.ts b/packages/node/src/indexer/runtime/base-runtime.service.ts index 406a3bde26..7e0a4bbaeb 100644 --- a/packages/node/src/indexer/runtime/base-runtime.service.ts +++ b/packages/node/src/indexer/runtime/base-runtime.service.ts @@ -14,10 +14,10 @@ type GetLatestFinalizedHeight = () => number; @Injectable() export abstract class BaseRuntimeService { - parentSpecVersion: number; - specVersionMap: SpecVersion[]; - protected currentRuntimeVersion: RuntimeVersion; - latestFinalizedHeight: number; + parentSpecVersion!: number; + specVersionMap!: SpecVersion[]; + protected currentRuntimeVersion!: RuntimeVersion; + latestFinalizedHeight!: number; constructor(protected apiService: ApiService) {} @@ -60,9 +60,8 @@ export abstract class BaseRuntimeService { const parentBlockHash = await this.api.rpc.chain.getBlockHash( Math.max(height - 1, 0), ); - const runtimeVersion = await this.api.rpc.state.getRuntimeVersion( - parentBlockHash, - ); + const runtimeVersion = + await this.api.rpc.state.getRuntimeVersion(parentBlockHash); const specVersion = runtimeVersion.specVersion.toNumber(); return specVersion; } diff --git a/packages/node/src/indexer/runtime/runtime.service.spec.ts b/packages/node/src/indexer/runtime/runtime.service.spec.ts index f20e007c09..f89eb5d2bb 100644 --- a/packages/node/src/indexer/runtime/runtime.service.spec.ts +++ b/packages/node/src/indexer/runtime/runtime.service.spec.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { BlockHash, RuntimeVersion } from '@polkadot/types/interfaces'; import { ApiService } from '../api.service'; import { @@ -24,7 +25,7 @@ const specVersions: SpecVersion[] = [ { id: '12', start: 443964, end: 528470 }, { id: '13', start: 528471, end: 687751 }, { id: '14', start: 687752, end: 746085 }, - { id: '15', start: 746086, end: null }, + { id: '15', start: 746086, end: 99999999 }, ]; const getApiService = (): ApiService => @@ -41,13 +42,14 @@ const getApiService = (): ApiService => // Treat 0 as 1 for parent hash of block 1 const blockNum = Math.max(parseInt(blockHash), 1); - const { id } = specVersions.find( + const specVersion = specVersions.find( (v) => v.start <= blockNum && (!v.end || v.end >= blockNum), ); + assert(specVersion, `Spec version not found for block ${blockNum}`); return Promise.resolve({ specVersion: { - toNumber: () => parseInt(id), + toNumber: () => parseInt(specVersion.id), }, } as RuntimeVersion); }, @@ -55,7 +57,7 @@ const getApiService = (): ApiService => }, getBlockRegistry: (hash: string) => Promise.resolve(), }, - } as any); + }) as any; const getDictionaryService = (): SubstrateDictionaryService => ({ @@ -74,7 +76,7 @@ const getDictionaryService = (): SubstrateDictionaryService => initDictionariesV2: () => { //TODO }, - } as any); + }) as any; describe('Runtime service', () => { let runtimeService: RuntimeService; diff --git a/packages/node/src/indexer/runtime/runtimeService.ts b/packages/node/src/indexer/runtime/runtimeService.ts index eafce9e07b..9d7562583f 100644 --- a/packages/node/src/indexer/runtime/runtimeService.ts +++ b/packages/node/src/indexer/runtime/runtimeService.ts @@ -25,14 +25,14 @@ export class RuntimeService extends BaseRuntimeService { async syncDictionarySpecVersions(height: number): Promise { try { // must check useDictionary before get SpecVersion, this will give the right dictionary to fetch SpecVersions - const response = await this.dictionaryService.getSpecVersions(); + const response = await this.dictionaryService?.getSpecVersions(); if (response !== undefined) { this.specVersionMap = response; } else if (this.specVersionMap === undefined) { this.specVersionMap = []; } } catch (e) { - logger.error(e, 'Failed to get spec versions'); + logger.error(e as Error, 'Failed to get spec versions'); } } @@ -40,7 +40,7 @@ export class RuntimeService extends BaseRuntimeService { async getSpecVersion( blockHeight: number, ): Promise<{ blockSpecVersion: number; syncedDictionary: boolean }> { - let blockSpecVersion: number; + let blockSpecVersion: number | undefined; let syncedDictionary = false; // we want to keep the specVersionMap in memory, and use it even useDictionary been disabled // therefore instead of check .useDictionary, we check it length before use it. diff --git a/packages/node/src/indexer/runtime/workerRuntimeService.ts b/packages/node/src/indexer/runtime/workerRuntimeService.ts index eab0159650..0b4c58cde3 100644 --- a/packages/node/src/indexer/runtime/workerRuntimeService.ts +++ b/packages/node/src/indexer/runtime/workerRuntimeService.ts @@ -14,7 +14,7 @@ export class WorkerRuntimeService extends BaseRuntimeService { ): void { this.specVersionMap = specVersionMap; if ( - latestFinalizedHeight !== undefined || + latestFinalizedHeight !== undefined && this.latestFinalizedHeight < latestFinalizedHeight ) { this.latestFinalizedHeight = latestFinalizedHeight; @@ -26,7 +26,7 @@ export class WorkerRuntimeService extends BaseRuntimeService { async getSpecVersion( blockHeight: number, ): Promise<{ blockSpecVersion: number; syncedDictionary: boolean }> { - let blockSpecVersion: number; + let blockSpecVersion: number | undefined; // we want to keep the specVersionMap in memory, and use it even useDictionary been disabled // therefore instead of check .useDictionary, we check it length before use it. if (this.specVersionMap && this.specVersionMap.length !== 0) { diff --git a/packages/node/src/indexer/store.service.test.ts b/packages/node/src/indexer/store.service.test.ts index 45df5e968f..916022f0fa 100644 --- a/packages/node/src/indexer/store.service.test.ts +++ b/packages/node/src/indexer/store.service.test.ts @@ -77,8 +77,8 @@ describe('Store service integration test', () => { replacements: { schema: schemaName }, }, ); - const expectedTables = result.map( - (t: { table_name: string }) => t.table_name, + const expectedTables = (result as { table_name: string }[]).map( + (t) => t.table_name, ); const columnResult = await sequelize.query( @@ -278,7 +278,7 @@ WHERE tempDir = (projectService as any).project.root; - const result = await sequelize.query( + const result: { enum_type: string }[] = await sequelize.query( ` SELECT n.nspname AS schema_name, t.typname AS enum_type, @@ -291,9 +291,11 @@ ORDER BY t.typname, e.enumsortorder;`, { type: QueryTypes.SELECT, replacements: { schema: schemaName } }, ); expect(result.length).toBe(3); - expect(result.map((r: { enum_type: string }) => r.enum_type)).toStrictEqual( - ['65c7fd4e5d', '65c7fd4e5d', '65c7fd4e5d'], - ); + expect(result.map((r) => r.enum_type)).toStrictEqual([ + '65c7fd4e5d', + '65c7fd4e5d', + '65c7fd4e5d', + ]); }); it('Able to drop notification triggers and functions', async () => { // if subscription is no longer enabled should be able to drop all prior triggers and functions related to subscription diff --git a/packages/node/src/indexer/worker/worker.service.ts b/packages/node/src/indexer/worker/worker.service.ts index 1d9a7410cb..81f1deefb6 100644 --- a/packages/node/src/indexer/worker/worker.service.ts +++ b/packages/node/src/indexer/worker/worker.service.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { Inject, Injectable } from '@nestjs/common'; import { NodeConfig, @@ -67,9 +68,10 @@ export class WorkerService extends BaseWorkerService< block: IBlock, dataSources: SubstrateDatasource[], ): Promise { - const runtimeVersion = !isFullBlock(block.block) - ? undefined - : await this.workerRuntimeService.getRuntimeVersion(block.block.block); + assert(isFullBlock(block.block), 'Block should be Full block'); + const runtimeVersion = await this.workerRuntimeService.getRuntimeVersion( + block.block.block, + ); return this.indexerManager.indexBlock(block, dataSources, runtimeVersion); } diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts index a4097fdd83..7c2d9cf755 100644 --- a/packages/node/src/indexer/worker/worker.ts +++ b/packages/node/src/indexer/worker/worker.ts @@ -33,7 +33,7 @@ import { WorkerService } from './worker.service'; const logger = getLogger(`worker #${threadId}`); -async function initWorker(startHeight: number): Promise { +async function initWorker(startHeight?: number): Promise { try { const app = await NestFactory.create(WorkerModule, { logger: new NestLogger(!!argv.debug), // TIP: If the worker is crashing comment out this line for better logging @@ -49,7 +49,7 @@ async function initWorker(startHeight: number): Promise { initWorkerServices(app, workerService); } catch (e) { console.log('Failed to start worker', e); - logger.error(e, 'Failed to start worker'); + logger.error(e as Error, 'Failed to start worker'); throw e; } } diff --git a/packages/node/src/utils/project.ts b/packages/node/src/utils/project.ts index 997d032caf..0e814d70db 100644 --- a/packages/node/src/utils/project.ts +++ b/packages/node/src/utils/project.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import fs from 'fs'; import path from 'path'; import { LocalReader, loadFromJsonOrYaml } from '@subql/common'; @@ -42,7 +43,7 @@ export async function getChainTypes( ): Promise { // If the project is load from local, we will direct load them if (reader instanceof LocalReader) { - return loadChainTypes(file, root); + return loadChainTypes(file, root) as ChainTypes; } else { // If it is stored in ipfs or other resources, we will use the corresponding reader to read the file // Because ipfs not provide extension of the file, it is difficult to determine its format @@ -50,6 +51,8 @@ export async function getChainTypes( // if it failed, we will give it another another attempt, and assume the script written in js // we will download it to a temp folder, and load them within sandbox const res = await reader.getFile(file); + assert(res, `File ${file} not found`); + let raw: unknown; try { raw = yaml.load(res); diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts index a7e5406cc8..2ee40886eb 100644 --- a/packages/node/src/utils/substrate.ts +++ b/packages/node/src/utils/substrate.ts @@ -51,6 +51,7 @@ export function wrapBlock( specVersion?: number, ): SubstrateBlock { return merge(signedBlock, { + // Note: ts-strict change -- t is possible for the runtime to pass in timestamp: getTimestamp(signedBlock), specVersion: specVersion, events, @@ -70,6 +71,7 @@ export function getTimestamp({ block: { extrinsics } }: SignedBlock): Date { return date; } } + throw new Error('timestamp not found'); } export function wrapExtrinsics( @@ -375,12 +377,18 @@ export async function fetchBlocksBatches( ? undefined : fetchRuntimeVersionRange(api, parentBlockHashs), ]); + return blocks.map((block, idx) => { const events = blockEvents[idx]; - const parentSpecVersion = - overallSpecVer !== undefined - ? overallSpecVer - : runtimeVersions[idx].specVersion.toNumber(); + let parentSpecVersion: number | undefined; + if (overallSpecVer !== undefined) { + parentSpecVersion = overallSpecVer; + } else { + parentSpecVersion = runtimeVersions + ? runtimeVersions[idx].specVersion.toNumber() + : undefined; + } + const wrappedBlock = wrapBlock(block, events.toArray(), parentSpecVersion); const wrappedExtrinsics = wrapExtrinsics(wrappedBlock, events); const wrappedEvents = wrapEvents(wrappedExtrinsics, events, wrappedBlock); @@ -450,8 +458,8 @@ export function calcInterval(api: ApiPromise): BN { (api.consts.timestamp?.minimumPeriod.gte(INTERVAL_THRESHOLD) ? api.consts.timestamp.minimumPeriod.mul(BN_TWO) : api.query.parachainSystem - ? DEFAULT_TIME.mul(BN_TWO) - : DEFAULT_TIME), + ? DEFAULT_TIME.mul(BN_TWO) + : DEFAULT_TIME), ); } diff --git a/packages/node/tsconfig.json b/packages/node/tsconfig.json index 4a94b628d0..621baaff7a 100644 --- a/packages/node/tsconfig.json +++ b/packages/node/tsconfig.json @@ -4,8 +4,7 @@ "sourceMap": true, "tsBuildInfoFile": "dist/.tsbuildinfo", "rootDir": "src", - "outDir": "./dist", - "strict": true + "outDir": "./dist" }, "references": [ { "path": "../common" }, From b20e664b3ff2f7f806e0002816c10fbf8f6d4f79 Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 18 Jun 2024 16:03:28 +0800 Subject: [PATCH 03/25] some change --- packages/node/src/indexer/api.service.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index be12c4c980..f4a23d66eb 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -22,7 +22,6 @@ import { IBlock, MetadataMismatchError, exitWithError, - IApiConnectionSpecific, } from '@subql/node-core'; import { SubstrateNodeConfig } from '../configure/NodeConfig'; import { SubqueryProject } from '../configure/SubqueryProject'; @@ -102,7 +101,8 @@ export class ApiService extends BaseApiService< ApiPromise, ApiAt, - IBlock[] | IBlock[] + IBlock[] | IBlock[], + ApiPromiseConnection > implements OnApplicationShutdown { @@ -155,15 +155,7 @@ export class ApiService chainTypes, }), //postConnectedHook - ( - connection: IApiConnectionSpecific< - ApiPromise, - ApiAt, - IBlock[] | IBlock[] - >, - endpoint: string, - index: number, - ) => { + (connection: ApiPromiseConnection, endpoint: string, index: number) => { const api = connection.unsafeApi; api.on('connected', () => { this.eventEmitter.emit(IndexerEvent.ApiConnected, { From a04458c71e0095957d8d546f13723072e331d9ea Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 18 Jun 2024 16:25:27 +0800 Subject: [PATCH 04/25] some changes --- packages/node/src/indexer/api.service.ts | 34 +++++++++++-- .../indexer/runtime/base-runtime.service.ts | 49 +++++++++++++++++-- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index f4a23d66eb..21a60c977b 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { ApiPromise } from '@polkadot/api'; @@ -106,10 +107,10 @@ export class ApiService > implements OnApplicationShutdown { - private fetchBlocksFunction!: FetchFunc; + private _fetchBlocksFunction?: FetchFunc; private fetchBlocksBatches: GetFetchFunc = () => this.fetchBlocksFunction; - private currentBlockHash!: string; - private currentBlockNumber!: number; + private _currentBlockHash?: string; + private _currentBlockNumber?: number; private nodeConfig: SubstrateNodeConfig; @@ -125,6 +126,33 @@ export class ApiService this.updateBlockFetching(); } + private get fetchBlocksFunction(): FetchFunc { + assert(this._fetchBlocksFunction, 'fetchBlocksFunction not initialized'); + return this._fetchBlocksFunction; + } + + private set fetchBlocksFunction(value: FetchFunc) { + this._fetchBlocksFunction = value; + } + + private get currentBlockHash(): string { + assert(this._currentBlockHash, 'currentBlockHash not initialized'); + return this._currentBlockHash; + } + + private set currentBlockHash(value: string) { + this._currentBlockHash = value; + } + + private get currentBlockNumber(): number { + assert(this._currentBlockNumber, 'currentBlockNumber not initialized'); + return this._currentBlockNumber; + } + + private set currentBlockNumber(value: number) { + this._currentBlockNumber = value; + } + async onApplicationShutdown(): Promise { await this.connectionPoolService.onApplicationShutdown(); } diff --git a/packages/node/src/indexer/runtime/base-runtime.service.ts b/packages/node/src/indexer/runtime/base-runtime.service.ts index 7e0a4bbaeb..8119c8c284 100644 --- a/packages/node/src/indexer/runtime/base-runtime.service.ts +++ b/packages/node/src/indexer/runtime/base-runtime.service.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { Injectable } from '@nestjs/common'; import { ApiPromise } from '@polkadot/api'; import { RuntimeVersion } from '@polkadot/types/interfaces'; @@ -14,13 +15,53 @@ type GetLatestFinalizedHeight = () => number; @Injectable() export abstract class BaseRuntimeService { - parentSpecVersion!: number; - specVersionMap!: SpecVersion[]; - protected currentRuntimeVersion!: RuntimeVersion; - latestFinalizedHeight!: number; + private _parentSpecVersion?: number; + private _specVersionMap?: SpecVersion[]; + protected _currentRuntimeVersion?: RuntimeVersion; + private _latestFinalizedHeight?: number; constructor(protected apiService: ApiService) {} + get parentSpecVersion(): number { + assert( + this._parentSpecVersion !== undefined, + 'parentSpecVersion is undefined', + ); + return this._parentSpecVersion; + } + set parentSpecVersion(value: number) { + this._parentSpecVersion = value; + } + get specVersionMap(): SpecVersion[] { + assert(this._specVersionMap !== undefined, 'specVersionMap is undefined'); + return this._specVersionMap; + } + set specVersionMap(value: SpecVersion[]) { + this._specVersionMap = value; + } + + protected get currentRuntimeVersion(): RuntimeVersion { + assert( + this._currentRuntimeVersion !== undefined, + 'currentRuntimeVersion is undefined', + ); + return this._currentRuntimeVersion; + } + protected set currentRuntimeVersion(value: RuntimeVersion) { + this._currentRuntimeVersion = value; + } + + get latestFinalizedHeight(): number { + assert( + this._latestFinalizedHeight !== undefined, + 'latestFinalizedHeight is undefined', + ); + return this._latestFinalizedHeight; + } + set latestFinalizedHeight(value: number) { + this._latestFinalizedHeight = value; + } + async specChanged(height: number, specVersion: number): Promise { if (this.parentSpecVersion !== specVersion) { const parentSpecVersionCopy = this.parentSpecVersion; From 105a60b6c79d7a41d9c75e5f2965aa96ced3c12a Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 18 Jun 2024 16:28:31 +0800 Subject: [PATCH 05/25] some changes --- .../blockDispatcher/block-dispatcher.service.ts | 10 +++++++++- .../worker-block-dispatcher.service.ts | 12 +++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index 60a3bb5da8..b7f762d106 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -30,7 +30,7 @@ export class BlockDispatcherService extends BlockDispatcher implements OnApplicationShutdown { - private runtimeService!: RuntimeService; + private _runtimeService?: RuntimeService; constructor( private apiService: ApiService, @@ -72,6 +72,14 @@ export class BlockDispatcherService ); } + private get runtimeService(): RuntimeService { + assert(this._runtimeService, 'Runtime service not initialized'); + return this._runtimeService; + } + private set runtimeService(value: RuntimeService) { + this._runtimeService = value; + } + async init( onDynamicDsCreated: (height: number) => void, runtimeService?: RuntimeService, diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index da3dfc8696..200f84754c 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import path from 'path'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; @@ -39,7 +40,7 @@ export class WorkerBlockDispatcherService > implements OnApplicationShutdown { - private runtimeService!: RuntimeService; + private _runtimeService?: RuntimeService; constructor( nodeConfig: NodeConfig, @@ -89,6 +90,15 @@ export class WorkerBlockDispatcherService ); } + private get runtimeService(): RuntimeService { + assert(this._runtimeService, 'RuntimeService not initialized'); + return this._runtimeService; + } + + private set runtimeService(runtimeService: RuntimeService) { + this._runtimeService = runtimeService; + } + async init( onDynamicDsCreated: (height: number) => void, runtimeService?: RuntimeService, From bb971bbe777b73f095e430dce08be4f9599af18b Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 18 Jun 2024 17:04:25 +0800 Subject: [PATCH 06/25] change somes --- packages/node/src/indexer/api.service.ts | 2 +- .../indexer/blockDispatcher/block-dispatcher.service.ts | 7 +++---- packages/node/src/indexer/indexer.manager.ts | 4 ++-- packages/node/src/indexer/worker/worker.service.ts | 7 +++---- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 21a60c977b..40798de7e6 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -251,7 +251,7 @@ export class ApiService async getPatchedApi( header: Header, - runtimeVersion: RuntimeVersion, + runtimeVersion?: RuntimeVersion, ): Promise { this.currentBlockHash = header.hash.toString(); this.currentBlockNumber = header.number.toNumber(); diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index b7f762d106..d5e74ad5dd 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -91,10 +91,9 @@ export class BlockDispatcherService protected async indexBlock( block: IBlock | IBlock, ): Promise { - assert(isFullBlock(block.block), 'Block must be a full block'); - const runtimeVersion = await this.runtimeService.getRuntimeVersion( - block.block.block, - ); + const runtimeVersion = !isFullBlock(block.block) + ? undefined + : await this.runtimeService.getRuntimeVersion(block.block.block); return this.indexerManager.indexBlock( block, diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index afc3f8939d..cb1e011454 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -78,7 +78,7 @@ export class IndexerManager extends BaseIndexerManager< async indexBlock( block: IBlock, dataSources: SubstrateDatasource[], - runtimeVersion: RuntimeVersion, + runtimeVersion?: RuntimeVersion, ): Promise { return super.internalIndexBlock(block, dataSources, () => this.getApi(block.block, runtimeVersion), @@ -88,7 +88,7 @@ export class IndexerManager extends BaseIndexerManager< // eslint-disable-next-line @typescript-eslint/require-await private async getApi( block: LightBlockContent | BlockContent, - runtimeVersion: RuntimeVersion, + runtimeVersion?: RuntimeVersion, ): Promise { return this.apiService.getPatchedApi( block.block.block.header, diff --git a/packages/node/src/indexer/worker/worker.service.ts b/packages/node/src/indexer/worker/worker.service.ts index 81f1deefb6..7b87ebdfbe 100644 --- a/packages/node/src/indexer/worker/worker.service.ts +++ b/packages/node/src/indexer/worker/worker.service.ts @@ -68,10 +68,9 @@ export class WorkerService extends BaseWorkerService< block: IBlock, dataSources: SubstrateDatasource[], ): Promise { - assert(isFullBlock(block.block), 'Block should be Full block'); - const runtimeVersion = await this.workerRuntimeService.getRuntimeVersion( - block.block.block, - ); + const runtimeVersion = !isFullBlock(block.block) + ? undefined + : await this.workerRuntimeService.getRuntimeVersion(block.block.block); return this.indexerManager.indexBlock(block, dataSources, runtimeVersion); } From 61a1eae47487d26739254cf8cf8eeecfa1ff0586 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 19 Jun 2024 16:55:16 +0800 Subject: [PATCH 07/25] some change --- packages/node-core/src/utils/project.ts | 8 ++++---- packages/node/src/configure/SubqueryProject.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/node-core/src/utils/project.ts b/packages/node-core/src/utils/project.ts index f9941fbcc2..53b09a930a 100644 --- a/packages/node-core/src/utils/project.ts +++ b/packages/node-core/src/utils/project.ts @@ -229,13 +229,13 @@ export async function initHotSchemaReload(schema: string, storeService: StoreSer await storeService.initHotSchemaReloadQueries(schema); } -type IsRuntimeDs = (ds: BaseDataSource) => boolean; +type IsRuntimeDs = (ds: DS) => ds is DS; // eslint-disable-next-line @typescript-eslint/require-await export async function insertBlockFiltersCronSchedules( dataSources: DS[], getBlockTimestamp: (height: number) => Promise, - isRuntimeDs: IsRuntimeDs, + isRuntimeDs: IsRuntimeDs, blockHandlerKind: string ): Promise { dataSources = await Promise.all( @@ -282,7 +282,7 @@ export async function loadProjectTemplates + isCustomDs: IsCustomDs ): Promise { if (!templates || !templates.length) { return []; @@ -291,7 +291,7 @@ export async function loadProjectTemplates ({ ...ds, name: templates[index].name, - })) as T[]; // How to get rid of cast here? + })) as T[]; } export function getStartHeight(dataSources: BaseDataSource[]): number { diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index 2be5557d59..e8eb22eca4 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -72,7 +72,7 @@ export class SubqueryProject implements ISubqueryProject { this.#dataSources = await insertBlockFiltersCronSchedules( this.dataSources, getTimestamp, - isRuntimeDs, // Note: ts-strict change --- Not sure it should be modified + isRuntimeDs, SubstrateHandlerKind.Block, ); } @@ -172,7 +172,7 @@ async function loadProjectFromManifestBase( projectManifest.templates, root, reader, - isCustomDs, // Note: ts-strict change --- Not sure it should be modified + isCustomDs, ); const runner = projectManifest.runner; assert( From 239766c67b48493c4976fd665f7ad6b96fabe54e Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 19 Jun 2024 17:15:14 +0800 Subject: [PATCH 08/25] some changes --- packages/node-core/src/utils/project.ts | 2 +- .../src/configure/SchemaMigration.service.test.ts | 2 +- packages/node/src/configure/SubqueryProject.ts | 11 +++-------- packages/node/src/indexer/dictionary/types.ts | 2 +- .../indexer/dictionary/v1/substrateDictionaryV1.ts | 2 +- .../node/src/indexer/runtime/base-runtime.service.ts | 8 ++++++-- packages/node/src/utils/substrate.ts | 12 ++++-------- 7 files changed, 17 insertions(+), 22 deletions(-) diff --git a/packages/node-core/src/utils/project.ts b/packages/node-core/src/utils/project.ts index 65905bfbd3..9c594ddb77 100644 --- a/packages/node-core/src/utils/project.ts +++ b/packages/node-core/src/utils/project.ts @@ -17,7 +17,7 @@ import {exitWithError} from '../process'; const logger = getLogger('Project-Utils'); -export async function getValidPort(argvPort: number | undefined): Promise { +export async function getValidPort(argvPort?: number): Promise { const validate = (x: any) => { const p = parseInt(x); return isNaN(p) ? null : p; diff --git a/packages/node/src/configure/SchemaMigration.service.test.ts b/packages/node/src/configure/SchemaMigration.service.test.ts index a4a428bd14..3e5213d110 100644 --- a/packages/node/src/configure/SchemaMigration.service.test.ts +++ b/packages/node/src/configure/SchemaMigration.service.test.ts @@ -58,7 +58,7 @@ describe('SchemaMigration integration tests', () => { await apiService.init(); await projectService.init(1); - const dbResults: string[][] = await sequelize.query( + const dbResults = await sequelize.query( `SELECT table_name FROM information_schema.tables WHERE table_schema= :schema;`, { type: QueryTypes.SELECT, replacements: { schema: schemaName } }, ); diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index e8eb22eca4..abb373b34a 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -146,14 +146,9 @@ async function loadProjectFromManifestBase( ), ); - let schemaString: string | undefined; - try { - schemaString = await reader.getFile(projectManifest.schema.file); - } catch (e) { - throw new Error( - `unable to fetch the schema from ${projectManifest.schema.file}`, - ); - } + const schemaString: string = await reader.getFile( + projectManifest.schema.file, + ); assert(schemaString, 'Schema file is empty'); const schema = buildSchemaFromString(schemaString); diff --git a/packages/node/src/indexer/dictionary/types.ts b/packages/node/src/indexer/dictionary/types.ts index 0203bf297f..a93961cf7b 100644 --- a/packages/node/src/indexer/dictionary/types.ts +++ b/packages/node/src/indexer/dictionary/types.ts @@ -6,7 +6,7 @@ import { MetaData } from '@subql/utils'; export type SpecVersion = { id: string; start: number; //start with this block - end: number; + end: number | null; }; export type SpecVersionDictionary = { diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts index 51a4881c41..4ffd66a8ca 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts @@ -223,7 +223,7 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { return buildDictionaryV1QueryEntries(dataSources, this.getDsProcessor); } - parseSpecVersions(raw: SpecVersionDictionary | undefined): SpecVersion[] { + parseSpecVersions(raw?: SpecVersionDictionary): SpecVersion[] { if (raw === undefined) { return []; } diff --git a/packages/node/src/indexer/runtime/base-runtime.service.ts b/packages/node/src/indexer/runtime/base-runtime.service.ts index 8119c8c284..41041881a3 100644 --- a/packages/node/src/indexer/runtime/base-runtime.service.ts +++ b/packages/node/src/indexer/runtime/base-runtime.service.ts @@ -92,7 +92,9 @@ export abstract class BaseRuntimeService { ): number | undefined { //return undefined block can not find inside range const spec = specVersions.find( - (spec) => blockHeight >= spec.start && blockHeight <= spec.end, + (spec) => + blockHeight >= spec.start && + (spec.end !== null ? blockHeight <= spec.end : true), ); return spec ? Number(spec.id) : undefined; } @@ -123,7 +125,9 @@ export abstract class BaseRuntimeService { } else { for (const specVersion of this.specVersionMap) { if ( - specVersion.start > parentSpecVersion.end && + (parentSpecVersion.end !== null + ? specVersion.start > parentSpecVersion.end + : true) && specVersion.start <= height ) { const blockHash = await this.api.rpc.chain.getBlockHash( diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts index 2ee40886eb..183459523e 100644 --- a/packages/node/src/utils/substrate.ts +++ b/packages/node/src/utils/substrate.ts @@ -380,14 +380,10 @@ export async function fetchBlocksBatches( return blocks.map((block, idx) => { const events = blockEvents[idx]; - let parentSpecVersion: number | undefined; - if (overallSpecVer !== undefined) { - parentSpecVersion = overallSpecVer; - } else { - parentSpecVersion = runtimeVersions - ? runtimeVersions[idx].specVersion.toNumber() - : undefined; - } + const parentSpecVersion = + overallSpecVer !== undefined + ? overallSpecVer + : runtimeVersions?.[idx].specVersion.toNumber(); const wrappedBlock = wrapBlock(block, events.toArray(), parentSpecVersion); const wrappedExtrinsics = wrapExtrinsics(wrappedBlock, events); From 9bda434f5afa6e0a36b9466ddb7b323b34425dc0 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 19 Jun 2024 17:24:15 +0800 Subject: [PATCH 09/25] some changes --- .../dictionary/substrateDictionary.service.ts | 1 - .../dictionary/v1/substrateDictionaryV1.ts | 16 +++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts index a3a38a42dd..f0f59ec2d7 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts @@ -58,7 +58,6 @@ export class SubstrateDictionaryService extends DictionaryService< this.project, this.nodeConfig, this.dsProcessorService.getDsProcessor.bind( - // Note: ts-strict change this.dsProcessorService, ), endpoint, diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts index 4ffd66a8ca..ca181bdf42 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts @@ -30,15 +30,17 @@ import { SpecVersion, SpecVersionDictionary } from '../types'; function eventFilterToQueryEntry( filter: SubstrateEventFilter, ): DictionaryV1QueryEntry { + const conditions: DictionaryQueryCondition[] = []; + if (filter.module) { + conditions.push({ field: 'module', value: filter.module }); + } + if (filter.method) { + conditions.push({ field: 'event', value: filter.method }); + } + return { entity: 'events', - conditions: [ - { field: 'module', value: filter.module }, // Note: ts-strict change -- There are pre-judgments at runtime, but they don't cover all of them - { - field: 'event', - value: filter.method, // Note: ts-strict change -- There are pre-judgments at runtime, but they don't cover all of them - }, - ], + conditions: conditions, }; } From 1bff8b275010ee60767745a7b84852c7b335a304 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 19 Jun 2024 18:09:39 +0800 Subject: [PATCH 10/25] some change --- .../node/src/configure/SubqueryProject.ts | 2 +- .../indexer/runtime/base-runtime.service.ts | 39 ++++++------------- .../node/src/indexer/worker/worker.service.ts | 2 +- packages/node/src/utils/substrate.ts | 1 - packages/node/tsconfig.json | 3 +- packages/types/src/interfaces.ts | 2 +- 6 files changed, 16 insertions(+), 33 deletions(-) diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index abb373b34a..fc42ccee9c 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -167,7 +167,7 @@ async function loadProjectFromManifestBase( projectManifest.templates, root, reader, - isCustomDs, + isCustomDs as any, // TODO: There is no good way to temporarily ); const runner = projectManifest.runner; assert( diff --git a/packages/node/src/indexer/runtime/base-runtime.service.ts b/packages/node/src/indexer/runtime/base-runtime.service.ts index 41041881a3..72fe104f86 100644 --- a/packages/node/src/indexer/runtime/base-runtime.service.ts +++ b/packages/node/src/indexer/runtime/base-runtime.service.ts @@ -15,29 +15,23 @@ type GetLatestFinalizedHeight = () => number; @Injectable() export abstract class BaseRuntimeService { - private _parentSpecVersion?: number; - private _specVersionMap?: SpecVersion[]; + parentSpecVersion?: number; + specVersionMap: SpecVersion[] = []; protected _currentRuntimeVersion?: RuntimeVersion; private _latestFinalizedHeight?: number; constructor(protected apiService: ApiService) {} - get parentSpecVersion(): number { + get latestFinalizedHeight(): number { assert( - this._parentSpecVersion !== undefined, - 'parentSpecVersion is undefined', + this._latestFinalizedHeight !== undefined, + 'latestFinalizedHeight is undefined', ); - return this._parentSpecVersion; - } - set parentSpecVersion(value: number) { - this._parentSpecVersion = value; - } - get specVersionMap(): SpecVersion[] { - assert(this._specVersionMap !== undefined, 'specVersionMap is undefined'); - return this._specVersionMap; + return this._latestFinalizedHeight; } - set specVersionMap(value: SpecVersion[]) { - this._specVersionMap = value; + + set latestFinalizedHeight(value: number) { + this._latestFinalizedHeight = value; } protected get currentRuntimeVersion(): RuntimeVersion { @@ -51,17 +45,6 @@ export abstract class BaseRuntimeService { this._currentRuntimeVersion = value; } - get latestFinalizedHeight(): number { - assert( - this._latestFinalizedHeight !== undefined, - 'latestFinalizedHeight is undefined', - ); - return this._latestFinalizedHeight; - } - set latestFinalizedHeight(value: number) { - this._latestFinalizedHeight = value; - } - async specChanged(height: number, specVersion: number): Promise { if (this.parentSpecVersion !== specVersion) { const parentSpecVersionCopy = this.parentSpecVersion; @@ -88,10 +71,10 @@ export abstract class BaseRuntimeService { getSpecFromMap( blockHeight: number, - specVersions: SpecVersion[], + specVersions?: SpecVersion[], ): number | undefined { //return undefined block can not find inside range - const spec = specVersions.find( + const spec = specVersions?.find( (spec) => blockHeight >= spec.start && (spec.end !== null ? blockHeight <= spec.end : true), diff --git a/packages/node/src/indexer/worker/worker.service.ts b/packages/node/src/indexer/worker/worker.service.ts index 7b87ebdfbe..bde16c2704 100644 --- a/packages/node/src/indexer/worker/worker.service.ts +++ b/packages/node/src/indexer/worker/worker.service.ts @@ -18,7 +18,7 @@ import { IndexerManager } from '../indexer.manager'; import { WorkerRuntimeService } from '../runtime/workerRuntimeService'; import { BlockContent, isFullBlock, LightBlockContent } from '../types'; -export type FetchBlockResponse = { specVersion: number; parentHash: string }; +export type FetchBlockResponse = { specVersion?: number; parentHash: string }; @Injectable() export class WorkerService extends BaseWorkerService< diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts index 183459523e..8c53f065ae 100644 --- a/packages/node/src/utils/substrate.ts +++ b/packages/node/src/utils/substrate.ts @@ -51,7 +51,6 @@ export function wrapBlock( specVersion?: number, ): SubstrateBlock { return merge(signedBlock, { - // Note: ts-strict change -- t is possible for the runtime to pass in timestamp: getTimestamp(signedBlock), specVersion: specVersion, events, diff --git a/packages/node/tsconfig.json b/packages/node/tsconfig.json index 621baaff7a..4a94b628d0 100644 --- a/packages/node/tsconfig.json +++ b/packages/node/tsconfig.json @@ -4,7 +4,8 @@ "sourceMap": true, "tsBuildInfoFile": "dist/.tsbuildinfo", "rootDir": "src", - "outDir": "./dist" + "outDir": "./dist", + "strict": true }, "references": [ { "path": "../common" }, diff --git a/packages/types/src/interfaces.ts b/packages/types/src/interfaces.ts index df9f07d652..090e2bff5b 100644 --- a/packages/types/src/interfaces.ts +++ b/packages/types/src/interfaces.ts @@ -8,7 +8,7 @@ import {IEvent} from '@polkadot/types/types'; export interface SubstrateBlock extends SignedBlock { // parent block's spec version, can be used to decide the correct metadata that should be used for this block. - specVersion: number; + specVersion?: number; timestamp: Date; events: EventRecord[]; } From 850628e9b150e4e4c53d4b6d1f0a755868cb7319 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 19 Jun 2024 18:10:50 +0800 Subject: [PATCH 11/25] some changes --- packages/node/src/indexer/runtime/runtime.service.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node/src/indexer/runtime/runtime.service.spec.ts b/packages/node/src/indexer/runtime/runtime.service.spec.ts index f89eb5d2bb..453ba3cfba 100644 --- a/packages/node/src/indexer/runtime/runtime.service.spec.ts +++ b/packages/node/src/indexer/runtime/runtime.service.spec.ts @@ -25,7 +25,7 @@ const specVersions: SpecVersion[] = [ { id: '12', start: 443964, end: 528470 }, { id: '13', start: 528471, end: 687751 }, { id: '14', start: 687752, end: 746085 }, - { id: '15', start: 746086, end: 99999999 }, + { id: '15', start: 746086, end: null }, ]; const getApiService = (): ApiService => From e68525aa8256eb8d03066f5a632d62207f53751e Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 19 Jun 2024 18:17:05 +0800 Subject: [PATCH 12/25] some changes --- .../dictionary/substrateDictionary.service.ts | 2 +- .../src/indexer/runtime/base-runtime.service.ts | 14 +------------- .../node/src/indexer/runtime/runtimeService.ts | 5 ++++- .../src/indexer/runtime/workerRuntimeService.ts | 1 + 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts index f0f59ec2d7..6559e72b63 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts @@ -59,7 +59,7 @@ export class SubstrateDictionaryService extends DictionaryService< this.nodeConfig, this.dsProcessorService.getDsProcessor.bind( this.dsProcessorService, - ), + ) as any, // TODO: There is no good way to temporarily endpoint, ); dictionariesV1.push(dictionaryV1); diff --git a/packages/node/src/indexer/runtime/base-runtime.service.ts b/packages/node/src/indexer/runtime/base-runtime.service.ts index 72fe104f86..9236eb1757 100644 --- a/packages/node/src/indexer/runtime/base-runtime.service.ts +++ b/packages/node/src/indexer/runtime/base-runtime.service.ts @@ -18,22 +18,10 @@ export abstract class BaseRuntimeService { parentSpecVersion?: number; specVersionMap: SpecVersion[] = []; protected _currentRuntimeVersion?: RuntimeVersion; - private _latestFinalizedHeight?: number; + latestFinalizedHeight?: number; constructor(protected apiService: ApiService) {} - get latestFinalizedHeight(): number { - assert( - this._latestFinalizedHeight !== undefined, - 'latestFinalizedHeight is undefined', - ); - return this._latestFinalizedHeight; - } - - set latestFinalizedHeight(value: number) { - this._latestFinalizedHeight = value; - } - protected get currentRuntimeVersion(): RuntimeVersion { assert( this._currentRuntimeVersion !== undefined, diff --git a/packages/node/src/indexer/runtime/runtimeService.ts b/packages/node/src/indexer/runtime/runtimeService.ts index 9d7562583f..ca1236f9ae 100644 --- a/packages/node/src/indexer/runtime/runtimeService.ts +++ b/packages/node/src/indexer/runtime/runtimeService.ts @@ -49,7 +49,10 @@ export class RuntimeService extends BaseRuntimeService { } if (blockSpecVersion === undefined) { blockSpecVersion = await this.getSpecFromApi(blockHeight); - if (blockHeight + SPEC_VERSION_BLOCK_GAP < this.latestFinalizedHeight) { + if ( + this.latestFinalizedHeight && + blockHeight + SPEC_VERSION_BLOCK_GAP < this.latestFinalizedHeight + ) { // Ask to sync local specVersionMap with dictionary await this.syncDictionarySpecVersions(blockHeight); syncedDictionary = true; diff --git a/packages/node/src/indexer/runtime/workerRuntimeService.ts b/packages/node/src/indexer/runtime/workerRuntimeService.ts index 0b4c58cde3..41a6b1c321 100644 --- a/packages/node/src/indexer/runtime/workerRuntimeService.ts +++ b/packages/node/src/indexer/runtime/workerRuntimeService.ts @@ -15,6 +15,7 @@ export class WorkerRuntimeService extends BaseRuntimeService { this.specVersionMap = specVersionMap; if ( latestFinalizedHeight !== undefined && + this.latestFinalizedHeight && this.latestFinalizedHeight < latestFinalizedHeight ) { this.latestFinalizedHeight = latestFinalizedHeight; From f4eb261788c61d85c59fe05ad1b3dffb709fb08e Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 20 Jun 2024 13:41:30 +0800 Subject: [PATCH 13/25] some changes --- packages/node/src/indexer/api.service.ts | 8 ++------ .../src/indexer/dictionary/v1/substrateDictionaryV1.ts | 4 ++-- packages/node/src/utils/project.ts | 1 - 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/node/src/indexer/api.service.ts b/packages/node/src/indexer/api.service.ts index 40798de7e6..a35f350a2e 100644 --- a/packages/node/src/indexer/api.service.ts +++ b/packages/node/src/indexer/api.service.ts @@ -131,10 +131,6 @@ export class ApiService return this._fetchBlocksFunction; } - private set fetchBlocksFunction(value: FetchFunc) { - this._fetchBlocksFunction = value; - } - private get currentBlockHash(): string { assert(this._currentBlockHash, 'currentBlockHash not initialized'); return this._currentBlockHash; @@ -235,13 +231,13 @@ export class ApiService : SubstrateUtil.fetchBlocksBatches; if (this.nodeConfig?.profiler) { - this.fetchBlocksFunction = profilerWrap( + this._fetchBlocksFunction = profilerWrap( fetchFunc, 'SubstrateUtil', 'fetchBlocksBatches', ); } else { - this.fetchBlocksFunction = fetchFunc; + this._fetchBlocksFunction = fetchFunc; } } diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts index ca181bdf42..a2dc485b62 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts @@ -131,8 +131,8 @@ export function buildDictionaryV1QueryEntries< handler.kind, getDsProcessor, ); - } else { - if (handler.filter) filterList = [handler.filter]; + } else if (handler.filter) { + filterList = [handler.filter]; } // Filter out any undefined filterList = filterList.filter(Boolean); diff --git a/packages/node/src/utils/project.ts b/packages/node/src/utils/project.ts index b92d6a24b9..9c03de7726 100644 --- a/packages/node/src/utils/project.ts +++ b/packages/node/src/utils/project.ts @@ -52,7 +52,6 @@ export async function getChainTypes( // if it failed, we will give it another attempt, and assume the script written in js // we will download it to a temp folder, and load them within sandbox const res = await reader.getFile(file); - assert(res, `File ${file} not found`); let raw: unknown; try { From 1285d30c23d08539d5da94ba118984620afa1bfe Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 20 Jun 2024 13:44:49 +0800 Subject: [PATCH 14/25] some change --- packages/node/src/utils/project.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/node/src/utils/project.ts b/packages/node/src/utils/project.ts index 9c03de7726..783cfee9b8 100644 --- a/packages/node/src/utils/project.ts +++ b/packages/node/src/utils/project.ts @@ -44,7 +44,7 @@ export async function getChainTypes( // If the project is load from local, we will direct load them let raw: unknown; if (reader instanceof LocalReader) { - return loadChainTypes(file, root) as ChainTypes; + raw = loadChainTypes(file, root); } else { // If it is stored in ipfs or other resources, we will use the corresponding reader to read the file // Because ipfs not provide extension of the file, it is difficult to determine its format @@ -52,7 +52,6 @@ export async function getChainTypes( // if it failed, we will give it another attempt, and assume the script written in js // we will download it to a temp folder, and load them within sandbox const res = await reader.getFile(file); - let raw: unknown; try { raw = yaml.load(res); From eb2c62098671a02fa7d9be9140051ae08cd7bb06 Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 20 Jun 2024 23:47:07 +0800 Subject: [PATCH 15/25] isCustomDs convert templateIsCustomDs --- packages/node-core/src/utils/project.ts | 20 ++++++++++++++----- .../node/src/configure/SubqueryProject.ts | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/node-core/src/utils/project.ts b/packages/node-core/src/utils/project.ts index 9c594ddb77..b86f3b4f41 100644 --- a/packages/node-core/src/utils/project.ts +++ b/packages/node-core/src/utils/project.ts @@ -5,7 +5,14 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import {DEFAULT_PORT, findAvailablePort, GithubReader, IPFSReader, LocalReader} from '@subql/common'; -import {BaseAssetsDataSource, BaseCustomDataSource, BaseDataSource, Reader, TemplateBase} from '@subql/types-core'; +import { + BaseAssetsDataSource, + BaseCustomDataSource, + BaseDataSource, + BaseTemplateDataSource, + Reader, + TemplateBase, +} from '@subql/types-core'; import {getAllEntitiesRelations} from '@subql/utils'; import {QueryTypes, Sequelize} from '@subql/x-sequelize'; import {stringToArray, getSchedule} from 'cron-converter'; @@ -278,20 +285,23 @@ export async function insertBlockFiltersCronSchedules( +export async function loadProjectTemplates( templates: T[] | undefined, root: string, reader: Reader, - isCustomDs: IsCustomDs + isCustomDs: IsCustomDs> ): Promise { if (!templates || !templates.length) { return []; } - const dsTemplates = await updateDataSourcesV1_0_0(templates, reader, root, isCustomDs); + + const templateIsCustomDs = (template: T): template is T & BaseCustomDataSource => + isCustomDs(template) && 'name' in template; + const dsTemplates = await updateDataSourcesV1_0_0(templates, reader, root, templateIsCustomDs); return dsTemplates.map((ds, index) => ({ ...ds, name: templates[index].name, - })) as T[]; + })); } export function getStartHeight(dataSources: BaseDataSource[]): number { diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index fc42ccee9c..abb373b34a 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -167,7 +167,7 @@ async function loadProjectFromManifestBase( projectManifest.templates, root, reader, - isCustomDs as any, // TODO: There is no good way to temporarily + isCustomDs, ); const runner = projectManifest.runner; assert( From 692ef71392ac84f45482def706171510f25b4534 Mon Sep 17 00:00:00 2001 From: Tate Date: Fri, 21 Jun 2024 15:25:59 +0800 Subject: [PATCH 16/25] convert GetDsProcessor --- .../dictionary/substrateDictionary.service.ts | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts index 6559e72b63..f8dd3b2864 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts @@ -5,8 +5,10 @@ import assert from 'assert'; import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NETWORK_FAMILY } from '@subql/common'; +import { isCustomDs } from '@subql/common-substrate'; import { NodeConfig, DictionaryService, getLogger } from '@subql/node-core'; import { SubstrateBlock, SubstrateDatasource } from '@subql/types'; +import { DsProcessor } from '@subql/types-core'; import { SubqueryProject } from '../../configure/SubqueryProject'; import { DsProcessorService } from '../ds-processor.service'; import { SpecVersion } from './types'; @@ -57,9 +59,9 @@ export class SubstrateDictionaryService extends DictionaryService< const dictionaryV1 = await SubstrateDictionaryV1.create( this.project, this.nodeConfig, - this.dsProcessorService.getDsProcessor.bind( + convertGetDsProcessor(this.dsProcessorService.getDsProcessor).bind( this.dsProcessorService, - ) as any, // TODO: There is no good way to temporarily + ), endpoint, ); dictionariesV1.push(dictionaryV1); @@ -102,3 +104,26 @@ export class SubstrateDictionaryService extends DictionaryService< return dict.getSpecVersions(); } } + +function convertGetDsProcessor( + f: DsProcessorService['getDsProcessor'], +): (ds: SubstrateDatasource) => DsProcessor { + function isSubstrateProcessor( + p: ReturnType, + ): p is DsProcessor { + // Since the current package is the substrate node startup package, it must be true here. + return true; + } + + return (ds: SubstrateDatasource) => { + if (!isCustomDs(ds)) { + throw new Error('data source is not a custom data source'); + } + + const processor = f(ds); + if (!isSubstrateProcessor(processor)) { + throw new Error('data source is not a substrate data source'); + } + return processor; + }; +} From ee3b2548d8a7257444b5add0bed8803e7d3a1d55 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 26 Jun 2024 11:50:07 +0800 Subject: [PATCH 17/25] some change --- packages/node-core/src/indexer/connectionPool.service.ts | 9 ++++++--- packages/node-core/src/indexer/sandbox.ts | 8 ++++++-- packages/node-core/src/utils/fetchHelpers.ts | 4 ++-- packages/node/src/indexer/apiPromise.connection.ts | 5 ++++- .../src/indexer/dictionary/v1/substrateDictionaryV1.ts | 8 +++++--- packages/node/src/indexer/runtime/runtimeService.ts | 4 ++-- packages/node/src/indexer/worker/worker.ts | 4 ++-- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index ab67d63f8a..5f5045e3a6 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -40,7 +40,10 @@ export class ConnectionPoolService) { + constructor( + private nodeConfig: NodeConfig, + private poolStateManager: ConnectionPoolStateManager + ) { this.cacheSizeThreshold = this.nodeConfig.batchSize; } @@ -112,8 +115,8 @@ export class ConnectionPoolService( ): Promise { try { return await request(); - } catch (e) { + } catch (e: any) { if (!shouldRetry(e)) throw e; if (retries > 1) { await delay(RETRY_DELAY); return retryOnFail(request, shouldRetry, --retries); } else { - logger.error(e as Error, `Retries failed after ${RETRY_COUNT}`); + logger.error(e, `Retries failed after ${RETRY_COUNT}`); throw e; } } diff --git a/packages/node/src/indexer/apiPromise.connection.ts b/packages/node/src/indexer/apiPromise.connection.ts index 486dcafe7e..c6329c3793 100644 --- a/packages/node/src/indexer/apiPromise.connection.ts +++ b/packages/node/src/indexer/apiPromise.connection.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { ApiPromise, WsProvider } from '@polkadot/api'; import { ProviderInterface } from '@polkadot/rpc-provider/types'; import { RegisteredTypes } from '@polkadot/types/types'; @@ -58,7 +59,7 @@ export class ApiPromiseConnection fetchBlocksBatches: GetFetchFunc, args: { chainTypes: RegisteredTypes }, ): Promise { - let provider: ProviderInterface | undefined; + let provider: ProviderInterface; let throwOnConnect = false; const headers = { @@ -72,6 +73,8 @@ export class ApiPromiseConnection } else if (endpoint.startsWith('http')) { provider = createCachedProvider(new HttpProvider(endpoint, headers)); throwOnConnect = true; + } else { + throw new Error(`Invalid endpoint: ${endpoint}`); } const apiOption = { diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts index a2dc485b62..b9c4d3aff7 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts @@ -170,7 +170,9 @@ export function buildDictionaryV1QueryEntries< } break; } - default: + default: { + throw new Error(`Unsupported handler kind: ${baseHandlerKind}`); + } } } } @@ -267,8 +269,8 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { const _metadata = resp.data._metadata; const specVersions = resp.data.specVersions; return { _metadata, specVersions }; - } catch (err) { - logger.warn(err as Error, `failed to fetch specVersion result`); + } catch (err: any) { + logger.warn(err, `failed to fetch specVersion result`); return undefined; } } diff --git a/packages/node/src/indexer/runtime/runtimeService.ts b/packages/node/src/indexer/runtime/runtimeService.ts index ca1236f9ae..231153d811 100644 --- a/packages/node/src/indexer/runtime/runtimeService.ts +++ b/packages/node/src/indexer/runtime/runtimeService.ts @@ -31,8 +31,8 @@ export class RuntimeService extends BaseRuntimeService { } else if (this.specVersionMap === undefined) { this.specVersionMap = []; } - } catch (e) { - logger.error(e as Error, 'Failed to get spec versions'); + } catch (e: any) { + logger.error(e, 'Failed to get spec versions'); } } diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts index 7c2d9cf755..07cb8534e2 100644 --- a/packages/node/src/indexer/worker/worker.ts +++ b/packages/node/src/indexer/worker/worker.ts @@ -47,9 +47,9 @@ async function initWorker(startHeight?: number): Promise { const workerService = app.get(WorkerService); initWorkerServices(app, workerService); - } catch (e) { + } catch (e: any) { console.log('Failed to start worker', e); - logger.error(e as Error, 'Failed to start worker'); + logger.error(e, 'Failed to start worker'); throw e; } } From 452187a8a675d82d196497d30ae209e534f9ff29 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 26 Jun 2024 15:40:51 +0800 Subject: [PATCH 18/25] some change --- packages/node/src/indexer/api.service.test.ts | 14 +++++++------- packages/node/src/utils/substrate.ts | 9 +++++---- packages/types/src/interfaces.ts | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/node/src/indexer/api.service.test.ts b/packages/node/src/indexer/api.service.test.ts index bc7a27d329..7326979267 100644 --- a/packages/node/src/indexer/api.service.test.ts +++ b/packages/node/src/indexer/api.service.test.ts @@ -101,7 +101,7 @@ describe('ApiService', () => { const blockhash = await api.rpc.chain.getBlockHash(2); const validators = await api.query.session.validators.at(blockhash); const block = await api.rpc.chain.getBlock(blockhash); - const mockBlock = wrapBlock(block, []) as unknown as SubstrateBlock; + const mockBlock = wrapBlock(block, [], 1) as unknown as SubstrateBlock; const runtimeVersion = { specVersion: 1 } as unknown as RuntimeVersion; const patchedApi = await apiService.getPatchedApi( mockBlock.block.header, @@ -120,7 +120,7 @@ describe('ApiService', () => { const api = apiService.api; const blockhash = await api.rpc.chain.getBlockHash(6721189); const block = await api.rpc.chain.getBlock(blockhash); - const mockBlock = wrapBlock(block, []) as unknown as SubstrateBlock; + const mockBlock = wrapBlock(block, [], 13) as unknown as SubstrateBlock; const runtimeVersion = { specVersion: 13 } as unknown as RuntimeVersion; const patchedApi = await apiService.getPatchedApi( mockBlock.block.header, @@ -147,7 +147,7 @@ describe('ApiService', () => { const api = apiService.api; const blockhash = await api.rpc.chain.getBlockHash(6721195); const block = await api.rpc.chain.getBlock(blockhash); - const mockBlock = wrapBlock(block, []) as unknown as SubstrateBlock; + const mockBlock = wrapBlock(block, [], 9090) as unknown as SubstrateBlock; const runtimeVersion = { specVersion: 9090 } as unknown as RuntimeVersion; // step 1, get early block, original polkadot api query result const earlyBlockhash = await api.rpc.chain.getBlockHash(5661443); @@ -176,7 +176,7 @@ describe('ApiService', () => { const api = apiService.api; const blockhash = await api.rpc.chain.getBlockHash(5661443); const block = await api.rpc.chain.getBlock(blockhash); - const mockBlock = wrapBlock(block, []) as unknown as SubstrateBlock; + const mockBlock = wrapBlock(block, [], 9050) as unknown as SubstrateBlock; const runtimeVersion = { specVersion: 9050 } as unknown as RuntimeVersion; // step 1, get future block, original polkadot api query result const futureBlockhash = await api.rpc.chain.getBlockHash(6721195); @@ -208,7 +208,7 @@ describe('ApiService', () => { } const block = await api.rpc.chain.getBlock(blockhash); - const mockBlock = wrapBlock(block, []) as unknown as SubstrateBlock; + const mockBlock = wrapBlock(block, [], 28) as unknown as SubstrateBlock; const runtimeVersion = { specVersion: 28 } as unknown as RuntimeVersion; const patchedApi = await apiService.getPatchedApi( mockBlock.block.header, @@ -382,7 +382,7 @@ describe('ApiService', () => { const api = apiService.api; const blockhash = await api.rpc.chain.getBlockHash(1); const block = await api.rpc.chain.getBlock(blockhash); - const mockBlock = wrapBlock(block, []) as unknown as SubstrateBlock; + const mockBlock = wrapBlock(block, [], 1) as unknown as SubstrateBlock; const runtimeVersion = { specVersion: 1 } as unknown as RuntimeVersion; const patchedApi = await apiService.getPatchedApi( @@ -403,7 +403,7 @@ describe('ApiService', () => { const blockNumber = 1545235; const blockhash = await api.rpc.chain.getBlockHash(blockNumber); const block = await api.rpc.chain.getBlock(blockhash); - const mockBlock = wrapBlock(block, []) as unknown as SubstrateBlock; + const mockBlock = wrapBlock(block, [], 1103) as unknown as SubstrateBlock; const runtimeVersion = { specVersion: 1103 } as unknown as RuntimeVersion; const patchedApi = await apiService.getPatchedApi( diff --git a/packages/node/src/utils/substrate.ts b/packages/node/src/utils/substrate.ts index 8c53f065ae..408097335e 100644 --- a/packages/node/src/utils/substrate.ts +++ b/packages/node/src/utils/substrate.ts @@ -1,6 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { ApiPromise } from '@polkadot/api'; import { Vec } from '@polkadot/types'; import '@polkadot/api-augment/substrate'; @@ -32,6 +33,7 @@ import { merge, range } from 'lodash'; import { SubqlProjectBlockFilter } from '../configure/SubqueryProject'; import { ApiPromiseConnection } from '../indexer/apiPromise.connection'; import { BlockContent, LightBlockContent } from '../indexer/types'; + const logger = getLogger('fetch'); const INTERVAL_THRESHOLD = BN_THOUSAND.div(BN_TWO); const DEFAULT_TIME = new BN(6_000); @@ -48,7 +50,7 @@ export function substrateHeaderToHeader(header: SubstrateHeader): Header { export function wrapBlock( signedBlock: SignedBlock, events: EventRecord[], - specVersion?: number, + specVersion: number, ): SubstrateBlock { return merge(signedBlock, { timestamp: getTimestamp(signedBlock), @@ -380,9 +382,8 @@ export async function fetchBlocksBatches( return blocks.map((block, idx) => { const events = blockEvents[idx]; const parentSpecVersion = - overallSpecVer !== undefined - ? overallSpecVer - : runtimeVersions?.[idx].specVersion.toNumber(); + overallSpecVer ?? runtimeVersions?.[idx].specVersion.toNumber(); + assert(parentSpecVersion !== undefined, 'parentSpecVersion is undefined'); const wrappedBlock = wrapBlock(block, events.toArray(), parentSpecVersion); const wrappedExtrinsics = wrapExtrinsics(wrappedBlock, events); diff --git a/packages/types/src/interfaces.ts b/packages/types/src/interfaces.ts index 090e2bff5b..df9f07d652 100644 --- a/packages/types/src/interfaces.ts +++ b/packages/types/src/interfaces.ts @@ -8,7 +8,7 @@ import {IEvent} from '@polkadot/types/types'; export interface SubstrateBlock extends SignedBlock { // parent block's spec version, can be used to decide the correct metadata that should be used for this block. - specVersion?: number; + specVersion: number; timestamp: Date; events: EventRecord[]; } From a47e15b2f0fe34f629186249c71408077f3811af Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 26 Jun 2024 15:56:23 +0800 Subject: [PATCH 19/25] some change --- .../src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts index 5d2e40b096..888a69243a 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts @@ -140,8 +140,8 @@ describe('Building dictionary query entries', () => { { entity: 'events', conditions: [ - { field: 'module', value: 'module' }, { field: 'event', value: 'event' }, + { field: 'module', value: 'module' }, ], }, ]); From 6e11c92c92c1194501889cecd7d862e15dd534dd Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 26 Jun 2024 16:28:06 +0800 Subject: [PATCH 20/25] some change --- .../src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts index 888a69243a..5bed539736 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.spec.ts @@ -229,7 +229,7 @@ describe('Building dictionary query entries', () => { it('create dictionary call filter condition and remove undefined fields', () => { const result1 = buildDictionaryV1QueryEntries( [makeDs([callHandlerWithUndefined])], - () => undefined, + () => undefined as any, ); expect(result1).toEqual([ { @@ -247,7 +247,7 @@ describe('Building dictionary query entries', () => { it('create dictionary event filter condition and remove undefined fields', () => { const result1 = buildDictionaryV1QueryEntries( [makeDs([eventHandlerWithUndefined])], - () => undefined, + () => undefined as any, ); expect(result1).toEqual([ { From 248a66636252b133447cf982a5a57479cc7c2d84 Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 27 Jun 2024 10:37:59 +0800 Subject: [PATCH 21/25] some change --- packages/node-core/CHANGELOG.md | 3 +++ packages/node/CHANGELOG.md | 3 +++ packages/node/src/indexer/store.service.test.ts | 9 ++++----- packages/node/src/utils/project.ts | 1 - 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index 156b19e34f..c57ccb8aa6 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Enable ts strict model + ## [10.9.0] - 2024-06-21 ### Fixed - Store service index check failing with snake_case fields (#2461) diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 84d01fb823..0eb71cb455 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Enable ts strict model + ## [4.6.6] - 2024-06-21 ### Fixed - Update with common-substrate, fix `EventFilter` incorrectly extend `BlockFilter`, lead dictionary error (#2463) diff --git a/packages/node/src/indexer/store.service.test.ts b/packages/node/src/indexer/store.service.test.ts index 916022f0fa..2355e5d274 100644 --- a/packages/node/src/indexer/store.service.test.ts +++ b/packages/node/src/indexer/store.service.test.ts @@ -64,7 +64,7 @@ describe('Store service integration test', () => { tempDir = (projectService as any).project.root; - const [result] = await sequelize.query( + const result = await sequelize.query<{ table_name: string }>( ` SELECT table_name @@ -74,12 +74,11 @@ describe('Store service integration test', () => { table_schema = :schema; `, { + type: QueryTypes.SELECT, replacements: { schema: schemaName }, }, ); - const expectedTables = (result as { table_name: string }[]).map( - (t) => t.table_name, - ); + const expectedTables = result.map((t) => t.table_name); const columnResult = await sequelize.query( ` @@ -278,7 +277,7 @@ WHERE tempDir = (projectService as any).project.root; - const result: { enum_type: string }[] = await sequelize.query( + const result: any = await sequelize.query( ` SELECT n.nspname AS schema_name, t.typname AS enum_type, diff --git a/packages/node/src/utils/project.ts b/packages/node/src/utils/project.ts index 783cfee9b8..89b0ab6e82 100644 --- a/packages/node/src/utils/project.ts +++ b/packages/node/src/utils/project.ts @@ -52,7 +52,6 @@ export async function getChainTypes( // if it failed, we will give it another attempt, and assume the script written in js // we will download it to a temp folder, and load them within sandbox const res = await reader.getFile(file); - let raw: unknown; try { raw = yaml.load(res); } catch (e) { From 17f537e036410441a78fa80c28ae78ac97ca87be Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 27 Jun 2024 17:40:02 +1200 Subject: [PATCH 22/25] Fix issue with SubstrateDataSourceProcessor type and follow on effects (#2469) --- .../dictionary/substrateDictionary.service.ts | 25 +----------- .../dictionary/v1/substrateDictionaryV1.ts | 39 +++++++------------ packages/types/src/project.ts | 38 ++++++++---------- 3 files changed, 32 insertions(+), 70 deletions(-) diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts index f8dd3b2864..fddfead296 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts @@ -59,7 +59,7 @@ export class SubstrateDictionaryService extends DictionaryService< const dictionaryV1 = await SubstrateDictionaryV1.create( this.project, this.nodeConfig, - convertGetDsProcessor(this.dsProcessorService.getDsProcessor).bind( + this.dsProcessorService.getDsProcessor.bind( this.dsProcessorService, ), endpoint, @@ -104,26 +104,3 @@ export class SubstrateDictionaryService extends DictionaryService< return dict.getSpecVersions(); } } - -function convertGetDsProcessor( - f: DsProcessorService['getDsProcessor'], -): (ds: SubstrateDatasource) => DsProcessor { - function isSubstrateProcessor( - p: ReturnType, - ): p is DsProcessor { - // Since the current package is the substrate node startup package, it must be true here. - return true; - } - - return (ds: SubstrateDatasource) => { - if (!isCustomDs(ds)) { - throw new Error('data source is not a custom data source'); - } - - const processor = f(ds); - if (!isSubstrateProcessor(processor)) { - throw new Error('data source is not a substrate data source'); - } - return processor; - }; -} diff --git a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts index 205595e841..6f225bb421 100644 --- a/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/substrateDictionaryV1.ts @@ -19,14 +19,16 @@ import { SubstrateBlockFilter, SubstrateDatasource } from '@subql/types'; import { DictionaryQueryCondition, DictionaryQueryEntry as DictionaryV1QueryEntry, - DsProcessor, } from '@subql/types-core'; import { buildQuery, GqlNode, GqlQuery } from '@subql/utils'; import { sortBy, uniqBy } from 'lodash'; import { SubqueryProject } from '../../../configure/SubqueryProject'; import { isBaseHandler, isCustomHandler } from '../../../utils/project'; +import { DsProcessorService } from '../../ds-processor.service'; import { SpecVersion, SpecVersionDictionary } from '../types'; +type GetDsProcessor = DsProcessorService['getDsProcessor']; + function eventFilterToQueryEntry( filter: SubstrateEventFilter, ): DictionaryV1QueryEntry { @@ -60,12 +62,10 @@ function callFilterToQueryEntry( }; } -function getBaseHandlerKind< - P extends DsProcessor = DsProcessor, ->( +function getBaseHandlerKind( ds: SubstrateDataSource, handler: SubstrateHandler, - getDsProcessor: (ds: SubstrateDatasource) => P, + getDsProcessor: GetDsProcessor, ): SubstrateHandlerKind | undefined { if (isRuntimeDs(ds) && isBaseHandler(handler)) { return handler.kind; @@ -81,13 +81,10 @@ function getBaseHandlerKind< } } -function getBaseHandlerFilters< - T extends SubstrateRuntimeHandlerFilter, - P extends DsProcessor = DsProcessor, ->( +function getBaseHandlerFilters( ds: SubstrateDataSource, handlerKind: string, - getDsProcessor: (ds: SubstrateDatasource) => P, + getDsProcessor: GetDsProcessor, ): T[] { if (isCustomDs(ds)) { const plugin = getDsProcessor(ds); @@ -101,11 +98,9 @@ function getBaseHandlerFilters< } // eslint-disable-next-line complexity -export function buildDictionaryV1QueryEntries< - P extends DsProcessor = DsProcessor, ->( +export function buildDictionaryV1QueryEntries( dataSources: SubstrateDatasource[], - getDsProcessor: (ds: SubstrateDatasource) => P, + getDsProcessor: GetDsProcessor, ): DictionaryV1QueryEntry[] { const queryEntries: DictionaryV1QueryEntry[] = []; @@ -117,11 +112,9 @@ export function buildDictionaryV1QueryEntries< if (isCustomDs(ds)) { assert(plugin, 'plugin should be defined'); const processor = plugin.handlerProcessors[handler.kind]; - if (processor.dictionaryQuery) { - const queryEntry = processor.dictionaryQuery( - (handler as SubstrateCustomHandler).filter, - ds, - ); + const filter = (handler as SubstrateCustomHandler).filter; + if (processor.dictionaryQuery && filter) { + const queryEntry = processor.dictionaryQuery(filter, ds); if (queryEntry) { queryEntries.push(queryEntry); continue; @@ -193,9 +186,7 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { constructor( project: SubqueryProject, nodeConfig: NodeConfig, - protected getDsProcessor: ( - ds: SubstrateDatasource, - ) => DsProcessor, + protected getDsProcessor: GetDsProcessor, dictionaryUrl: string, chainId?: string, ) { @@ -205,9 +196,7 @@ export class SubstrateDictionaryV1 extends DictionaryV1 { static async create( project: SubqueryProject, nodeConfig: NodeConfig, - getDsProcessor: ( - ds: SubstrateDatasource, - ) => DsProcessor, + getDsProcessor: GetDsProcessor, dictionaryUrl: string, chainId?: string, ): Promise { diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 74ba0cc677..1a215d7bbe 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -238,7 +238,7 @@ type ISubstrateDatasource = BaseDataSource). */ export interface SubstrateRuntimeDatasource< - M extends SubstrateMapping = SubstrateMapping + M extends SubstrateMapping = SubstrateMapping, > extends ISubstrateDatasource { /** * The kind of the datasource, which is `substrate/Runtime`. @@ -263,7 +263,7 @@ export type SubstrateDatasource = SubstrateRuntimeDatasource | SubstrateCustomDa export interface SubstrateCustomDatasource< K extends string = string, M extends SubstrateMapping = SubstrateMapping, - O = any + O = any, > extends BaseCustomDataSource { /** * The kind of the custom datasource. This should follow the pattern `substrate/*`. @@ -294,7 +294,7 @@ export type HandlerInputTransformer_0_0_0< IM extends RuntimeHandlerInputMap, T extends SubstrateHandlerKind, E, - DS extends SubstrateCustomDatasource = SubstrateCustomDatasource + DS extends SubstrateCustomDatasource = SubstrateCustomDatasource, > = BaseHandlerInputTransformer_0_0_0; /** @@ -306,19 +306,18 @@ export type HandlerInputTransformer_1_0_0< T extends SubstrateHandlerKind, F extends Record, E, - DS extends SubstrateCustomDatasource = SubstrateCustomDatasource + DS extends SubstrateCustomDatasource = SubstrateCustomDatasource, > = BaseHandlerInputTransformer_1_0_0; type SecondLayerHandlerProcessorArray< K extends string, F extends Record, T, - IT extends AnyTuple = AnyTuple, - DS extends SubstrateCustomDatasource = SubstrateCustomDatasource + DS extends SubstrateCustomDatasource = SubstrateCustomDatasource, > = - | SecondLayerHandlerProcessor - | SecondLayerHandlerProcessor - | SecondLayerHandlerProcessor; + | SecondLayerHandlerProcessor + | SecondLayerHandlerProcessor + | SecondLayerHandlerProcessor; /** * @deprecated use types core version. datasource processors need updating before this can be removed @@ -327,10 +326,10 @@ export type SubstrateDatasourceProcessor< K extends string, F extends Record, DS extends SubstrateCustomDatasource = SubstrateCustomDatasource, - P extends Record> = Record< + P extends Record> = Record< string, - SecondLayerHandlerProcessorArray - > + SecondLayerHandlerProcessorArray + >, > = DsProcessor; /** @@ -340,9 +339,8 @@ export type SecondLayerHandlerProcessor_0_0_0< K extends SubstrateHandlerKind, F extends Record, E, - IT extends AnyTuple = AnyTuple, - DS extends SubstrateCustomDatasource = SubstrateCustomDatasource -> = BaseSecondLayerHandlerProcessor_0_0_0, K, F, E, DS, ApiPromise>; + DS extends SubstrateCustomDatasource = SubstrateCustomDatasource, +> = BaseSecondLayerHandlerProcessor_0_0_0; /** * @deprecated use types core version. datasource processors need updating before this can be removed @@ -351,9 +349,8 @@ export type SecondLayerHandlerProcessor_1_0_0< K extends SubstrateHandlerKind, F extends Record, E, - IT extends AnyTuple = AnyTuple, - DS extends SubstrateCustomDatasource = SubstrateCustomDatasource -> = BaseSecondLayerHandlerProcessor_1_0_0, K, F, E, DS, ApiPromise>; + DS extends SubstrateCustomDatasource = SubstrateCustomDatasource, +> = BaseSecondLayerHandlerProcessor_1_0_0; /** * @deprecated use types core version. datasource processors need updating before this can be removed @@ -362,9 +359,8 @@ export type SecondLayerHandlerProcessor< K extends SubstrateHandlerKind, F extends Record, E, - IT extends AnyTuple = AnyTuple, - DS extends SubstrateCustomDatasource = SubstrateCustomDatasource -> = BaseSecondLayerHandlerProcessor, K, F, E, DS, ApiPromise>; + DS extends SubstrateCustomDatasource = SubstrateCustomDatasource, +> = BaseSecondLayerHandlerProcessor; /** * Represents a Substrate subquery network configuration, which is based on the CommonSubqueryNetworkConfig template. From 2551ea3f64059deb0341f084f89837122ecc9abc Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 2 Jul 2024 10:12:57 +0800 Subject: [PATCH 23/25] some change --- packages/node/src/utils/project.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/node/src/utils/project.ts b/packages/node/src/utils/project.ts index 89b0ab6e82..12e779a336 100644 --- a/packages/node/src/utils/project.ts +++ b/packages/node/src/utils/project.ts @@ -1,7 +1,6 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import assert from 'assert'; import fs from 'fs'; import path from 'path'; import { LocalReader, loadFromJsonOrYaml } from '@subql/common'; From 51cdb23e057ebad970b568a7e91036f81797ceb5 Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 2 Jul 2024 11:06:53 +0800 Subject: [PATCH 24/25] some change --- packages/node/src/indexer/runtime/runtime.service.spec.ts | 2 +- packages/types/CHANGELOG.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/node/src/indexer/runtime/runtime.service.spec.ts b/packages/node/src/indexer/runtime/runtime.service.spec.ts index 453ba3cfba..0c4434dc10 100644 --- a/packages/node/src/indexer/runtime/runtime.service.spec.ts +++ b/packages/node/src/indexer/runtime/runtime.service.spec.ts @@ -49,7 +49,7 @@ const getApiService = (): ApiService => return Promise.resolve({ specVersion: { - toNumber: () => parseInt(specVersion.id), + toNumber: () => parseInt(specVersion.id, 10), }, } as RuntimeVersion); }, diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index 9034a159ab..9ec6bd4bc8 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## Changed +- Enable strict mode for the other packages. + ## [3.7.0] - 2024-06-21 ### Changed - Bump version with `@subql/types-core` From 7fe76c72a2abc254a7b57e80df75d48cf3554f25 Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 2 Jul 2024 12:06:06 +0800 Subject: [PATCH 25/25] some change --- .../node/src/indexer/dictionary/substrateDictionary.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts index fddfead296..8c09f6f037 100644 --- a/packages/node/src/indexer/dictionary/substrateDictionary.service.ts +++ b/packages/node/src/indexer/dictionary/substrateDictionary.service.ts @@ -88,7 +88,7 @@ export class SubstrateDictionaryService extends DictionaryService< private getV1Dictionary(): SubstrateDictionaryV1 | undefined { // TODO this needs to be removed once Substrate supports V2 dictionaries - if (!this._currentDictionaryIndex) return undefined; + if (this._currentDictionaryIndex === undefined) return undefined; const dict = this._dictionaries[this._currentDictionaryIndex]; if (!dict) return undefined; assert(