diff --git a/.yarn/versions/0a6cfead.yml b/.yarn/versions/0a6cfead.yml new file mode 100644 index 0000000000..6cd4b4fa5a --- /dev/null +++ b/.yarn/versions/0a6cfead.yml @@ -0,0 +1,3 @@ +undecided: + - "@subql/cli" + - "@subql/common-substrate" diff --git a/.yarn/versions/61b6fc48.yml b/.yarn/versions/61b6fc48.yml new file mode 100644 index 0000000000..b7b31dc49a --- /dev/null +++ b/.yarn/versions/61b6fc48.yml @@ -0,0 +1,3 @@ +undecided: + - "@subql/node-core" + - "@subql/types-core" diff --git a/.yarn/versions/6fdfdb04.yml b/.yarn/versions/6fdfdb04.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.yarn/versions/936dc023.yml b/.yarn/versions/936dc023.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.yarn/versions/b4e981f5.yml b/.yarn/versions/b4e981f5.yml new file mode 100644 index 0000000000..6efcbbe778 --- /dev/null +++ b/.yarn/versions/b4e981f5.yml @@ -0,0 +1,3 @@ +undecided: + - "@subql/common" + - "@subql/testing" diff --git a/.yarn/versions/cb885b36.yml b/.yarn/versions/cb885b36.yml new file mode 100644 index 0000000000..f3d404be5f --- /dev/null +++ b/.yarn/versions/cb885b36.yml @@ -0,0 +1,2 @@ +undecided: + - "@subql/node" diff --git a/.yarn/versions/d11aeeaf.yml b/.yarn/versions/d11aeeaf.yml new file mode 100644 index 0000000000..75f21d7001 --- /dev/null +++ b/.yarn/versions/d11aeeaf.yml @@ -0,0 +1,2 @@ +undecided: + - "@subql/node-core" diff --git a/.yarn/versions/d2b1da3c.yml b/.yarn/versions/d2b1da3c.yml new file mode 100644 index 0000000000..ed92b63a52 --- /dev/null +++ b/.yarn/versions/d2b1da3c.yml @@ -0,0 +1,4 @@ +undecided: + - "@subql/cli" + - "@subql/node" + - "@subql/node-core" diff --git a/.yarn/versions/dc0e431b.yml b/.yarn/versions/dc0e431b.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 75e4ba245c..79bb32c9d2 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [5.3.0] - 2024-10-21 +### Changed +- Improve codegen error messages (#2567) +- Update codegen to match changes to store interface making options.limit required on getByField(s) methods (#2567) + ## [5.2.8] - 2024-09-25 ### Changed - Bump common, Added manifest support for query-subgraph. @@ -682,7 +687,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - support subcommand codegen - support subcommand init -[Unreleased]: https://github.com/subquery/subql/compare/cli/5.2.8...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/cli/5.3.0...HEAD +[5.3.0]: https://github.com/subquery/subql/compare/cli/5.2.8...cli/5.3.0 [5.2.8]: https://github.com/subquery/subql/compare/cli/5.2.7...cli/5.2.8 [5.2.7]: https://github.com/subquery/subql/compare/cli/5.2.6...cli/5.2.7 [5.2.6]: https://github.com/subquery/subql/compare/cli/5.2.4...cli/5.2.6 diff --git a/packages/cli/README.md b/packages/cli/README.md index 0b2291de41..eda7bf4727 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -23,7 +23,7 @@ $ npm install -g @subql/cli $ subql COMMAND running command... $ subql (--version) -@subql/cli/5.2.7-0 linux-x64 node-v18.20.4 +@subql/cli/5.2.9-0 linux-x64 node-v18.20.4 $ subql --help [COMMAND] USAGE $ subql COMMAND @@ -71,7 +71,7 @@ DESCRIPTION Build this SubQuery project code ``` -_See code: [lib/commands/build/index.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/build/index.js)_ +_See code: [lib/commands/build/index.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/build/index.js)_ ## `subql codegen` @@ -89,7 +89,7 @@ DESCRIPTION Generate schemas for graph node ``` -_See code: [lib/commands/codegen/index.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/codegen/index.js)_ +_See code: [lib/commands/codegen/index.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/codegen/index.js)_ ## `subql codegen:generate` @@ -112,7 +112,7 @@ DESCRIPTION Generate Project.yaml and mapping functions based on provided ABI ``` -_See code: [lib/commands/codegen/generate.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/codegen/generate.js)_ +_See code: [lib/commands/codegen/generate.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/codegen/generate.js)_ ## `subql deployment` @@ -161,7 +161,7 @@ DESCRIPTION Deploy to hosted service ``` -_See code: [lib/commands/deployment/index.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/deployment/index.js)_ +_See code: [lib/commands/deployment/index.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/deployment/index.js)_ ## `subql deployment:delete` @@ -180,7 +180,7 @@ DESCRIPTION Delete Deployment ``` -_See code: [lib/commands/deployment/delete.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/deployment/delete.js)_ +_See code: [lib/commands/deployment/delete.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/deployment/delete.js)_ ## `subql deployment:deploy` @@ -225,7 +225,7 @@ DESCRIPTION Deployment to hosted service ``` -_See code: [lib/commands/deployment/deploy.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/deployment/deploy.js)_ +_See code: [lib/commands/deployment/deploy.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/deployment/deploy.js)_ ## `subql deployment:promote` @@ -244,7 +244,7 @@ DESCRIPTION Promote Deployment ``` -_See code: [lib/commands/deployment/promote.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/deployment/promote.js)_ +_See code: [lib/commands/deployment/promote.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/deployment/promote.js)_ ## `subql init [PROJECTNAME]` @@ -268,7 +268,7 @@ DESCRIPTION Initialize a scaffold subquery project ``` -_See code: [lib/commands/init.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/init.js)_ +_See code: [lib/commands/init.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/init.js)_ ## `subql migrate` @@ -287,7 +287,7 @@ DESCRIPTION Schema subgraph project to subquery project ``` -_See code: [lib/commands/migrate.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/migrate.js)_ +_See code: [lib/commands/migrate.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/migrate.js)_ ## `subql multi-chain:add` @@ -306,7 +306,7 @@ DESCRIPTION Add new chain manifest to multi-chain configuration ``` -_See code: [lib/commands/multi-chain/add.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/multi-chain/add.js)_ +_See code: [lib/commands/multi-chain/add.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/multi-chain/add.js)_ ## `subql multi-chain:deploy` @@ -352,7 +352,7 @@ DESCRIPTION Multi-chain deployment to hosted service ``` -_See code: [lib/commands/multi-chain/deploy.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/multi-chain/deploy.js)_ +_See code: [lib/commands/multi-chain/deploy.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/multi-chain/deploy.js)_ ## `subql project` @@ -377,7 +377,7 @@ DESCRIPTION Create/Delete project ``` -_See code: [lib/commands/project/index.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/project/index.js)_ +_See code: [lib/commands/project/index.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/project/index.js)_ ## `subql project:create-project` @@ -401,7 +401,7 @@ DESCRIPTION Create Project on Hosted Service ``` -_See code: [lib/commands/project/create-project.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/project/create-project.js)_ +_See code: [lib/commands/project/create-project.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/project/create-project.js)_ ## `subql project:delete-project` @@ -419,7 +419,7 @@ DESCRIPTION Delete Project on Hosted Service ``` -_See code: [lib/commands/project/delete-project.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/project/delete-project.js)_ +_See code: [lib/commands/project/delete-project.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/project/delete-project.js)_ ## `subql publish` @@ -438,6 +438,6 @@ DESCRIPTION Upload this SubQuery project to IPFS ``` -_See code: [lib/commands/publish.js](https://github.com/packages/cli/blob/v5.2.7-0/lib/commands/publish.js)_ +_See code: [lib/commands/publish.js](https://github.com/packages/cli/blob/v5.2.9-0/lib/commands/publish.js)_ diff --git a/packages/cli/package.json b/packages/cli/package.json index 06dfb18e55..fc83cd24b6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@subql/cli", "description": "cli for subquery", - "version": "5.2.8", + "version": "5.3.0", "author": "Ian He", "bin": { "subql": "./bin/run" diff --git a/packages/cli/src/controller/codegen-controller.ts b/packages/cli/src/controller/codegen-controller.ts index 0249f57561..016a37b41e 100644 --- a/packages/cli/src/controller/codegen-controller.ts +++ b/packages/cli/src/controller/codegen-controller.ts @@ -91,7 +91,7 @@ export async function generateJsonInterfaces(projectPath: string, schema: string await renderTemplate(INTERFACE_TEMPLATE_PATH, path.join(typesDir, `interfaces.ts`), interfaceTemplate); exportTypes.interfaces = true; } catch (e) { - throw new Error(`When render json interfaces having problems.`); + throw new Error(`Codegen failed for json interface.`, {cause: e}); } } } @@ -116,7 +116,7 @@ export async function generateEnums(projectPath: string, schema: string): Promis await renderTemplate(ENUM_TEMPLATE_PATH, path.join(typesDir, `enums.ts`), enumsTemplate); exportTypes.enums = true; } catch (e) { - throw new Error(`When render enums having problems.`); + throw new Error(`Codegen failed for enums.`, {cause: e}); } } } @@ -262,7 +262,7 @@ export async function codegen(projectPath: string, fileNames: string[] = [DEFAUL }, }); } catch (e) { - throw new Error(`When render index in types having problems.`); + throw new Error(`Codegen failed for indexes.`, {cause: e}); } console.log(`* Types index generated !`); } @@ -322,7 +322,7 @@ export async function generateModels(projectPath: string, schema: string): Promi ); } catch (e) { console.error(e); - throw new Error(`When render entity ${className} to schema having problems.`); + throw new Error(`Codegen failed for entity ${className}.`, {cause: e}); } console.log(`* Schema ${className} generated !`); } @@ -339,7 +339,7 @@ export async function generateModels(projectPath: string, schema: string): Promi }); exportTypes.models = true; } catch (e) { - throw new Error(`When render index in models having problems.`); + throw new Error(`Failed to codgen for model indexes.`, {cause: e}); } console.log(`* Models index generated !`); } diff --git a/packages/cli/src/template/model.ts.ejs b/packages/cli/src/template/model.ts.ejs index d3cf6db3eb..5f40440eb0 100644 --- a/packages/cli/src/template/model.ts.ejs +++ b/packages/cli/src/template/model.ts.ejs @@ -2,7 +2,7 @@ import {Entity, FunctionPropertyNames, FieldsExpression, GetOptions } from "@subql/types-core"; import assert from 'assert'; <%if (props.importJsonInterfaces.length !== 0) { %> -import {<% props.importJsonInterfaces.forEach(function(interface){ %> +import { <% props.importJsonInterfaces.forEach(function(interface){ %> <%= interface %>, <% }); %>} from '../interfaces'; <% } %> @@ -11,7 +11,7 @@ import {<% props.importEnums.forEach(function(e){ %> <%= e %>, <% }); %>} from '../enums';<% } %> -export type <%= props.className %>Props = Omit<<%=props.className %>, NonNullable>>| '_name'>; +export type <%= props.className %>Props = Omit<<%=props.className %>, NonNullable>> | '_name'>; export class <%= props.className %> implements Entity { @@ -30,18 +30,18 @@ export class <%= props.className %> implements Entity { return '<%=props.entityName %>'; } - async save(): Promise{ + async save(): Promise { let id = this.id; assert(id !== null, "Cannot save <%=props.className %> entity without an ID"); await store.set('<%=props.entityName %>', id.toString(), this); } - static async remove(id:string): Promise{ + static async remove(id: string): Promise { assert(id !== null, "Cannot remove <%=props.className %> entity without an ID"); await store.remove('<%=props.entityName %>', id.toString()); } - static async get(id:string): Promise<<%=props.className %> | undefined>{ + static async get(id: string): Promise<<%=props.className %> | undefined> { assert((id !== null && id !== undefined), "Cannot get <%=props.className %> entity without an ID"); const record = await store.get('<%=props.entityName %>', id.toString()); if (record) { @@ -51,17 +51,21 @@ export class <%= props.className %> implements Entity { } } <% props.indexedFields.forEach(function(field){ %> - static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>): Promise<<%=props.className %><%=field.unique ? '' : '[]' %> | undefined>{ - <% if (field.unique) {%> - const record = await store.getOneByField('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>); - if (record) { - return this.create(record as <%= props.className %>Props); - } else { - return; - } - <% } else { %>const records = await store.getByField('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>); - return records.map(record => this.create(record as <%= props.className %>Props));<% }%> + <% if (field.unique) {%> + + static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>): Promise<<%=props.className %> | undefined> { + const record = await store.getOneByField('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>); + if (record) { + return this.create(record as <%= props.className %>Props); + } else { + return; + } + } + <% } else { %>static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>, options: GetOptions<<%=props.className %>>): Promise<<%=props.className %>[]> { + const records = await store.getByField<<%=props.className %>>('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>, options); + return records.map(record => this.create(record as <%= props.className %>Props)); } + <% }%> <% }); %> /** @@ -69,8 +73,8 @@ export class <%= props.className %> implements Entity { * * ⚠️ This function will first search cache data followed by DB data. Please consider this when using order and offset options.⚠️ * */ - static async getByFields(filter: FieldsExpression<<%= props.className %>Props>[], options?: GetOptions<<%= props.className %>Props>): Promise<<%=props.className %>[]> { - const records = await store.getByFields('<%=props.entityName %>', filter, options); + static async getByFields(filter: FieldsExpression<<%= props.className %>Props>[], options: GetOptions<<%= props.className %>Props>): Promise<<%=props.className %>[]> { + const records = await store.getByFields<<%=props.className %>>('<%=props.entityName %>', filter, options); return records.map(record => this.create(record as <%= props.className %>Props)); } diff --git a/packages/common-substrate/CHANGELOG.md b/packages/common-substrate/CHANGELOG.md index c193889387..f2edaa1c5e 100644 --- a/packages/common-substrate/CHANGELOG.md +++ b/packages/common-substrate/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [4.3.2] - 2024-10-23 +### Changed +- Bump version with `@subql/common` + ## [4.3.1] - 2024-09-25 ### Changed - Bump common, Added manifest support for query-subgraph. @@ -189,7 +193,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - init commit -[Unreleased]: https://github.com/subquery/subql/compare/common-substrate/4.3.1...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/common-substrate/4.3.2...HEAD +[4.3.2]: https://github.com/subquery/subql/compare/common-substrate/4.3.1...common-substrate/4.3.2 [4.3.1]: https://github.com/subquery/subql/compare/common-substrate/4.3.0...common-substrate/4.3.1 [4.3.0]: https://github.com/subquery/subql/compare/common-substrate/4.2.0...common-substrate/4.3.0 [4.2.0]: https://github.com/subquery/subql/compare/common-substrate/4.1.1...common-substrate/4.2.0 diff --git a/packages/common-substrate/package.json b/packages/common-substrate/package.json index 7adccba14b..3147da7893 100644 --- a/packages/common-substrate/package.json +++ b/packages/common-substrate/package.json @@ -1,6 +1,6 @@ { "name": "@subql/common-substrate", - "version": "4.3.1", + "version": "4.3.2", "description": "", "scripts": { "build": "rm -rf dist && tsc -b", diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md index 4040756722..f432d017bc 100644 --- a/packages/common/CHANGELOG.md +++ b/packages/common/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [5.1.4] - 2024-10-23 +### Fixed +- Bump version `@subql/types-core` + ## [5.1.3] - 2024-09-25 ### Added - `Runner.query` support `@subql/query-subgraph` option @@ -415,7 +419,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - init commit -[Unreleased]: https://github.com/subquery/subql/compare/common/5.1.3...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/common/5.1.4...HEAD +[5.1.4]: https://github.com/subquery/subql/compare/common/5.1.3...common/5.1.4 [5.1.3]: https://github.com/subquery/subql/compare/common/5.1.2...common/5.1.3 [5.1.2]: https://github.com/subquery/subql/compare/common/5.1.1...common/5.1.2 [5.1.1]: https://github.com/subquery/subql/compare/common/5.1.0...common/5.1.1 diff --git a/packages/common/package.json b/packages/common/package.json index a82bfbb7bd..308f766670 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -1,6 +1,6 @@ { "name": "@subql/common", - "version": "5.1.3", + "version": "5.1.4", "description": "", "scripts": { "build": "rm -rf dist && tsc -b", diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index 5560f37664..d6964f56d8 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -6,6 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [14.1.7] - 2024-10-30 +### Changed +- Bump `@subql/common` dependency + +## [14.1.6] - 2024-10-21 +### Fixed +- Issues with setting a large block range for bypass blocks (#2566) +- Test runner not setting lastProcessedHeight leading to data not being flushed (#2569) +- Unable to rewind unfinalized blocks on startup (#2570) +- Store `getByFields` returning removed cache data (#2571) + +### Changed +- Throw error when store getByField(s) options.limit exceeds queryLimit option (#2567) + ## [14.1.5] - 2024-09-25 ### Changed - Bump common, Added manifest support for query-subgraph. @@ -826,7 +840,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Move blockchain agnostic code from `node` to `node-core` package. (#1222) -[Unreleased]: https://github.com/subquery/subql/compare/node-core/14.1.5...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/node-core/14.1.7...HEAD +[14.1.7]: https://github.com/subquery/subql/compare/node-core/14.1.6...node-core/14.1.7 +[14.1.6]: https://github.com/subquery/subql/compare/node-core/14.1.5...node-core/14.1.6 [14.1.5]: https://github.com/subquery/subql/compare/node-core/14.1.4...node-core/14.1.5 [14.1.4]: https://github.com/subquery/subql/compare/node-core/14.1.3...node-core/14.1.4 [14.1.3]: https://github.com/subquery/subql/compare/node-core/14.1.2...node-core/14.1.3 diff --git a/packages/node-core/package.json b/packages/node-core/package.json index 986b1fabbe..548d19f5e2 100644 --- a/packages/node-core/package.json +++ b/packages/node-core/package.json @@ -1,6 +1,6 @@ { "name": "@subql/node-core", - "version": "14.1.5", + "version": "14.1.7", "description": "Common node features that are agnostic to blockchains", "homepage": "https://github.com/subquery/subql", "repository": "github:subquery/subql", diff --git a/packages/node-core/src/indexer/fetch.service.spec.ts b/packages/node-core/src/indexer/fetch.service.spec.ts index c290757739..2e4d4eda6e 100644 --- a/packages/node-core/src/indexer/fetch.service.spec.ts +++ b/packages/node-core/src/indexer/fetch.service.spec.ts @@ -3,7 +3,7 @@ import {EventEmitter2} from '@nestjs/event-emitter'; import {SchedulerRegistry} from '@nestjs/schedule'; -import {BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry, IProjectNetworkConfig} from '@subql/types-core'; +import {BaseDataSource, BaseHandler, BaseMapping, DictionaryQueryEntry} from '@subql/types-core'; import {range} from 'lodash'; import { BaseUnfinalizedBlocksService, @@ -83,11 +83,6 @@ const nodeConfig = new NodeConfig({ networkDictionary: [''], }); -const getNetworkConfig = () => - ({ - dictionary: 'https://example.com', - }) as IProjectNetworkConfig; - const mockDs: BaseDataSource = { kind: 'mock/DataSource', startBlock: 1, @@ -164,9 +159,9 @@ describe('Fetch Service', () => { let fetchService: TestFetchService; let blockDispatcher: IBlockDispatcher; let dictionaryService: DictionaryService; - let networkConfig: IProjectNetworkConfig; let dataSources: BaseDataSource[]; let unfinalizedBlocksService: BaseUnfinalizedBlocksService; + let projectService: IProjectService; let spyOnEnqueueSequential: jest.SpyInstance< void | Promise, @@ -183,7 +178,7 @@ describe('Fetch Service', () => { const eventEmitter = new EventEmitter2(); const schedulerRegistry = new SchedulerRegistry(); - const projectService = { + projectService = { getStartBlockFromDataSources: jest.fn(() => Math.min(...dataSources.map((ds) => ds.startBlock ?? 0))), getAllDataSources: jest.fn(() => dataSources), getDataSourcesMap: jest.fn(() => { @@ -197,16 +192,15 @@ describe('Fetch Service', () => { }); return new BlockHeightMap(x); }), + bypassBlocks: [], } as any as IProjectService; blockDispatcher = getBlockDispatcher(); dictionaryService = getDictionaryService(); - networkConfig = getNetworkConfig(); fetchService = new TestFetchService( nodeConfig, projectService, - networkConfig, blockDispatcher, dictionaryService, eventEmitter, @@ -331,7 +325,8 @@ describe('Fetch Service', () => { ); await fetchService.init(1); - expect((fetchService as any).bypassBlocks).toEqual(range(301, 500)); + + expect((fetchService as any).getDatasourceBypassBlocks()).toEqual([`301-499`]); }); it('checks chain heads at an interval', async () => { @@ -614,7 +609,7 @@ describe('Fetch Service', () => { }); it('skips bypassBlocks', async () => { - (fetchService as any).networkConfig.bypassBlocks = [3]; + projectService.bypassBlocks = [3]; await fetchService.init(1); @@ -625,13 +620,10 @@ describe('Fetch Service', () => { it('transforms bypassBlocks', async () => { // Set a range so on init its transformed - (fetchService as any).networkConfig.bypassBlocks = ['2-5']; + projectService.bypassBlocks = ['2-5']; await fetchService.init(1); - // This doesn't work as they get removed after that height is processed - // expect((fetchService as any).bypassBlocks).toEqual([2, 3, 4, 5]); - // Note the batch size is smaller because we exclude from the initial batch size expect(enqueueBlocksSpy).toHaveBeenCalledWith([1, 6, 7, 8, 9, 10], 10); }); diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts index 17d5f10539..5e80595589 100644 --- a/packages/node-core/src/indexer/fetch.service.ts +++ b/packages/node-core/src/indexer/fetch.service.ts @@ -2,22 +2,21 @@ // SPDX-License-Identifier: GPL-3.0 import assert from 'assert'; -import util from 'util'; import {OnApplicationShutdown} from '@nestjs/common'; import {EventEmitter2} from '@nestjs/event-emitter'; import {SchedulerRegistry} from '@nestjs/schedule'; -import {BaseDataSource, IProjectNetworkConfig} from '@subql/types-core'; -import {range, without} from 'lodash'; +import {BaseDataSource} from '@subql/types-core'; +import {range} from 'lodash'; import {NodeConfig} from '../configure'; import {IndexerEvent} from '../events'; import {getLogger} from '../logger'; -import {cleanedBatchBlocks, delay, transformBypassBlocks, waitForBatchSize} from '../utils'; +import {delay, filterBypassBlocks, waitForBatchSize} from '../utils'; import {IBlockDispatcher} from './blockDispatcher'; import {mergeNumAndBlocksToNums} from './dictionary'; import {DictionaryService} from './dictionary/dictionary.service'; -import {getBlockHeight, mergeNumAndBlocks} from './dictionary/utils'; +import {mergeNumAndBlocks} from './dictionary/utils'; import {StoreCacheService} from './storeCache'; -import {Header, IBlock, IProjectService} from './types'; +import {BypassBlocks, Header, IBlock, IProjectService} from './types'; import {IUnfinalizedBlocksServiceUtil} from './unfinalizedBlocks.service'; const logger = getLogger('FetchService'); @@ -28,7 +27,6 @@ export abstract class BaseFetchService; @@ -48,7 +46,6 @@ export abstract class BaseFetchService, - protected networkConfig: IProjectNetworkConfig, protected blockDispatcher: B, protected dictionaryService: DictionaryService, private eventEmitter: EventEmitter2, @@ -77,31 +74,7 @@ export abstract class BaseFetchService { - this.bypassBlocks = []; - - if (this.networkConfig?.bypassBlocks !== undefined) { - this.bypassBlocks = transformBypassBlocks(this.networkConfig.bypassBlocks).filter((blk) => blk >= startHeight); - } - - this.updateBypassBlocksFromDatasources(); const interval = await this.getChainInterval(); await Promise.all([this.getFinalizedBlockHead(), this.getBestBlockHead()]); @@ -344,49 +317,52 @@ export abstract class BaseFetchService | number)[], latestHeight: number): Promise { - const cleanedBatchBlocks = this.filteredBlockBatch(enqueuingBlocks); + const cleanedBatchBlocks = filterBypassBlocks(enqueuingBlocks, [ + ...this.projectService.bypassBlocks, + ...this.getDatasourceBypassBlocks(), + ]); await this.blockDispatcher.enqueueBlocks( cleanedBatchBlocks, - this.getLatestBufferHeight(cleanedBatchBlocks, enqueuingBlocks, latestHeight) + this.getLatestBufferHeight(enqueuingBlocks, latestHeight) ); } /** * - * @param cleanedBatchBlocks * @param rawBatchBlocks * @param latestHeight * @private */ - private getLatestBufferHeight( - cleanedBatchBlocks: (IBlock | number)[], - rawBatchBlocks: (IBlock | number)[], - latestHeight: number - ): number { + private getLatestBufferHeight(rawBatchBlocks: (IBlock | number)[], latestHeight: number): number { // When both BatchBlocks are empty, mean no blocks to enqueue and full synced, // we are safe to update latestBufferHeight to this number - if (cleanedBatchBlocks.length === 0 && rawBatchBlocks.length === 0) { + if (rawBatchBlocks.length === 0) { return latestHeight; } - return Math.max(...mergeNumAndBlocksToNums(cleanedBatchBlocks, rawBatchBlocks)); + return Math.max(...mergeNumAndBlocksToNums([], rawBatchBlocks)); } - private filteredBlockBatch(currentBatchBlocks: (number | IBlock)[]): (number | IBlock)[] { - if (!this.bypassBlocks.length || !currentBatchBlocks) { - return currentBatchBlocks; - } + /** + * If a projects datasources are not continuious we can add add them to the bypass blocks + * */ + private getDatasourceBypassBlocks(): BypassBlocks { + const datasources = this.projectService.getDataSourcesMap().getAll(); + + const heights = Array.from(datasources.keys()); - const cleanedBatch = cleanedBatchBlocks(this.bypassBlocks, currentBatchBlocks); + const bypassBlocks: BypassBlocks = []; - const pollutedBlocks = this.bypassBlocks.filter( - (b) => b < Math.max(...currentBatchBlocks.map((b) => getBlockHeight(b))) - ); - if (pollutedBlocks.length) { - // inspect limits the number of logged blocks to 100 - logger.info(`Bypassing blocks: ${util.inspect(pollutedBlocks, {maxArrayLength: 100})}`); + for (let i = 0; i < heights.length - 1; i++) { + const currentHeight = heights[i]; + const nextHeight = heights[i + 1]; + + const currentDS = datasources.get(currentHeight); + // If the value for the current height is an empty array, then it's a gap + if (currentDS?.length === 0) { + bypassBlocks.push(`${currentHeight}-${nextHeight - 1}`); + } } - this.bypassBlocks = without(this.bypassBlocks, ...pollutedBlocks); - return cleanedBatch; + return bypassBlocks; } private nextEndBlockHeight(startBlockHeight: number, scaledBatchSize: number): number { diff --git a/packages/node-core/src/indexer/project.service.spec.ts b/packages/node-core/src/indexer/project.service.spec.ts index 4b224c2e75..4870290b72 100644 --- a/packages/node-core/src/indexer/project.service.spec.ts +++ b/packages/node-core/src/indexer/project.service.spec.ts @@ -1,11 +1,18 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {NodeConfig} from '../configure'; +import {EventEmitter2} from '@nestjs/event-emitter'; +import {buildSchemaFromString} from '@subql/utils'; +import {NodeConfig, ProjectUpgradeService} from '../configure'; import {BaseDsProcessorService} from './ds-processor.service'; import {DynamicDsService} from './dynamic-ds.service'; import {BaseProjectService} from './project.service'; -import {ISubqueryProject} from './types'; +import {Header, ISubqueryProject} from './types'; +import { + BaseUnfinalizedBlocksService, + METADATA_LAST_FINALIZED_PROCESSED_KEY, + METADATA_UNFINALIZED_BLOCKS_KEY, +} from './unfinalizedBlocks.service'; class TestProjectService extends BaseProjectService { packageVersion = '1.0.0'; @@ -17,6 +24,40 @@ class TestProjectService extends BaseProjectService { onProjectChange(project: any): void { return; } + + protected async getExistingProjectSchema(): Promise { + return Promise.resolve('test'); + } +} + +class TestUnfinalizedBlocksService extends BaseUnfinalizedBlocksService { + // eslint-disable-next-line @typescript-eslint/require-await + protected async getFinalizedHead(): Promise
{ + return { + blockHash: 'asdf', + blockHeight: 1000, + parentHash: 'efgh', + }; + } + + // eslint-disable-next-line @typescript-eslint/require-await + protected async getHeaderForHash(hash: string): Promise
{ + const num = parseInt(hash.slice(1), 10); + return { + blockHeight: num, + blockHash: hash, + parentHash: `b${num - 1}`, + }; + } + + // eslint-disable-next-line @typescript-eslint/require-await + protected async getHeaderForHeight(height: number): Promise
{ + return { + blockHeight: height, + blockHash: `b${height}`, + parentHash: `b${height - 1}`, + }; + } } describe('BaseProjectService', () => { @@ -225,4 +266,147 @@ describe('BaseProjectService', () => { ); }); }); + + // Tests initializing the project service to ensure that things are initialized in the correct order + // NOTE: this is not currently covering all scenarios + describe('initializing services', () => { + beforeAll(() => { + process.env.TZ = 'utc'; + }); + + const defaultProjects = [ + { + id: '1', + network: { + chainId: '1', + }, + dataSources: [{startBlock: 1}], + schema: buildSchemaFromString(`type TestEntity @entity { + id: ID! + fieldOne: String + fieldTwo: Int + # fieldThree: BigInt! +}`), + applyCronTimestamps: jest.fn(), + } as unknown as ISubqueryProject, + ]; + + const setupProject = async ( + startBlock = 1, + unfinalizedBlocks: Header[] = [], + lastFinalizedHeight?: number, + projects: ISubqueryProject[] = defaultProjects + ) => { + const project = projects[0]; + + const projectUpgradeService = await ProjectUpgradeService.create(project, (id: string) => + Promise.resolve(projects[parseInt(id, 10)]) + ); + + const nodeConfig = {unsafe: false} as unknown as NodeConfig; + + const storeService = { + init: jest.fn(), + initCoreTables: jest.fn(), + historical: true, + storeCache: { + metadata: { + findMany: jest.fn(() => ({})), + find: jest.fn((key: string) => { + switch (key) { + case METADATA_LAST_FINALIZED_PROCESSED_KEY: + return lastFinalizedHeight; + case METADATA_UNFINALIZED_BLOCKS_KEY: + return JSON.stringify(unfinalizedBlocks); + case 'lastProcessedHeight': + return startBlock - 1; + case 'deployments': + return JSON.stringify({1: '1'}); + default: + return undefined; + } + }), + set: jest.fn(), + flush: jest.fn(), + }, + resetCache: jest.fn(), + flushCache: jest.fn(), + _flushCache: jest.fn(), + }, + rewind: jest.fn(), + } as unknown as any; + + service = new TestProjectService( + { + validateProjectCustomDatasources: jest.fn(), + } as unknown as BaseDsProcessorService, // dsProcessorService + {networkMeta: {}} as unknown as any, //apiService + null as unknown as any, // poiService + null as unknown as any, // poiSyncService + { + transaction: jest.fn(() => ({ + rollback: jest.fn(), + commit: jest.fn(), + })), + } as unknown as any, // sequelize + project, // project + projectUpgradeService, // projectUpgradeService + storeService, // storeService + nodeConfig, + { + init: jest.fn(), + getDynamicDatasources: jest.fn(() => []), + resetDynamicDatasource: jest.fn(), + } as unknown as DynamicDsService, // dynamicDsService + new EventEmitter2(), // eventEmitter + new TestUnfinalizedBlocksService(nodeConfig, storeService.storeCache) // unfinalizedBlocksService + ); + }; + + it('succeeds with no rewinds', async () => { + await setupProject(); + + await expect(service.init()).resolves.not.toThrow(); + }); + + it('succeeds with a project upgrade rewind', async () => { + const projects = [ + { + ...defaultProjects[0], + id: '0', + parent: {block: 20, untilBlock: 20, reference: '1'}, + schema: buildSchemaFromString(`type TestEntity @entity { + id: ID! + fieldOne: String + fieldTwo: Int + fieldThree: BigInt! +}`), + }, + ...defaultProjects, + ]; + await setupProject(100, [], 100, projects); + + const reindexSpy = jest.spyOn(service, 'reindex'); + await service.init(); + // await expect(service.init()).resolves.not.toThrow(); + expect(reindexSpy).toHaveReturnedTimes(1); + }); + + it('succeeds with an unfinalized blocks rewind', async () => { + await setupProject( + 95, + [ + {blockHeight: 100, blockHash: 'a100', parentHash: 'a99'}, + {blockHeight: 99, blockHash: 'a99', parentHash: 'a98'}, + ], + 90 + ); + + const reindexSpy = jest.spyOn(service, 'reindex'); + + await expect(service.init()).resolves.not.toThrow(); + + expect(reindexSpy).toHaveReturnedTimes(1); + }); + }); }); diff --git a/packages/node-core/src/indexer/project.service.ts b/packages/node-core/src/indexer/project.service.ts index bb25413bde..09048f666a 100644 --- a/packages/node-core/src/indexer/project.service.ts +++ b/packages/node-core/src/indexer/project.service.ts @@ -20,7 +20,7 @@ import {PoiSyncService} from './poi'; import {PoiService} from './poi/poi.service'; import {StoreService} from './store.service'; import {isCachePolicy} from './storeCache'; -import {ISubqueryProject, IProjectService} from './types'; +import {ISubqueryProject, IProjectService, BypassBlocks} from './types'; import {IUnfinalizedBlocksService} from './unfinalizedBlocks.service'; const logger = getLogger('Project'); @@ -84,11 +84,15 @@ export abstract class BaseProjectService< return this._blockOffset; } + get bypassBlocks(): BypassBlocks { + return this.project.network.bypassBlocks ?? []; + } + protected get isHistorical(): boolean { return this.storeService.historical; } - private async getExistingProjectSchema(): Promise { + protected async getExistingProjectSchema(): Promise { return getExistingProjectSchema(this.nodeConfig, this.sequelize); } @@ -133,6 +137,7 @@ export abstract class BaseProjectService< void this.poiSyncService.syncPoi(undefined); } + const reindexedUpgrade = await this.initUpgradeService(this.startHeight); // Unfinalized is dependent on POI in some cases, it needs to be init after POI is init const reindexedUnfinalized = await this.initUnfinalizedInternal(); @@ -140,8 +145,6 @@ export abstract class BaseProjectService< this._startHeight = reindexedUnfinalized; } - const reindexedUpgrade = await this.initUpgradeService(this.startHeight); - if (reindexedUpgrade !== undefined) { this._startHeight = reindexedUpgrade; } diff --git a/packages/node-core/src/indexer/store/store.ts b/packages/node-core/src/indexer/store/store.ts index a83ae841bf..87a4ca78eb 100644 --- a/packages/node-core/src/indexer/store/store.ts +++ b/packages/node-core/src/indexer/store/store.ts @@ -5,7 +5,6 @@ import assert from 'assert'; import {Store as IStore, Entity, FieldsExpression, GetOptions} from '@subql/types-core'; import {Transaction} from '@subql/x-sequelize'; import {NodeConfig} from '../../configure'; -import {getLogger} from '../../logger'; import {monitorWrite} from '../../process'; import {handledStringify} from '../../utils'; import {IStoreModelProvider} from '../storeCache'; @@ -13,8 +12,6 @@ import {StoreOperations} from '../StoreOperations'; import {OperationType} from '../types'; import {EntityClass} from './entity'; -const logger = getLogger('Store'); - /* A context is provided to allow it to be updated by the owner of the class instance */ type Context = { blockHeight: number; @@ -36,17 +33,9 @@ export class Store implements IStore { this.#context = context; } - #queryLimitCheck(storeMethod: string, entity: string, options?: GetOptions) { - if (options) { - if (options.limit && this.#config.queryLimit < options.limit) { - logger.warn( - `store ${storeMethod} for entity ${entity} with ${options.limit} records exceeds config limit ${ - this.#config.queryLimit - }. Will use ${this.#config.queryLimit} as the limit.` - ); - } - - options.limit = options.limit ? Math.min(options.limit, this.#config.queryLimit) : this.#config.queryLimit; + #queryLimitCheck(storeMethod: string, entity: string, options: GetOptions) { + if (options.limit > this.#config.queryLimit) { + throw new Error(`Query limit exceeds the maximum allowed value of ${this.#config.queryLimit}`); } } @@ -64,7 +53,7 @@ export class Store implements IStore { entity: string, field: keyof T, value: T[keyof T] | T[keyof T][], - options: GetOptions = {} + options: GetOptions ): Promise { try { const indexed = this.#context.isIndexed(entity, String(field)); diff --git a/packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts b/packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts index 6c37d9dd50..ca2f9ce71c 100644 --- a/packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts +++ b/packages/node-core/src/indexer/storeCache/model/cacheModel.spec.ts @@ -229,6 +229,11 @@ describe('cacheModel', () => { testModel = new CachedModel(sequelize.model('entity1'), true, {} as NodeConfig, () => i++); }); + it('throws when trying to set undefined', () => { + expect(() => testModel.set('0x01', undefined as any, 1)).toThrow(); + expect(() => testModel.set('0x01', null as any, 1)).toThrow(); + }); + // it should keep same behavior as hook we used it('when get data after flushed, it should exclude block range', async () => { const spyDbGet = jest.spyOn(testModel.model, 'findOne'); diff --git a/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts b/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts index fadf57e5e2..eff3b0b774 100644 --- a/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts +++ b/packages/node-core/src/indexer/storeCache/model/cacheModel.test.ts @@ -374,12 +374,28 @@ describe('cacheModel integration', () => { cacheModel = new CachedModel(model, false, new NodeConfig({} as any), () => i++); }); + it('behaves as expected with removals', async () => { + // Create data but dont flush, this puts it in the cache + await setDefaultData('0x01', 1, undefined, false); + + // Remove the data, this could happen at any block > 1 but before flushing. + cacheModel.remove('0x01', 1); + + // The result should be undefined as it will be marked as removed + const res0 = await cacheModel.get('0x01'); + expect(res0).toBeUndefined(); + + // The data will still be in the set cache but it should not be included in the result + const res = await cacheModel.getByFields([], {offset: 0, limit: 10, orderBy: 'selfStake', orderDirection: 'ASC'}); + expect(res).toEqual([]); + }); + afterAll(async () => { await sequelize.dropSchema(schema, {logging: false}); await sequelize.close(); }); - async function setDefaultData(id: string, height: number, data?: any): Promise { + async function setDefaultData(id: string, height: number, data?: any, flushData = true): Promise { cacheModel.set( id, data ?? { @@ -391,7 +407,9 @@ describe('cacheModel integration', () => { }, height ); - await flush(height + 1); + if (flushData) { + await flush(height + 1); + } } describe('cached data and db data compare', () => { diff --git a/packages/node-core/src/indexer/storeCache/model/cacheModel.ts b/packages/node-core/src/indexer/storeCache/model/cacheModel.ts index 683a9b853c..2f7ef5771c 100644 --- a/packages/node-core/src/indexer/storeCache/model/cacheModel.ts +++ b/packages/node-core/src/indexer/storeCache/model/cacheModel.ts @@ -136,6 +136,7 @@ export class CachedModel [(value: SetValueModel) => value.getLatest()?.data?.[fullOptions.orderBy]], [fullOptions.orderDirection.toLowerCase() as 'asc' | 'desc'] ) + .filter((value) => !value.getLatest()?.removed) // Ensure the data has not been marked for removal .filter((value) => value.matchesFields(filters)) // This filters out any removed/undefined .map((value) => value.getLatest()?.data) .map((value) => cloneDeep(value)) as T[]; @@ -187,9 +188,10 @@ export class CachedModel // eslint-disable-next-line @typescript-eslint/require-await async set(id: string, data: T, blockHeight: number): Promise { - if (this.setCache[id] === undefined) { - this.setCache[id] = new SetValueModel(); + if (data === undefined || data === null) { + throw new Error('Cannot set undefined or null data. If you wish to remove data, use remove()'); } + this.setCache[id] ??= new SetValueModel(); const copiedData = cloneDeep(data); this.setCache[id].set(copiedData, blockHeight, this.getNextStoreOperationIndex()); // Experimental, this means getCache keeps duplicate data from setCache, @@ -324,13 +326,16 @@ export class CachedModel } private filterRemoveRecordByHeight(blockHeight: number, lessEqt: boolean): Record { - return Object.entries(this.removeCache).reduce((acc, [key, value]) => { - if (lessEqt ? value.removedAtBlock <= blockHeight : value.removedAtBlock > blockHeight) { - acc[key] = value; - } + return Object.entries(this.removeCache).reduce( + (acc, [key, value]) => { + if (lessEqt ? value.removedAtBlock <= blockHeight : value.removedAtBlock > blockHeight) { + acc[key] = value; + } - return acc; - }, {} as Record); + return acc; + }, + {} as Record + ); } private filterRecordsWithHeight(blockHeight: number): FilteredHeightRecords { diff --git a/packages/node-core/src/indexer/test.runner.spec.ts b/packages/node-core/src/indexer/test.runner.spec.ts index e425b15b28..1a977c03c6 100644 --- a/packages/node-core/src/indexer/test.runner.spec.ts +++ b/packages/node-core/src/indexer/test.runner.spec.ts @@ -6,6 +6,11 @@ import {TestRunner} from './test.runner'; jest.mock('@subql/x-sequelize'); +const mockStoreCache = { + flushCache: jest.fn().mockResolvedValue(undefined), + metadata: {set: jest.fn()}, +}; + describe('TestRunner', () => { let testRunner: TestRunner; let sequelizeMock: jest.Mocked; @@ -22,7 +27,7 @@ describe('TestRunner', () => { storeServiceMock = { setBlockHeight: jest.fn(), getStore: jest.fn().mockReturnValue({}), - storeCache: {flushCache: jest.fn().mockResolvedValue(undefined)}, + storeCache: mockStoreCache, }; sandboxMock = { @@ -83,7 +88,7 @@ describe('TestRunner', () => { (testRunner as any).storeService = { getStore: () => storeMock, setBlockHeight: jest.fn(), - storeCache: {flushCache: jest.fn().mockResolvedValue(undefined)}, + storeCache: mockStoreCache, } as any; await testRunner.runTest(testMock, sandboxMock, indexBlock); @@ -118,7 +123,7 @@ describe('TestRunner', () => { (testRunner as any).storeService = { getStore: () => storeMock, setBlockHeight: jest.fn(), - storeCache: {flushCache: jest.fn().mockResolvedValue(undefined)}, + storeCache: mockStoreCache, } as any; await testRunner.runTest(testMock, sandboxMock, indexBlock); @@ -175,7 +180,7 @@ describe('TestRunner', () => { (testRunner as any).storeService = { getStore: () => storeMock, setBlockHeight: jest.fn(), - storeCache: {flushCache: jest.fn().mockResolvedValue(undefined)}, + storeCache: mockStoreCache, } as any; await testRunner.runTest(testMock, sandboxMock, indexBlock); diff --git a/packages/node-core/src/indexer/test.runner.ts b/packages/node-core/src/indexer/test.runner.ts index 4cd68fce5f..16eff11b22 100644 --- a/packages/node-core/src/indexer/test.runner.ts +++ b/packages/node-core/src/indexer/test.runner.ts @@ -58,6 +58,8 @@ export class TestRunner { const [block] = await this.apiService.fetchBlocks([test.blockHeight]); this.storeService.setBlockHeight(test.blockHeight); + // Ensure a block height is set so that data is flushed correctly + this.storeService.modelProvider.metadata.set('lastProcessedHeight', test.blockHeight - 1); const store = this.storeService.getStore(); sandbox.freeze(store, 'store'); @@ -113,7 +115,9 @@ export class TestRunner { return value; }; failedAttributes.push( - `\t\tattribute: "${attr}":\n\t\t\texpected: "${fmtValue(expectedAttr)}"\n\t\t\tactual: "${fmtValue(actualAttr)}"\n` + `\t\tattribute: "${attr}":\n\t\t\texpected: "${fmtValue(expectedAttr)}"\n\t\t\tactual: "${fmtValue( + actualAttr + )}"\n` ); } }); diff --git a/packages/node-core/src/indexer/types.ts b/packages/node-core/src/indexer/types.ts index 5ef43989e5..3b22684897 100644 --- a/packages/node-core/src/indexer/types.ts +++ b/packages/node-core/src/indexer/types.ts @@ -43,6 +43,7 @@ export interface IIndexerManager { export interface IProjectService { blockOffset: number | undefined; startHeight: number; + bypassBlocks: BypassBlocks; reindex(lastCorrectHeight: number): Promise; /** * This is used everywhere but within indexing blocks, see comment on getDataSources for more info @@ -68,3 +69,5 @@ export type Header = { blockHash: string; parentHash: string | undefined; }; + +export type BypassBlocks = (number | `${number}-${number}`)[]; diff --git a/packages/node-core/src/indexer/worker/worker.store.service.ts b/packages/node-core/src/indexer/worker/worker.store.service.ts index db8965d58a..07a07026e1 100644 --- a/packages/node-core/src/indexer/worker/worker.store.service.ts +++ b/packages/node-core/src/indexer/worker/worker.store.service.ts @@ -1,14 +1,23 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {Store, FieldsExpression, GetOptions} from '@subql/types-core'; +import {Store, FieldsExpression, GetOptions, Entity} from '@subql/types-core'; import {unwrapProxyArgs} from './utils'; export type HostStore = { // This matches the store interface storeGet: (entity: string, id: string) => Promise; - storeGetByField: (entity: string, field: string, value: any, options?: GetOptions) => Promise; - storeGetByFields: (entity: string, filter: FieldsExpression[], options?: GetOptions) => Promise; + storeGetByField: ( + entity: string, + field: keyof T, + value: any, + options: GetOptions + ) => Promise; + storeGetByFields: ( + entity: string, + filter: FieldsExpression[], + options: GetOptions + ) => Promise; storeGetOneByField: (entity: string, field: string, value: any) => Promise; storeSet: (entity: string, id: string, data: any) => Promise; storeBulkCreate: (entity: string, data: any[]) => Promise; @@ -48,8 +57,8 @@ export const hostStoreToStore = (host: HostStore): Store => { export function storeHostFunctions(store: Store): HostStore { return { storeGet: store.get.bind(store), - storeGetByField: store.getByField.bind(store), - storeGetByFields: store.getByFields.bind(store), + storeGetByField: store.getByField.bind(store), + storeGetByFields: store.getByFields.bind(store), storeGetOneByField: store.getOneByField.bind(store), storeSet: store.set.bind(store), storeBulkCreate: store.bulkCreate.bind(store), diff --git a/packages/node-core/src/utils/blocks.spec.ts b/packages/node-core/src/utils/blocks.spec.ts index caa073d9f3..3bed025899 100644 --- a/packages/node-core/src/utils/blocks.spec.ts +++ b/packages/node-core/src/utils/blocks.spec.ts @@ -2,28 +2,28 @@ // SPDX-License-Identifier: GPL-3.0 import {range} from 'lodash'; -import {transformBypassBlocks, cleanedBatchBlocks} from './blocks'; +import {BypassBlocks} from '../indexer'; +import {filterBypassBlocks} from './blocks'; describe('bypass logic', () => { it('process bypassBlocks with ranges', () => { - let bypassBlocks = transformBypassBlocks([20, 40, '5-10', 20, 140]); - expect(bypassBlocks).toEqual([20, 40, 5, 6, 7, 8, 9, 10, 140]); + let bypassBlocks: BypassBlocks = [20, 40, '5-10', 20, 140]; let currentBlockBatch = [1, 5, 7, 8, 20, 40, 100, 120]; - const case_1 = cleanedBatchBlocks(bypassBlocks, currentBlockBatch); + const case_1 = filterBypassBlocks(currentBlockBatch, bypassBlocks); expect(case_1).toEqual([1, 100, 120]); - bypassBlocks = transformBypassBlocks([' 5 - 10 ', 20, 140]); + bypassBlocks = [' 5 - 10 ', 20, 140]; currentBlockBatch = [1, 5, 7, 8, 20, 40, 100, 120]; - const case_2 = cleanedBatchBlocks(bypassBlocks, currentBlockBatch); + const case_2 = filterBypassBlocks(currentBlockBatch, bypassBlocks); expect(case_2).toEqual([1, 40, 100, 120]); }); it('cleanedBatchBlocks with large amount blocks should not throw error Maximum call stack size exceeded', () => { - const bypassBlocks = transformBypassBlocks(['50051722-54939220']); + const bypassBlocks: BypassBlocks = ['50051722-54939220']; const currentBlockBatch = range(34312396, 34312495); - const case_1 = cleanedBatchBlocks(bypassBlocks, currentBlockBatch); + const case_1 = filterBypassBlocks(currentBlockBatch, bypassBlocks); expect(case_1).toEqual(currentBlockBatch); }); }); diff --git a/packages/node-core/src/utils/blocks.ts b/packages/node-core/src/utils/blocks.ts index 6654fa1211..46e783e8fc 100644 --- a/packages/node-core/src/utils/blocks.ts +++ b/packages/node-core/src/utils/blocks.ts @@ -2,42 +2,10 @@ // SPDX-License-Identifier: GPL-3.0 import {Schedule} from 'cron-converter'; -import {chunk, flatten, isNumber, range, uniq, without} from 'lodash'; import {getBlockHeight} from '../indexer/dictionary'; -import {IBlock} from '../indexer/types'; +import {BypassBlocks, IBlock} from '../indexer/types'; import {getLogger} from '../logger'; -export function cleanedBatchBlocks( - bypassBlocks: number[], - currentBlockBatch: (IBlock | number)[] -): (IBlock | number)[] { - // Use suggested work around to avoid Maximum call stack size exceeded issue when large numbers of transformedBlocks - // https://github.com/lodash/lodash/issues/5552 - const transformedBlocks = transformBypassBlocks(bypassBlocks); - let result = currentBlockBatch; - chunk(transformedBlocks, 10000).forEach((chunk) => { - result = without( - result.map((r) => getBlockHeight(r)), - ...chunk - ); - }); - return result; -} - -export function transformBypassBlocks(bypassBlocks: (number | string)[]): number[] { - if (!bypassBlocks?.length) return []; - - return uniq( - flatten( - bypassBlocks.map((bypassEntry) => { - if (isNumber(bypassEntry)) return [bypassEntry]; - const splitRange = bypassEntry.split('-').map((val) => parseInt(val.trim(), 10)); - return range(splitRange[0], splitRange[1] + 1); - }) - ) - ); -} - const logger = getLogger('timestamp-filter'); export type CronFilter = { @@ -61,3 +29,27 @@ export function filterBlockTimestamp(blockTimestamp: number, filter: CronFilter) return false; } } + +/** + * Takes a list of blocks or block numbers to be enqueued and indexed and removes any based on the bypassBlocks + * TODO this could further be optimised by returning the end of a block range to fast forward to. + * */ +export function filterBypassBlocks( + enqueuedBlocks: (IBlock | number)[], + bypassBlocks?: BypassBlocks +): (IBlock | number)[] { + if (!bypassBlocks?.length) return enqueuedBlocks; + + return enqueuedBlocks.filter((b) => { + if (typeof b === 'number') !inBypassBlocks(bypassBlocks, b); + return !inBypassBlocks(bypassBlocks, getBlockHeight(b)); + }); +} + +function inBypassBlocks(bypassBlocks: BypassBlocks, blockNum: number): boolean { + return bypassBlocks.some((bypassEntry) => { + if (typeof bypassEntry === 'number') return bypassEntry === blockNum; + const [start, end] = bypassEntry.split('-').map((val) => parseInt(val.trim(), 10)); + return blockNum >= start && blockNum <= end; + }); +} diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 7413b9ff4e..92a3f15a83 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [5.2.9] - 2024-10-30 +### Changed +- Bump `@subql/node-core` dependency + +## [5.2.8] - 2024-10-23 +### Changed +- Bump `@subql/common-substrate` dependency + +## [5.2.7] - 2024-10-21 +### Fixed +- Issues with setting a large block range for bypass blocks (#2566) + ## [5.2.6] - 2024-09-25 ### Changed - Bump common, Added manifest support for query-subgraph. @@ -1311,7 +1323,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - bump @polkadot/api to 3.1.1 -[Unreleased]: https://github.com/subquery/subql/compare/node/5.2.6...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/node/5.2.9...HEAD +[5.2.9]: https://github.com/subquery/subql/compare/node/5.2.8...node/5.2.9 +[5.2.8]: https://github.com/subquery/subql/compare/node/5.2.7...node/5.2.8 +[5.2.7]: https://github.com/subquery/subql/compare/node/5.2.6...node/5.2.7 [5.2.6]: https://github.com/subquery/subql/compare/node/5.2.5...node/5.2.6 [5.2.5]: https://github.com/subquery/subql/compare/node/5.2.3...node/5.2.5 [5.2.3]: https://github.com/subquery/subql/compare/node/5.2.2...node/5.2.3 diff --git a/packages/node/package.json b/packages/node/package.json index c6b4a7ed89..7dd8da8fb2 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@subql/node", - "version": "5.2.6", + "version": "5.2.9", "description": "", "author": "Ian He", "license": "GPL-3.0", diff --git a/packages/node/src/configure/SchemaMigration.service.test.ts b/packages/node/src/configure/SchemaMigration.service.test.ts index a666bbb0c5..757143fe4a 100644 --- a/packages/node/src/configure/SchemaMigration.service.test.ts +++ b/packages/node/src/configure/SchemaMigration.service.test.ts @@ -40,7 +40,9 @@ describe('SchemaMigration integration tests', () => { }); afterAll(async () => { - await rimraf(tempDir); + if (tempDir) { + await rimraf(tempDir); + } await sequelize?.close(); }); diff --git a/packages/node/src/indexer/fetch.service.spec.ts b/packages/node/src/indexer/fetch.service.spec.ts index 2b6efcc81b..fdd7f6fe8b 100644 --- a/packages/node/src/indexer/fetch.service.spec.ts +++ b/packages/node/src/indexer/fetch.service.spec.ts @@ -63,7 +63,6 @@ describe('FetchSevice', () => { null as any, // ApiService null as any, // NodeConfig projectService, // ProjectService - {} as any, // Project null as any, // BlockDispatcher, null as any, // DictionaryService null as any, // UnfinalizedBlocks diff --git a/packages/node/src/indexer/fetch.service.test.ts b/packages/node/src/indexer/fetch.service.test.ts index e1efc082f5..4f2027ba9e 100644 --- a/packages/node/src/indexer/fetch.service.test.ts +++ b/packages/node/src/indexer/fetch.service.test.ts @@ -26,7 +26,6 @@ describe('FetchService', () => { apiService, // ApiService null as any, // NodeConfig null as any, // ProjectService - {} as any, // Project null as any, // BlockDispatcher, null as any, // DictionaryService { diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 28ad4bfcce..d3927e5c8b 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -15,7 +15,6 @@ import { StoreCacheService, } from '@subql/node-core'; import { SubstrateDatasource, SubstrateBlock } from '@subql/types'; -import { SubqueryProject } from '../configure/SubqueryProject'; import { calcInterval, substrateHeaderToHeader } from '../utils/substrate'; import { ApiService } from './api.service'; import { ISubstrateBlockDispatcher } from './blockDispatcher/substrate-block-dispatcher'; @@ -37,7 +36,6 @@ export class FetchService extends BaseFetchService< private apiService: ApiService, nodeConfig: NodeConfig, @Inject('IProjectService') projectService: ProjectService, - @Inject('ISubqueryProject') project: SubqueryProject, @Inject('IBlockDispatcher') blockDispatcher: ISubstrateBlockDispatcher, dictionaryService: SubstrateDictionaryService, @@ -50,7 +48,6 @@ export class FetchService extends BaseFetchService< super( nodeConfig, projectService, - project.network, blockDispatcher, dictionaryService, eventEmitter, diff --git a/packages/testing/CHANGELOG.md b/packages/testing/CHANGELOG.md index fa9d996c8a..f517416834 100644 --- a/packages/testing/CHANGELOG.md +++ b/packages/testing/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.2.2] - 2024-10-23 +### Fixed +- Bump version `@subql/types-core` + ## [2.2.1] - 2024-07-09 ### Changed - Changes to ts build settings (#2475) @@ -38,7 +42,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Major release 2.0.0, align with other package versions -[Unreleased]: https://github.com/subquery/subql/compare/testing/2.2.1...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/testing/2.2.2...HEAD +[2.2.2]: https://github.com/subquery/subql/compare/testing/2.2.1...testing/2.2.2 [2.2.1]: https://github.com/subquery/subql/compare/testing/2.2.0...testing/2.2.1 [2.2.0]: https://github.com/subquery/subql/compare/testing/2.1.2...testing/2.2.0 [2.1.2]: https://github.com/subquery/subql/compare/testing/2.1.1...testing/2.1.2 diff --git a/packages/testing/package.json b/packages/testing/package.json index c6e099d2bb..f9653ce862 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -1,6 +1,6 @@ { "name": "@subql/testing", - "version": "2.2.1", + "version": "2.2.2", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "build": "rm -rf dist && tsc -b", diff --git a/packages/types-core/CHANGELOG.md b/packages/types-core/CHANGELOG.md index eb9ab22bd0..b17ef1b30b 100644 --- a/packages/types-core/CHANGELOG.md +++ b/packages/types-core/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.0.0] - 2024-10-21 +### Changed +- Update store interface making options.limit required on getByField(s) methods (#2567) + ## [1.1.1] - 2024-08-12 ### Fixed - DS Processor types not being able to distinguish input and filter types (#2522) @@ -70,7 +74,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Initial release of new package containing types common to all chains -[Unreleased]: https://github.com/subquery/subql/compare/types-core/1.1.1...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/types-core/2.0.0...HEAD +[2.0.0]: https://github.com/subquery/subql/compare/types-core/1.1.1...types-core/2.0.0 [1.1.1]: https://github.com/subquery/subql/compare/types-core/1.1.0...types-core/1.1.1 [1.1.0]: https://github.com/subquery/subql/compare/types-core/1.0.0...types-core/1.1.0 [1.0.0]: https://github.com/subquery/subql/compare/types-core/0.10.0...types-core/1.0.0 diff --git a/packages/types-core/package.json b/packages/types-core/package.json index 095358d566..96fa4a909f 100644 --- a/packages/types-core/package.json +++ b/packages/types-core/package.json @@ -1,6 +1,6 @@ { "name": "@subql/types-core", - "version": "1.1.1", + "version": "2.0.0", "description": "", "homepage": "https://github.com/subquery/subql", "repository": "github:subquery/subql", diff --git a/packages/types-core/src/store.ts b/packages/types-core/src/store.ts index 58487ed81e..1774e510df 100644 --- a/packages/types-core/src/store.ts +++ b/packages/types-core/src/store.ts @@ -20,8 +20,11 @@ export interface Entity { } export type GetOptions = { + /** + * The number of items to return, if this exceeds the query-limit flag it will throw + * */ + limit: number; offset?: number; - limit?: number; orderBy?: keyof T; orderDirection?: 'ASC' | 'DESC'; }; @@ -33,11 +36,11 @@ export interface Store { * * ⚠️ This function will first search cache data followed by DB data. Please consider this when using order and offset options.⚠️ * */ - getByFields(entity: string, filter: FieldsExpression[], options?: GetOptions): Promise; + getByFields(entity: string, filter: FieldsExpression[], options: GetOptions): Promise; /** * This is an alias for getByFields with a single filter * */ - getByField(entity: string, field: string, value: any, options?: GetOptions): Promise; + getByField(entity: string, field: keyof T, value: any, options: GetOptions): Promise; /** * This is an alias for getByField with limit set to 1 * */ diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index 4ffc44f75e..a8552dbf43 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.11.3] - 2024-10-23 +### Fixed +- Bump version `@subql/types-core` + ## [3.11.2] - 2024-08-14 ### Added - Update polkadot/api library @@ -267,7 +271,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - support block handler -[Unreleased]: https://github.com/subquery/subql/compare/types/3.11.2...HEAD +[Unreleased]: https://github.com/subquery/subql/compare/types/3.11.3...HEAD +[3.11.3]: https://github.com/subquery/subql/compare/types/3.11.2...types/3.11.3 [3.11.2]: https://github.com/subquery/subql/compare/types/3.11.1...types/3.11.2 [3.11.1]: https://github.com/subquery/subql/compare/types/3.11.0...types/3.11.1 [3.11.0]: https://github.com/subquery/subql/compare/types/3.10.0...types/3.11.0 diff --git a/packages/types/package.json b/packages/types/package.json index 45a24f7644..bfdc0500b3 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@subql/types", - "version": "3.11.2", + "version": "3.11.3", "description": "", "homepage": "https://github.com/subquery/subql", "repository": "github:subquery/subql", diff --git a/test/Dockerfile b/test/Dockerfile index 4045416b5a..b0f2dbef71 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,8 @@ FROM node:lts-bullseye WORKDIR /workdir +# Fix timezone to UTC +ENV TZ=utc + COPY . . RUN yarn diff --git a/yarn.lock b/yarn.lock index 267f76ecc8..282a2e7160 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6918,14 +6918,6 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@^1.0.0, @subql/types-core@workspace:*, @subql/types-core@workspace:^, @subql/types-core@workspace:packages/types-core": - version: 0.0.0-use.local - resolution: "@subql/types-core@workspace:packages/types-core" - dependencies: - package-json-type: ^1.0.3 - languageName: unknown - linkType: soft - "@subql/types-core@npm:0.10.0, @subql/types-core@npm:^0.10.0": version: 0.10.0 resolution: "@subql/types-core@npm:0.10.0" @@ -6944,6 +6936,21 @@ __metadata: languageName: node linkType: hard +"@subql/types-core@npm:^1.0.0": + version: 1.1.1 + resolution: "@subql/types-core@npm:1.1.1" + checksum: d804ba8f9a9a8bbce36e98ef3dd03e33602495e96ca8e04438c38b6631de102772b5f209de17d78657bcacc9c91f9af02dabd68edb8e6e24a2ca6e681e4636fc + languageName: node + linkType: hard + +"@subql/types-core@workspace:*, @subql/types-core@workspace:^, @subql/types-core@workspace:packages/types-core": + version: 0.0.0-use.local + resolution: "@subql/types-core@workspace:packages/types-core" + dependencies: + package-json-type: ^1.0.3 + languageName: unknown + linkType: soft + "@subql/types-cosmos@npm:3.5.1": version: 3.5.1 resolution: "@subql/types-cosmos@npm:3.5.1"