Merge branch 'main' into strict-node

.yarn/versions/1288810f.yml

@@ -0,0 +1,5 @@
+undecided:
+  - "@subql/cli"
+  - "@subql/common"
+  - "@subql/node"
+  - "@subql/node-core"

packages/cli/CHANGELOG.md

@@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [4.14.0] - 2024-06-18
 ### Removed
 - Deprecated flare SDK (#2428)
 
@@ -609,7 +610,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - support subcommand codegen
 - support subcommand init
 
-[Unreleased]: https://github.com/subquery/subql/compare/cli/4.13.1...HEAD
+[Unreleased]: https://github.com/subquery/subql/compare/cli/4.14.0...HEAD
+[4.14.0]: https://github.com/subquery/subql/compare/cli/4.13.1...cli/4.14.0
 [4.13.1]: https://github.com/subquery/subql/compare/cli/4.13.0...cli/4.13.1
 [4.13.0]: https://github.com/subquery/subql/compare/cli/4.12.0...cli/4.13.0
 [4.12.0]: https://github.com/subquery/subql/compare/cli/4.11.0...cli/4.12.0

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@subql/cli",
   "description": "cli for subquery",
-  "version": "4.13.2-0",
+  "version": "4.14.0",
   "author": "Ian He",
   "bin": {
     "subql": "./bin/run"
@@ -89,6 +89,5 @@
   "types": "lib/index.d.ts",
   "resolutions": {
     "node-fetch": "2.6.7"
-  },
-  "stableVersion": "4.13.1"
+  }
 }

packages/common/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+## [3.8.0] - 2024-06-18
 ### Changed
 - Deprecated flare SDK (#2428)
 
@@ -369,7 +371,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - init commit
 
-[Unreleased]: https://github.com/subquery/subql/compare/common/3.7.0...HEAD
+[Unreleased]: https://github.com/subquery/subql/compare/common/3.8.0...HEAD
+[3.8.0]: https://github.com/subquery/subql/compare/common/3.7.0...common/3.8.0
 [3.7.0]: https://github.com/subquery/subql/compare/common/3.6.0...common/3.7.0
 [3.6.0]: https://github.com/subquery/subql/compare/common/3.5.1...common/3.6.0
 [3.5.1]: https://github.com/subquery/subql/compare/common/3.5.0...common/3.5.1

packages/common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/common",
-  "version": "3.7.1-0",
+  "version": "3.8.0",
   "description": "",
   "scripts": {
     "build": "rm -rf dist && tsc -b",
@@ -34,6 +34,5 @@
     "@types/semver": "^7.3.9",
     "@types/tar": "^6.1.1",
     "@types/update-notifier": "^6"
-  },
-  "stableVersion": "3.7.0"
+  }
 }

packages/node-core/CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [10.6.0] - 2024-06-18
+### Fixed
+- Testing framework logging the same date if they were within 1s (#2453)
+
+### Fixed
+- Handle edge case when last processed height is same as chain target height, app will wait for new block rather than exit.
+
+### Changed
+- Skip using the dictionary if processing data within batch size of latest height (#2454)
+
 ## [10.5.1] - 2024-06-12
 ### Changed
 - Update `@subql/utils`
@@ -722,7 +732,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Move blockchain agnostic code from `node` to `node-core` package. (#1222)
 
-[Unreleased]: https://github.com/subquery/subql/compare/node-core/10.5.1...HEAD
+[Unreleased]: https://github.com/subquery/subql/compare/node-core/10.6.0...HEAD
+[10.6.0]: https://github.com/subquery/subql/compare/node-core/10.5.1...node-core/10.6.0
 [10.5.1]: https://github.com/subquery/subql/compare/node-core/10.5.0...node-core/10.5.1
 [10.5.0]: https://github.com/subquery/subql/compare/node-core/10.4.1...node-core/10.5.0
 [10.4.1]: https://github.com/subquery/subql/compare/node-core/10.4.0...node-core/10.4.1

packages/node-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-core",
-  "version": "10.5.1",
+  "version": "10.6.0",
   "description": "Common node features that are agnostic to blockchains",
   "homepage": "https://github.com/subquery/subql",
   "repository": "github:subquery/subql",

packages/node-core/src/indexer/fetch.service.spec.ts

@@ -76,7 +76,7 @@ const nodeConfig = new NodeConfig({
 const getNetworkConfig = () =>
   ({
     dictionary: 'https://example.com',
-  } as IProjectNetworkConfig);
+  }) as IProjectNetworkConfig;
 
 const mockDs: BaseDataSource = {
   kind: 'mock/DataSource',
@@ -130,7 +130,7 @@ const getDictionaryService = () =>
     initDictionaries: () => {
       /* TODO */
     },
-  } as any as DictionaryService<any, any>);
+  }) as any as DictionaryService<any, any>;
 
 const getBlockDispatcher = () => {
   const inst = {
@@ -655,8 +655,8 @@ describe('Fetch Service', () => {
   });
 
   it('throws if the start block is greater than the chain latest height', async () => {
-    await expect(() => fetchService.init(1001)).rejects.toThrow(
-      `The startBlock of dataSources in your project manifest (1001) is higher than the current chain height (1000). Please adjust your startBlock to be less that the current chain height.`
+    await expect(() => fetchService.init(1002)).rejects.toThrow(
+      `The startBlock of dataSources in your project manifest (1002) is higher than the current chain height (1000). Please adjust your startBlock to be less that the current chain height.`
     );
   });
 
@@ -676,11 +676,31 @@ describe('Fetch Service', () => {
     (fetchService as any).dictionaryService.scopedDictionaryEntries = () => {
       return undefined;
     };
+    fetchService.bestHeight = 500;
     const dictionarySpy = jest.spyOn((fetchService as any).dictionaryService, 'scopedDictionaryEntries');
     await fetchService.init(10);
     expect(dictionarySpy).toHaveBeenCalledTimes(1);
     expect(spyOnEnqueueSequential).toHaveBeenCalledTimes(1);
 
     expect(enqueueBlocksSpy).toHaveBeenLastCalledWith([10, 11, 12, 13, 14, 15, 16, 17, 18, 19], 19);
   });
+
+  it(`doesn't use dictionary if processing near latest height`, async () => {
+    enableDictionary();
+    (fetchService as any).dictionaryService.scopedDictionaryEntries = () => {
+      return undefined;
+    };
+    fetchService.bestHeight = 500;
+    const dictionarySpy = jest.spyOn((fetchService as any).dictionaryService, 'scopedDictionaryEntries');
+    await fetchService.init(490);
+    expect(dictionarySpy).toHaveBeenCalledTimes(0);
+    expect(spyOnEnqueueSequential).toHaveBeenCalledTimes(1);
+
+    expect(enqueueBlocksSpy).toHaveBeenLastCalledWith([490, 491, 492, 493, 494, 495, 496, 497, 498, 499], 499);
+  });
+
+  it('fetch init when last processed height is same as', async () => {
+    // when last processed height is 1000, finalized height is 1000
+    await expect(fetchService.init(1001)).resolves.not.toThrow();
+  });
 });

packages/node-core/src/indexer/fetch.service.ts

@@ -102,10 +102,20 @@ export abstract class BaseFetchService<DS extends BaseDataSource, B extends IBlo
 
     await Promise.all([this.getFinalizedBlockHead(), this.getBestBlockHead()]);
 
-    if (startHeight > this.latestHeight()) {
-      throw new Error(
-        `The startBlock of dataSources in your project manifest (${startHeight}) is higher than the current chain height (${this.latestHeight()}). Please adjust your startBlock to be less that the current chain height.`
-      );
+    const chainLatestHeight = this.latestHeight();
+    if (startHeight > chainLatestHeight) {
+      // This is at init stage, lastProcessedHeight should be always - 1 from the startHeight in this case
+      // this is reverse calculated from projectService.nextProcessHeight()
+      // Alternative, we can expose async function getLastProcessedHeight() to ensure accuracy.
+      if (startHeight - 1 === chainLatestHeight) {
+        logger.warn(
+          `Project last processed height is same as current chain height (${chainLatestHeight}). Please ensure the RPC endpoint provider is behaving correctly.`
+        );
+      } else {
+        throw new Error(
+          `The startBlock of dataSources in your project manifest (${startHeight}) is higher than the current chain height (${chainLatestHeight}). Please adjust your startBlock to be less that the current chain height.`
+        );
+      }
     }
 
     this.schedulerRegistry.addInterval(
@@ -200,11 +210,24 @@ export abstract class BaseFetchService<DS extends BaseDataSource, B extends IBlo
       const latestHeight = this.latestHeight();
 
       if (this.blockDispatcher.freeSize < scaledBatchSize || startBlockHeight > latestHeight) {
+        if (this.blockDispatcher.freeSize < scaledBatchSize) {
+          logger.debug(
+            `Fetch service is waiting for free space in the block dispatcher queue, free size: ${this.blockDispatcher.freeSize}, scaledBatchSize: ${scaledBatchSize}`
+          );
+        }
+        if (startBlockHeight > latestHeight) {
+          logger.debug(
+            `Fetch service is waiting for new blocks, startBlockHeight: ${startBlockHeight}, latestHeight: ${latestHeight}`
+          );
+        }
         await delay(1);
         continue;
       }
 
-      if (startBlockHeight < this.latestFinalizedHeight) {
+      // This could be latestBestHeight, dictionary should never include finalized blocks
+      // TODO add buffer so dictionary not used when project synced
+      if (startBlockHeight < this.latestBestHeight - scaledBatchSize) {
+        // if (startBlockHeight < this.latestFinalizedHeight) {
         try {
           const dictionary = await this.dictionaryService.scopedDictionaryEntries(
             startBlockHeight,

packages/node-core/src/indexer/test.runner.spec.ts

@@ -147,4 +147,47 @@ describe('TestRunner', () => {
     expect(summary?.entityName).toBeUndefined();
     expect(summary?.failedAttributes[0]).toEqual(expect.stringMatching(/^Runtime Error:\nError: Test error/));
   });
+
+  it('gives a sufficient error for timestamps within MS of each other', async () => {
+    const expectedEntity = {
+      _name: 'Entity1',
+      id: '1',
+      timestamp: new Date(1000),
+    };
+    const actualEntity = {
+      _name: 'Entity1',
+      id: '1',
+      timestamp: new Date(1001),
+    };
+
+    const testMock = {
+      name: 'test1',
+      blockHeight: 1,
+      handler: 'handler1',
+      expectedEntities: [expectedEntity],
+      dependentEntities: [],
+    };
+
+    const indexBlock = jest.fn().mockResolvedValue(undefined);
+    const storeMock = {
+      get: jest.fn().mockResolvedValue(actualEntity),
+    };
+    (testRunner as any).storeService = {
+      getStore: () => storeMock,
+      setBlockHeight: jest.fn(),
+      storeCache: {flushCache: jest.fn().mockResolvedValue(undefined)},
+    } as any;
+
+    await testRunner.runTest(testMock, sandboxMock, indexBlock);
+
+    expect((testRunner as any).failedTests).toBe(1);
+
+    const summary = (testRunner as any).failedTestSummary;
+    expect(summary?.testName).toEqual(testMock.name);
+    expect(summary?.entityId).toEqual(expectedEntity.id);
+    expect(summary?.entityName).toEqual(expectedEntity._name);
+    expect(summary?.failedAttributes[0]).toEqual(
+      `\t\tattribute: "timestamp":\n\t\t\texpected: "1970-01-01T00:00:01.000Z"\n\t\t\tactual:   "1970-01-01T00:00:01.001Z"\n`
+    );
+  });
 });

packages/node-core/src/indexer/test.runner.ts

@@ -102,8 +102,15 @@ export class TestRunner<A, SA, B, DS> {
             const actualAttr = (actualEntity as Record<string, any>)[attr] ?? null;
 
             if (!isEqual(expectedAttr, actualAttr)) {
+              // Converts dates into a format so that ms is visible
+              const fmtValue = (value: any) => {
+                if (value instanceof Date) {
+                  return value.toISOString();
+                }
+                return value;
+              };
               failedAttributes.push(
-                `\t\tattribute: "${attr}":\n\t\t\texpected: "${expectedAttr}"\n\t\t\tactual:   "${actualAttr}"\n`
+                `\t\tattribute: "${attr}":\n\t\t\texpected: "${fmtValue(expectedAttr)}"\n\t\t\tactual:   "${fmtValue(actualAttr)}"\n`
               );
             }
           });

packages/node/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [4.6.5] - 2024-06-18
+### Fixed
+- Add missing chainType validation for local mode
+
 ## [4.6.4] - 2024-06-12
 ### Changed
 - Update `@subql/utils`
@@ -1224,7 +1228,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - bump @polkadot/api to 3.1.1
 
-[Unreleased]: https://github.com/subquery/subql/compare/node/4.6.4...HEAD
+[Unreleased]: https://github.com/subquery/subql/compare/node/4.6.5...HEAD
+[4.6.5]: https://github.com/subquery/subql/compare/node/4.6.4...node/4.6.5
 [4.6.4]: https://github.com/subquery/subql/compare/node/4.6.3...node/4.6.4
 [4.6.3]: https://github.com/subquery/subql/compare/node/4.6.2...node/4.6.3
 [4.6.2]: https://github.com/subquery/subql/compare/node/4.6.1...node/4.6.2

packages/node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/node",
-  "version": "4.6.4",
+  "version": "4.6.5",
   "description": "",
   "author": "Ian He",
   "license": "GPL-3.0",

packages/node/src/utils/project.ts

@@ -42,27 +42,27 @@ export async function getChainTypes(
   file: string,
 ): Promise<ChainTypes> {
   // If the project is load from local, we will direct load them
+  let raw: unknown;
   if (reader instanceof LocalReader) {
     return loadChainTypes(file, root) as ChainTypes;
   } else {
     // If it is stored in ipfs or other resources, we will use the corresponding reader to read the file
     // Because ipfs not provide extension of the file, it is difficult to determine its format
     // We will use yaml.load to try to load the script and parse them to supported chain types
-    // if it failed, we will give it another another attempt, and assume the script written in js
+    // if it failed, we will give it another attempt, and assume the script written in js
     // we will download it to a temp folder, and load them within sandbox
     const res = await reader.getFile(file);
     assert(res, `File ${file} not found`);
 
     let raw: unknown;
     try {
       raw = yaml.load(res);
-      return parseChainTypes(raw);
     } catch (e) {
       const chainTypesPath = await saveFile(reader, root, file, res);
       raw = loadChainTypesFromJs(chainTypesPath); //root not required, as it been packed in single js
-      return parseChainTypes(raw);
     }
   }
+  return parseChainTypes(raw);
 }
 
 export function loadChainTypes(file: string, projectRoot: string): unknown {

packages/query/CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- enable ts strict mode
+
 ## [2.12.1] - 2024-06-12
 ### Changed
 - Update `@subql/utils`

packages/query/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/query",
-  "version": "2.12.1",
+  "version": "2.12.2-0",
   "description": "",
   "bin": {
     "subql-query": "./bin/run"
@@ -70,5 +70,6 @@
     "@types/rimraf": "^3.0.2",
     "@types/yargs": "^16.0.4",
     "nodemon": "^2.0.15"
-  }
+  },
+  "stableVersion": "2.12.1"
 }

packages/query/src/configure/configure.module.ts

@@ -14,7 +14,7 @@ async function ensurePool(poolConfig: PoolConfig): Promise<Pool> {
   const pgPool = new Pool(poolConfig);
   try {
     await pgPool.connect();
-  } catch (e) {
+  } catch (e: any) {
     if (JSON.stringify(e.message).includes(CONNECTION_SSL_ERROR_REGEX)) {
       poolConfig.ssl = undefined;
       return ensurePool(poolConfig);
@@ -49,7 +49,7 @@ export class ConfigureModule {
           }
 
           return sslConfig;
-        } catch (e) {
+        } catch (e: any) {
           getLogger('db config').error(e);
           throw e;
         }