Update indexer flags for deployment (#2274)

* Update indexer flags for deployment * Update changelog * Limit max workers to 5 * Update store cache async flag * Fix workers flag * update logging for disableStoreCacheAsync, fix typo * update types * add comment, fix typo --------- Co-authored-by: bz888 <[email protected]>
subquery · Feb 27, 2024 · 0e5ddff · 0e5ddff
1 parent 9211fbb
commit 0e5ddff
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 20 deletions.
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Publish command output flag not working (#2270)
 
+### Changed
+- Update deployemnt flags to match managed service (#2274)
+
 ## [4.2.7] - 2024-02-23
 ### Changed
 - Bump axios from 0.27.0 to 0.28.0 (#2262)

diff --git a/packages/cli/src/commands/deployment/deploy.ts b/packages/cli/src/commands/deployment/deploy.ts
@@ -17,6 +17,7 @@ import {
   splitEndpoints,
   updateDeployment,
 } from '../../controller/deploy-controller';
+import {IndexerAdvancedOpts, QueryAdvancedOpts, V3DeploymentIndexerType} from '../../types';
 import {addV, checkToken, promptWithDefaultValues, valueOrPrompt} from '../../utils';
 
 const ACCESS_TOKEN_PATH = path.resolve(process.env.HOME, '.subql/SUBQL_ACCESS_TOKEN');
@@ -39,7 +40,20 @@ export default class Deploy extends Command {
     indexerBatchSize: Flags.integer({description: 'Enter batchSize from 1 to 30', required: false}),
     indexerSubscription: Flags.boolean({description: 'Enable Indexer subscription', required: false}),
     disableHistorical: Flags.boolean({description: 'Disable Historical Data', required: false}),
-    indexerWorkers: Flags.integer({description: 'Enter worker threads from 1 to 30', required: false}),
+    indexerUnfinalized: Flags.boolean({
+      description: 'Index unfinalized blocks (requires Historical to be enabled)',
+      required: false,
+    }),
+    indexerStoreCacheThreshold: Flags.integer({
+      description: 'The number of items kept in the cache before flushing',
+      required: false,
+    }),
+    disableIndexerStoreCacheAsync: Flags.boolean({
+      description: 'If enabled the store cache will flush data asynchronously relative to indexing data.',
+      required: false,
+    }),
+    indexerWorkers: Flags.integer({description: 'Enter worker threads from 1 to 5', required: false, max: 5}),
+
     //query flags
     queryUnsafe: Flags.boolean({description: 'Enable indexer unsafe', required: false}),
     querySubscription: Flags.boolean({description: 'Enable Query subscription', required: false}),
@@ -80,20 +94,22 @@ export default class Deploy extends Command {
       endpoint = await promptWithDefaultValues(cli, 'Enter endpoint', undefined, null, true);
     }
 
-    const queryAD = {
+    const queryAD: QueryAdvancedOpts = {
       unsafe: flags.queryUnsafe,
       subscription: flags.querySubscription,
       queryTimeout: flags.queryTimeout,
       maxConnection: flags.queryMaxConnection,
-      Aggregate: flags.queryAggregate,
+      aggregate: flags.queryAggregate,
     };
 
-    const indexerAD = {
+    const indexerAD: IndexerAdvancedOpts = {
       unsafe: flags.indexerUnsafe,
       batchSize: flags.indexerBatchSize,
       subscription: flags.indexerSubscription,
       historicalData: !flags.disableHistorical,
-      workers: flags.indexerWorkers,
+      unfinalizedBlocks: flags.indexerUnfinalized,
+      storeCacheThreshold: flags.indexerStoreCacheThreshold,
+      disableStoreCacheAsync: !flags.disableIndexerStoreCacheAsync,
     };
 
     if (!dict) {
@@ -147,7 +163,7 @@ export default class Deploy extends Command {
       }
     }
     const projectInfo = await projectsInfo(authToken, org, projectName, ROOT_API_URL_PROD, flags.type);
-    const chains = [
+    const chains: V3DeploymentIndexerType[] = [
       {
         cid: ipfsCID,
         dictEndpoint: dict,
@@ -159,6 +175,14 @@ export default class Deploy extends Command {
       },
     ];
 
+    if (flags.indexerWorkers) {
+      chains[0].extraParams = {
+        workers: {
+          num: flags.indexerWorkers,
+        },
+      };
+    }
+
     if (projectInfo !== undefined) {
       await updateDeployment(
         org,
@@ -185,6 +209,15 @@ export default class Deploy extends Command {
         chains,
         ROOT_API_URL_PROD
       ).catch((e) => this.error(e));
+
+      // Managed service does some interesting things: `disableStoreCacheAsync: true` -> `--store-cache-async=true`, we are just inverting it to resolve logging confusion.
+      if (
+        deploymentOutput.configuration.config.indexer &&
+        deploymentOutput.configuration.config.indexer.disableStoreCacheAsync === false
+      ) {
+        deploymentOutput.configuration.config.indexer.disableStoreCacheAsync = true;
+      }
+
       this.log(`Project: ${deploymentOutput.projectKey}
       \nStatus: ${chalk.blue(deploymentOutput.status)}
       \nDeploymentID: ${deploymentOutput.id}

diff --git a/packages/cli/src/controller/deploy-controller.spec.ts b/packages/cli/src/controller/deploy-controller.spec.ts
@@ -139,6 +139,7 @@ describe('CLI deploy, delete, promote', () => {
     const validator = await ipfsCID_validate(projectSpec.ipfs, testAuth, ROOT_API_URL_DEV);
     expect(validator.valid).toBe(true);
   });
+
   it('to throw error for invalid ipfsCID', async () => {
     await expect(ipfsCID_validate('fake', testAuth, ROOT_API_URL_DEV)).rejects.toThrow(
       'Failed to validate IPFS CID: fake is not a valid subquery deployment id!'
@@ -152,6 +153,7 @@ describe('CLI deploy, delete, promote', () => {
       'https://api.subquery.network/sq/subquery/polkadot-dictionary'
     );
   });
+
   it('reDeploy to Hosted Service', async () => {
     const {ipfs, org, projectName, type} = projectSpec;
     const newIPFS = 'QmbKvrzwSmzTZi5jrhEpa6yDDHQXRURi5S4ztLgJLpBxAi';

diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts
@@ -11,18 +11,21 @@ export interface ProjectSpecBase {
 }
 
 export interface QueryAdvancedOpts {
-  unsafe?: boolean | undefined;
-  subscription?: boolean | undefined;
-  queryTimeout?: number | undefined;
-  maxConnection?: number | undefined;
-  Aggregate?: boolean | undefined;
+  unsafe?: boolean;
+  subscription?: boolean;
+  queryTimeout?: number;
+  maxConnection?: number;
+  aggregate?: boolean;
 }
 export interface IndexerAdvancedOpts {
-  unsafe?: boolean | undefined;
-  batchSize?: number | undefined;
-  subscription?: boolean | undefined;
-  historicalData?: boolean | undefined;
-  workers?: number | undefined;
+  unsafe?: boolean;
+  batchSize?: number;
+  subscription?: boolean;
+  historicalData?: boolean;
+  unfinalizedBlocks?: boolean;
+  proofOfIndex?: boolean;
+  storeCacheThreshold?: number;
+  disableStoreCacheAsync?: boolean; // Managed service does some interesting things: `disableStoreCacheAsync: true` -> `--store-cache-async=true` and default is false
 }
 
 export type ProjectSpecV0_0_1 = ProjectSpecBase;
@@ -85,9 +88,7 @@ export interface DeploymentDataType {
   configuration: {
     config: {
       query: Record<string, unknown>;
-      indexer: {
-        batchSize: number;
-      };
+      indexer: IndexerAdvancedOpts;
       role: string;
       chainId: string;
     };
@@ -138,4 +139,9 @@ export interface V3DeploymentIndexerType {
   indexerAdvancedSettings: {
     indexer: IndexerAdvancedOpts;
   };
+  extraParams?: {
+    workers?: {
+      num?: number;
+    };
+  };
 }
diff --git a/packages/node-core/src/yargs.ts b/packages/node-core/src/yargs.ts
@@ -170,7 +170,7 @@ export function yargsBuilder<
               },
               'store-cache-async': {
                 demandOption: false,
-                describe: 'If enabled the store cache will flush data asyncronously relative to indexing data',
+                describe: 'If enabled the store cache will flush data asynchronously relative to indexing data',
                 type: 'boolean',
               },
               'store-flush-interval': {