From 0e5ddff5d299dc0fb16c9a874652415e0ce7cdcc Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 28 Feb 2024 12:07:24 +1300 Subject: [PATCH] Update indexer flags for deployment (#2274) * Update indexer flags for deployment * Update changelog * Limit max workers to 5 * Update store cache async flag * Fix workers flag * update logging for disableStoreCacheAsync, fix typo * update types * add comment, fix typo --------- Co-authored-by: bz888 --- packages/cli/CHANGELOG.md | 3 ++ .../cli/src/commands/deployment/deploy.ts | 45 ++++++++++++++++--- .../src/controller/deploy-controller.spec.ts | 2 + packages/cli/src/types.ts | 32 +++++++------ packages/node-core/src/yargs.ts | 2 +- 5 files changed, 64 insertions(+), 20 deletions(-) diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 58d1cbc04d..f33b3b5d65 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Publish command output flag not working (#2270) +### Changed +- Update deployemnt flags to match managed service (#2274) + ## [4.2.7] - 2024-02-23 ### Changed - Bump axios from 0.27.0 to 0.28.0 (#2262) diff --git a/packages/cli/src/commands/deployment/deploy.ts b/packages/cli/src/commands/deployment/deploy.ts index c5aa28e29a..08e3ebb179 100644 --- a/packages/cli/src/commands/deployment/deploy.ts +++ b/packages/cli/src/commands/deployment/deploy.ts @@ -17,6 +17,7 @@ import { splitEndpoints, updateDeployment, } from '../../controller/deploy-controller'; +import {IndexerAdvancedOpts, QueryAdvancedOpts, V3DeploymentIndexerType} from '../../types'; import {addV, checkToken, promptWithDefaultValues, valueOrPrompt} from '../../utils'; const ACCESS_TOKEN_PATH = path.resolve(process.env.HOME, '.subql/SUBQL_ACCESS_TOKEN'); @@ -39,7 +40,20 @@ export default class Deploy extends Command { indexerBatchSize: Flags.integer({description: 'Enter batchSize from 1 to 30', required: false}), indexerSubscription: Flags.boolean({description: 'Enable Indexer subscription', required: false}), disableHistorical: Flags.boolean({description: 'Disable Historical Data', required: false}), - indexerWorkers: Flags.integer({description: 'Enter worker threads from 1 to 30', required: false}), + indexerUnfinalized: Flags.boolean({ + description: 'Index unfinalized blocks (requires Historical to be enabled)', + required: false, + }), + indexerStoreCacheThreshold: Flags.integer({ + description: 'The number of items kept in the cache before flushing', + required: false, + }), + disableIndexerStoreCacheAsync: Flags.boolean({ + description: 'If enabled the store cache will flush data asynchronously relative to indexing data.', + required: false, + }), + indexerWorkers: Flags.integer({description: 'Enter worker threads from 1 to 5', required: false, max: 5}), + //query flags queryUnsafe: Flags.boolean({description: 'Enable indexer unsafe', required: false}), querySubscription: Flags.boolean({description: 'Enable Query subscription', required: false}), @@ -80,20 +94,22 @@ export default class Deploy extends Command { endpoint = await promptWithDefaultValues(cli, 'Enter endpoint', undefined, null, true); } - const queryAD = { + const queryAD: QueryAdvancedOpts = { unsafe: flags.queryUnsafe, subscription: flags.querySubscription, queryTimeout: flags.queryTimeout, maxConnection: flags.queryMaxConnection, - Aggregate: flags.queryAggregate, + aggregate: flags.queryAggregate, }; - const indexerAD = { + const indexerAD: IndexerAdvancedOpts = { unsafe: flags.indexerUnsafe, batchSize: flags.indexerBatchSize, subscription: flags.indexerSubscription, historicalData: !flags.disableHistorical, - workers: flags.indexerWorkers, + unfinalizedBlocks: flags.indexerUnfinalized, + storeCacheThreshold: flags.indexerStoreCacheThreshold, + disableStoreCacheAsync: !flags.disableIndexerStoreCacheAsync, }; if (!dict) { @@ -147,7 +163,7 @@ export default class Deploy extends Command { } } const projectInfo = await projectsInfo(authToken, org, projectName, ROOT_API_URL_PROD, flags.type); - const chains = [ + const chains: V3DeploymentIndexerType[] = [ { cid: ipfsCID, dictEndpoint: dict, @@ -159,6 +175,14 @@ export default class Deploy extends Command { }, ]; + if (flags.indexerWorkers) { + chains[0].extraParams = { + workers: { + num: flags.indexerWorkers, + }, + }; + } + if (projectInfo !== undefined) { await updateDeployment( org, @@ -185,6 +209,15 @@ export default class Deploy extends Command { chains, ROOT_API_URL_PROD ).catch((e) => this.error(e)); + + // Managed service does some interesting things: `disableStoreCacheAsync: true` -> `--store-cache-async=true`, we are just inverting it to resolve logging confusion. + if ( + deploymentOutput.configuration.config.indexer && + deploymentOutput.configuration.config.indexer.disableStoreCacheAsync === false + ) { + deploymentOutput.configuration.config.indexer.disableStoreCacheAsync = true; + } + this.log(`Project: ${deploymentOutput.projectKey} \nStatus: ${chalk.blue(deploymentOutput.status)} \nDeploymentID: ${deploymentOutput.id} diff --git a/packages/cli/src/controller/deploy-controller.spec.ts b/packages/cli/src/controller/deploy-controller.spec.ts index 0ced703370..a36544ae69 100644 --- a/packages/cli/src/controller/deploy-controller.spec.ts +++ b/packages/cli/src/controller/deploy-controller.spec.ts @@ -139,6 +139,7 @@ describe('CLI deploy, delete, promote', () => { const validator = await ipfsCID_validate(projectSpec.ipfs, testAuth, ROOT_API_URL_DEV); expect(validator.valid).toBe(true); }); + it('to throw error for invalid ipfsCID', async () => { await expect(ipfsCID_validate('fake', testAuth, ROOT_API_URL_DEV)).rejects.toThrow( 'Failed to validate IPFS CID: fake is not a valid subquery deployment id!' @@ -152,6 +153,7 @@ describe('CLI deploy, delete, promote', () => { 'https://api.subquery.network/sq/subquery/polkadot-dictionary' ); }); + it('reDeploy to Hosted Service', async () => { const {ipfs, org, projectName, type} = projectSpec; const newIPFS = 'QmbKvrzwSmzTZi5jrhEpa6yDDHQXRURi5S4ztLgJLpBxAi'; diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index c42e4b1e9c..ed2c70c72b 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -11,18 +11,21 @@ export interface ProjectSpecBase { } export interface QueryAdvancedOpts { - unsafe?: boolean | undefined; - subscription?: boolean | undefined; - queryTimeout?: number | undefined; - maxConnection?: number | undefined; - Aggregate?: boolean | undefined; + unsafe?: boolean; + subscription?: boolean; + queryTimeout?: number; + maxConnection?: number; + aggregate?: boolean; } export interface IndexerAdvancedOpts { - unsafe?: boolean | undefined; - batchSize?: number | undefined; - subscription?: boolean | undefined; - historicalData?: boolean | undefined; - workers?: number | undefined; + unsafe?: boolean; + batchSize?: number; + subscription?: boolean; + historicalData?: boolean; + unfinalizedBlocks?: boolean; + proofOfIndex?: boolean; + storeCacheThreshold?: number; + disableStoreCacheAsync?: boolean; // Managed service does some interesting things: `disableStoreCacheAsync: true` -> `--store-cache-async=true` and default is false } export type ProjectSpecV0_0_1 = ProjectSpecBase; @@ -85,9 +88,7 @@ export interface DeploymentDataType { configuration: { config: { query: Record; - indexer: { - batchSize: number; - }; + indexer: IndexerAdvancedOpts; role: string; chainId: string; }; @@ -138,4 +139,9 @@ export interface V3DeploymentIndexerType { indexerAdvancedSettings: { indexer: IndexerAdvancedOpts; }; + extraParams?: { + workers?: { + num?: number; + }; + }; } diff --git a/packages/node-core/src/yargs.ts b/packages/node-core/src/yargs.ts index 1e4275759d..ab27a7293a 100644 --- a/packages/node-core/src/yargs.ts +++ b/packages/node-core/src/yargs.ts @@ -170,7 +170,7 @@ export function yargsBuilder< }, 'store-cache-async': { demandOption: false, - describe: 'If enabled the store cache will flush data asyncronously relative to indexing data', + describe: 'If enabled the store cache will flush data asynchronously relative to indexing data', type: 'boolean', }, 'store-flush-interval': {