[WIP] modular CLI (#2456)

* draft fix module import working draft Update packages/cli/src/modulars/moduleLoader.ts Co-authored-by: Scott Twiname <[email protected]> address comments * apply types for cosmos and eth module * remove network types * update inquirer-autocomplete-prompt version * Update packages/cli/src/modulars/moduleLoader.ts Co-authored-by: Scott Twiname <[email protected]> * Update packages/cli/src/modulars/moduleLoader.ts Co-authored-by: Scott Twiname <[email protected]> * add runtime check * relocate test to cosmos package * fix test * fix tests, changelog * fix test1 * Update dependencies * Fix remove ethers * Remove deprecate flag used in test --------- Co-authored-by: Scott Twiname <[email protected]>
subquery · Jul 3, 2024 · 803917b · 803917b
1 parent 2ea8cf6
commit 803917b
Show file tree

Hide file tree

Showing 34 changed files with 495 additions and 688 deletions.
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - The IPFS CID for multi-chain requires the directory CID.
 
+### Changed
+- Breaking Change: Removed common network packages from dependencies, only import required module from local/global when using. (#2456)
+
 ## [4.15.0] - 2024-06-21
 ### Changed
 - Update `@subql/utils`, `@subql/common` and other dependencies

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -10,22 +10,13 @@
   "dependencies": {
     "@oclif/core": "^2.15.0",
     "@subql/common": "workspace:*",
-    "@subql/common-algorand": "^3.4.0",
-    "@subql/common-concordium": "^3.6.0",
-    "@subql/common-cosmos": "^4.3.0",
-    "@subql/common-ethereum": "^3.8.3",
-    "@subql/common-near": "^3.5.0",
-    "@subql/common-stellar": "^3.5.0",
-    "@subql/common-substrate": "workspace:*",
     "@subql/utils": "workspace:*",
-    "algosdk": "^1.19.0",
     "boxen": "5.1.2",
     "cli-ux": "^6.0.9",
     "ejs": "^3.1.10",
-    "ethers": "^5.7.0",
     "fuzzy": "^0.1.3",
     "inquirer": "^8.2.0",
-    "inquirer-autocomplete-prompt": "^1.4.0",
+    "inquirer-autocomplete-prompt": "^2.0.0",
     "json5": "^2.2.3",
     "node-fetch": "2.6.7",
     "rimraf": "^3.0.2",
@@ -44,6 +35,13 @@
     "yaml-loader": "^0.6.0"
   },
   "devDependencies": {
+    "@subql/common-algorand": "^4.0.0",
+    "@subql/common-concordium": "^4.0.1",
+    "@subql/common-cosmos": "^5.0.0",
+    "@subql/common-ethereum": "^4.1.0",
+    "@subql/common-near": "^4.0.0",
+    "@subql/common-stellar": "^4.0.0",
+    "@subql/common-substrate": "workspace:*",
     "@types/ejs": "^3.1.0",
     "@types/inquirer": "^8.2.0",
     "@types/node": "^18.16.10",
@@ -54,6 +52,7 @@
     "eslint": "^8.8.0",
     "eslint-config-oclif": "^4.0.0",
     "eslint-config-oclif-typescript": "^1.0.2",
+    "ethers": "^5.7.0",
     "oclif": "^3.15.0"
   },
   "engines": {

diff --git a/packages/cli/src/commands/codegen/generate.ts b/packages/cli/src/commands/codegen/generate.ts
@@ -3,26 +3,25 @@
 
 import fs, {lstatSync} from 'fs';
 import path from 'path';
-import {EventFragment, FunctionFragment} from '@ethersproject/abi/src.ts/fragments';
+import type {EventFragment, FunctionFragment} from '@ethersproject/abi/src.ts/fragments';
 import {Command, Flags} from '@oclif/core';
-import {DEFAULT_MANIFEST, DEFAULT_TS_MANIFEST, extensionIsTs} from '@subql/common';
-import {SubqlRuntimeDatasource as EthereumDs} from '@subql/types-ethereum';
-import {parseContractPath} from 'typechain';
+import {DEFAULT_MANIFEST, DEFAULT_TS_MANIFEST, extensionIsTs, NETWORK_FAMILY} from '@subql/common';
+import type {SubqlRuntimeDatasource as EthereumDs} from '@subql/types-ethereum';
 import {
   constructMethod,
   filterExistingMethods,
   filterObjectsByStateMutability,
   generateHandlers,
   generateManifestTs,
   generateManifestYaml,
-  getAbiInterface,
   getManifestData,
   ManifestExtractor,
   prepareAbiDirectory,
   prepareInputFragments,
   tsExtractor,
   yamlExtractor,
 } from '../../controller/generate-controller';
+import {loadDependency} from '../../modulars';
 import {extractFromTs} from '../../utils';
 
 export interface SelectedMethod {
@@ -102,8 +101,8 @@ export default class Generate extends Command {
     } else {
       this.error('Invalid manifest path');
     }
-
-    const abiName = parseContractPath(abiPath).name;
+    const ethModule = loadDependency(NETWORK_FAMILY.ethereum);
+    const abiName = ethModule.parseContractPath(abiPath).name;
 
     if (fs.existsSync(path.join(root, 'src/mappings/', `${abiName}Handlers.ts`))) {
       throw new Error(`file: ${abiName}Handlers.ts already exists`);
@@ -113,7 +112,7 @@ export default class Generate extends Command {
     const abiFileName = path.basename(abiPath);
 
     // fragments from abi
-    const abiInterface = getAbiInterface(root, abiFileName);
+    const abiInterface = ethModule.getAbiInterface(root, abiFileName);
     const eventsFragments = abiInterface.events;
     const functionFragments = filterObjectsByStateMutability(abiInterface.functions);
 

diff --git a/packages/cli/src/commands/codegen/index.ts b/packages/cli/src/commands/codegen/index.ts
@@ -55,7 +55,7 @@ export default class Codegen extends Command {
     try {
       await codegen(root, manifests);
     } catch (err) {
-      console.error(err.message);
+      console.error(err.message, err.cause);
       process.exit(1);
     }
   }

diff --git a/packages/cli/src/commands/publish.ts b/packages/cli/src/commands/publish.ts
@@ -41,7 +41,13 @@ export default class Publish extends Command {
     }
 
     const fileToCidMap = await uploadToIpfs(fullPaths, authToken.trim(), multichainManifestPath, flags.ipfs).catch(
-      (e) => this.error(e)
+      (e) => {
+        // log further cause from error
+        if (e.cause) {
+          console.error(e.cause);
+        }
+        return this.error(e);
+      }
     );
 
     await Promise.all(

diff --git a/packages/cli/src/controller/codegen-controller.ts b/packages/cli/src/controller/codegen-controller.ts
@@ -2,81 +2,37 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import path from 'path';
-import {DEFAULT_MANIFEST, getManifestPath, getSchemaPath, loadFromJsonOrYaml} from '@subql/common';
-import {isCustomDs as isCustomConcordiumDs, isRuntimeDs as isRuntimeConcordiumDs} from '@subql/common-concordium';
 import {
-  isCustomCosmosDs,
-  isRuntimeCosmosDs,
-  generateProto,
-  tempProtoDir,
-  validateCosmosManifest,
-  ProjectManifestImpls as CosmosManifest,
-  generateCosmwasm,
-  CosmosCustomModuleImpl,
-} from '@subql/common-cosmos';
-import {
-  isCustomDs as isCustomEthereumDs,
-  isRuntimeDs as isRuntimeEthereumDs,
-  generateAbis,
-} from '@subql/common-ethereum';
-import {isCustomDs as isCustomNearDs, isRuntimeDs as isRuntimeNearDs} from '@subql/common-near';
-import {isCustomDs as isCustomStellarDs, isRuntimeDs as isRuntimeStellarDs} from '@subql/common-stellar';
-import {isCustomDs as isCustomSubstrateDs, SubstrateCustomDataSource} from '@subql/common-substrate';
-import {
-  RuntimeDatasourceTemplate as SubstrateDsTemplate,
-  CustomDatasourceTemplate as SubstrateCustomDsTemplate,
-} from '@subql/types';
-import {
-  RuntimeDatasourceTemplate as ConcordiumDsTemplate,
-  CustomDatasourceTemplate as ConcordiumCustomDsTemplate,
-} from '@subql/types-concordium';
-import {TemplateBase} from '@subql/types-core';
-import {
-  RuntimeDatasourceTemplate as CosmosDsTemplate,
-  CustomDatasourceTemplate as CosmosCustomDsTemplate,
-} from '@subql/types-cosmos';
-import {
-  RuntimeDatasourceTemplate as EthereumDsTemplate,
-  CustomDatasourceTemplate as EthereumCustomDsTemplate,
-  SubqlRuntimeDatasource as EthereumDs,
+  DEFAULT_MANIFEST,
+  getManifestPath,
+  getProjectNetwork,
+  getSchemaPath,
+  loadFromJsonOrYaml,
+  NETWORK_FAMILY,
+} from '@subql/common';
+import type {SubstrateCustomDatasource} from '@subql/types';
+import {BaseTemplateDataSource, ProjectManifestV1_0_0, TemplateBase} from '@subql/types-core';
+import type {
   SubqlCustomDatasource as EthereumCustomDs,
+  SubqlRuntimeDatasource as EthereumDs,
 } from '@subql/types-ethereum';
-import {
-  RuntimeDatasourceTemplate as NearDsTemplate,
-  CustomDatasourceTemplate as NearCustomDsTemplate,
-} from '@subql/types-near';
-import {
-  RuntimeDatasourceTemplate as StellarDsTemplate,
-  CustomDatasourceTemplate as StellarCustomDsTemplate,
-} from '@subql/types-stellar';
 import {
   getAllEntitiesRelations,
+  getAllEnums,
   getAllJsonObjects,
-  setJsonObjectType,
   getTypeByScalarName,
   GraphQLEntityField,
-  GraphQLJsonFieldType,
   GraphQLEntityIndex,
-  getAllEnums,
+  GraphQLJsonFieldType,
+  setJsonObjectType,
 } from '@subql/utils';
-import {upperFirst, uniq, uniqBy} from 'lodash';
-import {renderTemplate, prepareDirPath} from '../utils';
+import {uniq, uniqBy, upperFirst} from 'lodash';
+import {loadDependency} from '../modulars';
+import {prepareDirPath, renderTemplate} from '../utils';
 
-export type TemplateKind =
-  | SubstrateDsTemplate
-  | SubstrateCustomDsTemplate
-  | CosmosDsTemplate
-  | CosmosCustomDsTemplate
-  | EthereumDsTemplate
-  | EthereumCustomDsTemplate
-  | NearDsTemplate
-  | NearCustomDsTemplate
-  | StellarDsTemplate
-  | StellarCustomDsTemplate
-  | ConcordiumDsTemplate
-  | ConcordiumCustomDsTemplate;
+export type TemplateKind = BaseTemplateDataSource;
 
-export type DatasourceKind = SubstrateCustomDataSource | EthereumDs | EthereumCustomDs;
+export type DatasourceKind = SubstrateCustomDatasource | EthereumDs | EthereumCustomDs;
 
 const MODEL_TEMPLATE_PATH = path.resolve(__dirname, '../template/model.ts.ejs');
 const MODELS_INDEX_TEMPLATE_PATH = path.resolve(__dirname, '../template/models-index.ts.ejs');
@@ -233,12 +189,11 @@ export async function codegen(projectPath: string, fileNames: string[] = [DEFAUL
   await prepareDirPath(modelDir, true);
   await prepareDirPath(interfacesPath, false);
   const plainManifests = fileNames.map((fileName) => {
-    const project = loadFromJsonOrYaml(getManifestPath(projectPath, fileName));
-    return project as {
-      specVersion: string;
-      templates?: TemplateKind[];
-      dataSources: DatasourceKind[];
+    const project = loadFromJsonOrYaml(getManifestPath(projectPath, fileName)) as ProjectManifestV1_0_0 & {
+      networkFamily: NETWORK_FAMILY;
     };
+    project.networkFamily = getProjectNetwork(project);
+    return project;
   });
 
   const expectKeys = ['datasources', 'templates'];
@@ -278,14 +233,23 @@ export async function codegen(projectPath: string, fileNames: string[] = [DEFAUL
     datasources = datasources.concat(customDatasources);
   }
 
-  const chainTypes = getChaintypes(plainManifests);
-
-  if (chainTypes.length) {
-    await generateProto(chainTypes, projectPath, prepareDirPath, renderTemplate, upperFirst, tempProtoDir);
+  const cosmosManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.cosmos);
+  if (cosmosManifests.length > 0) {
+    const cosmosModule = loadDependency(NETWORK_FAMILY.cosmos);
+    await cosmosModule.projectCodegen(
+      plainManifests,
+      projectPath,
+      prepareDirPath,
+      renderTemplate,
+      upperFirst,
+      datasources
+    );
+  }
+  const ethManifests = plainManifests.filter((m) => m.networkFamily === NETWORK_FAMILY.ethereum);
+  if (ethManifests.length >= 0 || !!datasources.find((d) => d?.assets)) {
+    const ethModule = loadDependency(NETWORK_FAMILY.ethereum);
+    await ethModule.generateAbis(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);
   }
-  await generateCosmwasm(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);
-
-  await generateAbis(datasources, projectPath, prepareDirPath, upperFirst, renderTemplate);
 
   if (exportTypes.interfaces || exportTypes.models || exportTypes.enums || exportTypes.datasources) {
     try {
@@ -301,15 +265,6 @@ export async function codegen(projectPath: string, fileNames: string[] = [DEFAUL
   }
 }
 
-export function getChaintypes(
-  manifest: {templates?: TemplateKind[]; dataSources: DatasourceKind[]}[]
-): Map<string, CosmosCustomModuleImpl>[] {
-  return manifest
-    .filter((m) => validateCosmosManifest(m))
-    .map((m) => (m as CosmosManifest).network.chaintypes)
-    .filter((value) => value && Object.keys(value).length !== 0);
-}
-
 export async function generateSchemaModels(projectPath: string, schemaPath: string): Promise<void> {
   const modelDir = path.join(projectPath, MODEL_ROOT_DIR);
   const interfacesPath = path.join(projectPath, TYPE_ROOT_DIR, `interfaces.ts`);
@@ -390,7 +345,7 @@ export async function generateModels(projectPath: string, schema: string): Promi
 export async function generateDatasourceTemplates(projectPath: string, templates: TemplateKind[]): Promise<void> {
   const props = templates.map((t) => ({
     name: (t as TemplateBase).name,
-    args: hasParameters(t) ? 'Record<string, unknown>' : undefined,
+    args: 'Record<string, unknown>',
   }));
 
   const propsWithoutDuplicates = uniqBy(props, (prop) => `${prop.name}-${prop.args}`);
@@ -406,19 +361,3 @@ export async function generateDatasourceTemplates(projectPath: string, templates
   }
   console.log(`* Datasource template constructors generated !`);
 }
-
-function hasParameters(t: TemplateKind): boolean {
-  return (
-    isRuntimeCosmosDs(t as CosmosDsTemplate) ||
-    isCustomCosmosDs(t as CosmosDsTemplate) ||
-    isRuntimeEthereumDs(t as EthereumDsTemplate) ||
-    isCustomEthereumDs(t as EthereumDsTemplate) ||
-    isCustomSubstrateDs(t as SubstrateDsTemplate) ||
-    isRuntimeNearDs(t as NearDsTemplate) ||
-    isCustomNearDs(t as NearDsTemplate) ||
-    isRuntimeStellarDs(t as StellarDsTemplate) ||
-    isCustomStellarDs(t as StellarDsTemplate) ||
-    isRuntimeConcordiumDs(t as ConcordiumDsTemplate) ||
-    isCustomConcordiumDs(t as ConcordiumDsTemplate)
-  );
-}