Add SDK client library

This adds a basic sketch of the SDK client library. I mostly mirrored the Python SDK method structure, but would like to diverge from that to make things more ergonomic. Most of the complexity here isn't really in the SDK itself, but rather in the testing framework around it. Making circuit creation requests mockable with Nock required a MITM proxy and some fairly sophisticated request parsing to handle multipart form payloads and the non-determinism of the gzipped tarballs. The circuit creation implementation was also a little tricky in terms of supporting both browsers and node. With these more difficult parts figured out, it shouldn't be too bad to refactor things and clean them up. Note that the docstrings right now are pure fluff, I barely cleaned up what chatgpt output. I wanted to get something there so that I can test/develop the documentation build and then I'll clean these up and write real docs as I refactor the SDK's API. Merges #49
Sindri-Labs · Jan 15, 2024 · 36b4c10 · 36b4c10
1 parent f934942
commit 36b4c10
Show file tree

Hide file tree

Showing 35 changed files with 73,689 additions and 2,597 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 .npmrc
+.http-mitm-proxy/
 /dist/
 /node_modules/
diff --git a/ava.config.mjs b/ava.config.mjs
@@ -17,8 +17,8 @@ export default {
   timeout: ["dryrun", "record", "update", "wild"].includes(
     process.env.NOCK_BACK_MODE ?? "lockdown",
   )
-    ? "60s"
-    : "10s",
+    ? "240s"
+    : "60s",
   // Use child processes instead of threads because notch isn't thread safe.
   workerThreads: false,
 };
diff --git a/package.json b/package.json
@@ -19,7 +19,12 @@
     "cryptography",
     "crypto"
   ],
-  "files": ["dist/", "sindri-manifest.json", "src/", "templates/"],
+  "files": [
+    "dist/",
+    "sindri-manifest.json",
+    "src/",
+    "templates/"
+  ],
   "main": "dist/lib/index.js",
   "module": "dist/lib/index.mjs",
   "bin": {
@@ -52,7 +57,9 @@
     "test:fast": "ava",
     "test:record": "NOCK_BACK_MODE=update yarn test",
     "test:watch": "NODE_ENV=development NOCK_BACK_MODE=dryrun nodemon --watch src/ --watch test/ --ext js,cjs,mjs,ts,cts,mts --exec 'tsup --silent && ava'",
-    "type-check": "tsc --noEmit"
+    "type-check": "tsc --noEmit",
+    "/***** Hooks *****/": "",
+    "postinstall": "patch-package"
   },
   "repository": {
     "type": "git",
@@ -68,39 +75,52 @@
     "axios": "^1.6.2",
     "commander": "^11.1.0",
     "env-paths": "^2.2.1",
-    "form-data": "^4.0.0",
+    "formdata-node": "^6.0.3",
+    "gzip-js": "^0.3.2",
     "ignore-walk": "^6.0.4",
     "jsonschema": "^1.4.1",
     "lodash": "^4.17.21",
     "nunjucks": "^3.2.4",
+    "patch-package": "^8.0.0",
     "pino": "^8.16.2",
     "pino-pretty": "^10.2.3",
+    "postinstall-postinstall": "^2.1.0",
     "rc": "^1.2.8",
     "tar": "^6.2.0",
+    "tar-js": "^0.3.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {
     "@ava/typescript": "^4.1.0",
     "@commander-js/extra-typings": "^11.1.0",
     "@tsconfig/node18": "^18.2.2",
+    "@types/get-port": "^4.2.0",
+    "@types/gzip-js": "^0.3.5",
     "@types/ignore-walk": "^4.0.3",
     "@types/lodash": "^4.14.202",
     "@types/node": "^20.9.1",
     "@types/nunjucks": "^3.2.6",
+    "@types/proxy": "^1.0.4",
     "@types/tar": "^6.1.10",
+    "@types/tar-js": "^0.3.5",
     "@typescript-eslint/eslint-plugin": "^6.11.0",
     "@typescript-eslint/parser": "^6.11.0",
     "ava": "^6.0.1",
     "esbuild": "^0.19.11",
     "eslint": "^8.53.0",
     "eslint-config-prettier": "^9.0.0",
     "eslint-plugin-prettier": "^5.0.1",
+    "get-port": "^7.0.0",
+    "http-mitm-proxy": "^1.1.0",
+    "make-synchronous": "^1.0.0",
+    "mockdate": "^3.0.5",
     "nock": "^13.4.0",
-    "nock-puppeteer": "^14.4.1",
     "nodemon": "^3.0.2",
     "openapi-typescript-codegen": "^0.25.0",
+    "parse-multipart-data": "^1.5.0",
     "prettier": "^3.1.0",
     "puppeteer": "^21.7.0",
+    "rollup": "^4.9.5",
     "tsup": "^7.3.0",
     "tsx": "^4.7.0",
     "type-fest": "^4.8.2",

diff --git a/patches/http-mitm-proxy+1.1.0.patch b/patches/http-mitm-proxy+1.1.0.patch
@@ -0,0 +1,13 @@
+diff --git a/node_modules/http-mitm-proxy/dist/lib/proxy.js b/node_modules/http-mitm-proxy/dist/lib/proxy.js
+index e896595..8623f9f 100644
+--- a/node_modules/http-mitm-proxy/dist/lib/proxy.js
++++ b/node_modules/http-mitm-proxy/dist/lib/proxy.js
+@@ -361,7 +361,7 @@ class Proxy {
+         function makeConnection(port) {
+             const conn = net_1.default.connect({
+                 port,
+-                host: "0.0.0.0",
++                host: "localhost",
+                 allowHalfOpen: true,
+             }, () => {
+                 const connectKey = `${conn.localPort}:${conn.remotePort}`;
diff --git a/src/cli/config.ts b/src/cli/config.ts
@@ -1,108 +1,7 @@
-import fs from "fs";
-import path from "path";
-
 import { Command } from "@commander-js/extra-typings";
-import envPaths from "env-paths";
-import { cloneDeep, merge } from "lodash";
-import { z } from "zod";
-
-import { logger, print } from "cli/logging";
-import { OpenAPI } from "lib/api";
-
-const paths = envPaths("sindri", {
-  suffix: "",
-});
-const configPath = path.join(paths.config, "sindri.conf.json");
-
-const ConfigSchema = z.object({
-  auth: z
-    .nullable(
-      z.object({
-        apiKey: z.string(),
-        apiKeyId: z.string(),
-        apiKeyName: z.string(),
-        baseUrl: z.string().url(),
-        teamId: z.number(),
-        teamSlug: z.string(),
-      }),
-    )
-    .default(null),
-});
-
-type ConfigSchema = z.infer<typeof ConfigSchema>;
-
-const defaultConfig: ConfigSchema = ConfigSchema.parse({});
-
-const loadConfig = (): ConfigSchema => {
-  if (fs.existsSync(configPath)) {
-    logger.debug(`Loading config from "${configPath}".`);
-    try {
-      const configFileContents: string = fs.readFileSync(configPath, {
-        encoding: "utf-8",
-      });
-      const loadedConfig = ConfigSchema.parse(JSON.parse(configFileContents));
-      logger.debug("Config loaded successfully.");
-      return loadedConfig;
-    } catch (error) {
-      logger.warn(
-        `The config schema in "${configPath}" is invalid and will not be used.\n` +
-          `To remove it and start fresh, run:\n    rm ${configPath}`,
-      );
-      logger.debug(error);
-    }
-  }
-  logger.debug(
-    `Config file "${configPath}" does not exist, initializing default config.`,
-  );
-  return cloneDeep(defaultConfig);
-};
-
-export class Config {
-  protected _config!: ConfigSchema;
-  protected static instance: Config;
-
-  constructor() {
-    if (!Config.instance) {
-      this._config = loadConfig();
-      Config.instance = this;
-      // Prepare API the client with the loaded credentials.
-      if (this._config.auth) {
-        OpenAPI.BASE = this._config.auth.baseUrl;
-        OpenAPI.TOKEN = this._config.auth.apiKey;
-      }
-    }
-    return Config.instance;
-  }
-
-  get auth(): ConfigSchema["auth"] {
-    return cloneDeep(this._config.auth);
-  }
-
-  get config(): ConfigSchema {
-    return cloneDeep(this._config);
-  }
-
-  update(configData: Partial<ConfigSchema>) {
-    // Merge and validate the configs.
-    logger.debug("Merging in config update:");
-    logger.debug(configData);
-    const newConfig: ConfigSchema = cloneDeep(this._config);
-    merge(newConfig, configData);
-    this._config = ConfigSchema.parse(newConfig);
-
-    // Create the directory if it doesn't exist.
-    const directory = path.dirname(configPath);
-    if (!fs.existsSync(directory)) {
-      fs.mkdirSync(directory, { recursive: true });
-    }
 
-    // Write out the new config.
-    logger.debug(`Writing merged config to "${configPath}":`, this._config);
-    fs.writeFileSync(configPath, JSON.stringify(this._config, null, 2), {
-      encoding: "utf-8",
-    });
-  }
-}
+import { Config } from "lib/config";
+import { print } from "lib/logging";
 
 export const configListCommand = new Command()
   .name("list")

diff --git a/src/cli/deploy.ts b/src/cli/deploy.ts
@@ -1,16 +1,17 @@
+import { Blob } from "buffer";
 import { existsSync, readFileSync } from "fs";
 import path from "path";
 import process from "process";
 
 import { Command } from "@commander-js/extra-typings";
-import FormData from "form-data";
+import { FormData } from "formdata-node";
 import walk from "ignore-walk";
 import tar from "tar";
 
-import { Config } from "cli/config";
-import { logger } from "cli/logging";
 import { findFileUpwards } from "cli/utils";
 import { ApiError, CircuitsService, CircuitStatus } from "lib/api";
+import { Config } from "lib/config";
+import { logger } from "lib/logging";
 
 export const deployCommand = new Command()
   .name("deploy")
@@ -103,7 +104,7 @@ export const deployCommand = new Command()
           // Always exclude `.git` subdirectories.
           !/(^|\/)\.git(\/|$)/.test(file),
       );
-    // Alows include the `sindri.json` file.
+    // Always include the `sindri.json` file.
     const sindriJsonFilename = path.basename(sindriJsonPath);
     if (!files.includes(sindriJsonFilename)) {
       files.push(sindriJsonFilename);
@@ -115,23 +116,23 @@ export const deployCommand = new Command()
     );
     formData.append(
       "files",
-      tar
-        .c(
-          {
-            gzip: true,
-            onwarn: (code: string, message: string) => {
-              logger.warn(`While creating tarball: ${code} - ${message}`);
+      new Blob([
+        tar
+          .c(
+            {
+              gzip: true,
+              onwarn: (code: string, message: string) => {
+                logger.warn(`While creating tarball: ${code} - ${message}`);
+              },
+              prefix: `${circuitName}/`,
+              sync: true,
             },
-            prefix: `${circuitName}/`,
-            sync: true,
-          },
-          files,
-        )
-        // @ts-expect-error - @types/tar doesn't handle the `sync` option correctly.
-        .read(),
-      {
-        filename: tarballFilename,
-      },
+            files,
+          )
+          // @ts-expect-error - @types/tar doesn't handle the `sync` option correctly.
+          .read(),
+      ]),
+      tarballFilename,
     );
 
     // Attach the tags to the form data.

diff --git a/src/cli/index.ts b/src/cli/index.ts
@@ -3,15 +3,16 @@ import { argv, exit } from "process";
 
 import { Command } from "@commander-js/extra-typings";
 
-import { Config, configCommand } from "cli/config";
+import { configCommand } from "cli/config";
 import { initCommand } from "cli/init";
 import { deployCommand } from "cli/deploy";
 import { lintCommand } from "cli/lint";
-import { logger } from "cli/logging";
 import { loginCommand } from "cli/login";
 import { logoutCommand } from "cli/logout";
 import { whoamiCommand } from "cli/whoami";
 import { loadPackageJson } from "cli/utils";
+import { Config } from "lib/config";
+import { logger } from "lib/logging";
 
 export const program = new Command()
   .name("sindri")

diff --git a/src/cli/init.ts b/src/cli/init.ts
@@ -6,8 +6,8 @@ import process from "process";
 import { Command } from "@commander-js/extra-typings";
 import { confirm, input, select } from "@inquirer/prompts";
 
-import { logger } from "cli/logging";
 import { scaffoldDirectory } from "cli/utils";
+import { logger } from "lib/logging";
 
 export const initCommand = new Command()
   .name("init")

diff --git a/src/cli/lint.ts b/src/cli/lint.ts
@@ -6,8 +6,8 @@ import { Command } from "@commander-js/extra-typings";
 import type { Schema } from "jsonschema";
 import { Validator as JsonValidator } from "jsonschema";
 
-import { logger } from "cli/logging";
 import { findFileUpwards, loadSindriManifestJsonSchema } from "cli/utils";
+import { logger } from "lib/logging";
 
 export const lintCommand = new Command()
   .name("lint")

diff --git a/src/cli/logging.ts b/src/cli/logging.ts
diff --git a/src/cli/login.ts b/src/cli/login.ts
@@ -9,15 +9,15 @@ import {
   select,
 } from "@inquirer/prompts";
 
-import { Config } from "cli/config";
-import { logger } from "cli/logging";
 import {
   ApiError,
   AuthorizationService,
   InternalService,
   OpenAPI,
   TokenService,
 } from "lib/api";
+import { Config } from "lib/config";
+import { logger } from "lib/logging";
 
 export const loginCommand = new Command()
   .name("login")

diff --git a/src/cli/logout.ts b/src/cli/logout.ts
@@ -1,9 +1,9 @@
 import { Command } from "@commander-js/extra-typings";
 import { confirm } from "@inquirer/prompts";
 
-import { Config } from "cli/config";
-import { logger } from "cli/logging";
 import { AuthorizationService } from "lib/api";
+import { Config } from "lib/config";
+import { logger } from "lib/logging";
 
 export const logoutCommand = new Command()
   .name("logout")

diff --git a/src/cli/utils.ts b/src/cli/utils.ts
@@ -7,7 +7,7 @@ import type { Schema } from "jsonschema";
 import nunjucks from "nunjucks";
 import type { PackageJson } from "type-fest";
 
-import { logger } from "cli/logging";
+import { logger } from "lib/logging";
 
 const currentFilePath = fileURLToPath(import.meta.url);
 const currentDirectoryPath = path.dirname(currentFilePath);

diff --git a/src/cli/whoami.ts b/src/cli/whoami.ts
@@ -2,9 +2,9 @@ import process from "process";
 
 import { Command } from "@commander-js/extra-typings";
 
-import { Config } from "cli/config";
-import { logger, print } from "cli/logging";
 import { ApiError, InternalService } from "lib/api";
+import { Config } from "lib/config";
+import { logger, print } from "lib/logging";
 
 export const whoamiCommand = new Command()
   .name("whoami")