implement example apps benchmarking (#117)

---------

Co-authored-by: Victor Berchet <[email protected]>

benchmarking/.gitignore

@@ -0,0 +1 @@
+results/

benchmarking/README.md

@@ -0,0 +1,17 @@
+# Benchmarking
+
+This directory contains a script for running full end to end benchmarks against the example applications.
+
+What the script does:
+
+- takes all the example applications from the [`./examples` directory](../examples/)
+  (excluding the ones specified in the `exampleAppsNotToBenchmark` set in [`./src/cloudflare.ts`](./src/cloudflare.ts))
+- in parallel for each application:
+  - builds the application by running its `build:worker` script
+  - deploys the application to production (with `wrangler deploy`)
+  - takes the production deployment url
+  - benchmarks the application's response time by fetching from the deployment url a number of times
+
+> [!note]
+> This is the first cut at benchmarking our solution, later we can take the script in this directory,
+> generalize it and make it more reusable if we want

benchmarking/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@opennextjs-cloudflare/benchmarking",
+  "private": true,
+  "type": "module",
+  "devDependencies": {
+    "tsx": "catalog:",
+    "@tsconfig/strictest": "catalog:",
+    "@types/node": "catalog:",
+    "ora": "^8.1.0"
+  },
+  "scripts": {
+    "benchmark": "tsx src/index.ts"
+  }
+}

benchmarking/src/benchmarking.ts

@@ -0,0 +1,132 @@
+import nodeTimesPromises from "node:timers/promises";
+import nodeFsPromises from "node:fs/promises";
+import nodePath from "node:path";
+import { getPercentile } from "./utils";
+
+export type FetchBenchmark = {
+  iterationsMs: number[];
+  averageMs: number;
+  p90Ms: number;
+};
+
+export type BenchmarkingResults = {
+  name: string;
+  path: string;
+  fetchBenchmark: FetchBenchmark;
+}[];
+
+type BenchmarkFetchOptions = {
+  numberOfIterations?: number;
+  maxRandomDelayMs?: number;
+  fetch: (deploymentUrl: string) => Promise<Response>;
+};
+
+const defaultOptions: Required<Omit<BenchmarkFetchOptions, "fetch">> = {
+  numberOfIterations: 20,
+  maxRandomDelayMs: 15_000,
+};
+
+/**
+ * Benchmarks the response time of an application end-to-end by:
+ *  - building the application
+ *  - deploying it
+ *  - and fetching from it (multiple times)
+ *
+ * @param options.build function implementing how the application is to be built
+ * @param options.deploy function implementing how the application is deployed (returning the url of the deployment)
+ * @param options.fetch function indicating how to fetch from the application (in case a specific route needs to be hit, cookies need to be applied, etc...)
+ * @returns the benchmarking results for the application
+ */
+export async function benchmarkApplicationResponseTime({
+  build,
+  deploy,
+  fetch,
+}: {
+  build: () => Promise<void>;
+  deploy: () => Promise<string>;
+  fetch: (deploymentUrl: string) => Promise<Response>;
+}): Promise<FetchBenchmark> {
+  await build();
+  const deploymentUrl = await deploy();
+  return benchmarkFetch(deploymentUrl, { fetch });
+}
+
+/**
+ * Benchmarks a fetch operation by running it multiple times and computing the average time (in milliseconds) such fetch operation takes.
+ *
+ * @param url The url to fetch from
+ * @param options options for the benchmarking
+ * @returns the computed average alongside all the single call times
+ */
+async function benchmarkFetch(url: string, options: BenchmarkFetchOptions): Promise<FetchBenchmark> {
+  const benchmarkFetchCall = async () => {
+    const preTimeMs = performance.now();
+    const resp = await options.fetch(url);
+    const postTimeMs = performance.now();
+
+    if (!resp.ok) {
+      throw new Error(`Error: Failed to fetch from "${url}"`);
+    }
+
+    return postTimeMs - preTimeMs;
+  };
+
+  const resolvedOptions = { ...defaultOptions, ...options };
+
+  const iterationsMs = await Promise.all(
+    new Array(resolvedOptions.numberOfIterations).fill(null).map(async () => {
+      // let's add a random delay before we make the fetch
+      await nodeTimesPromises.setTimeout(Math.round(Math.random() * resolvedOptions.maxRandomDelayMs));
+
+      return benchmarkFetchCall();
+    })
+  );
+
+  const averageMs = iterationsMs.reduce((time, sum) => sum + time) / iterationsMs.length;
+
+  const p90Ms = getPercentile(iterationsMs, 90);
+
+  return {
+    iterationsMs,
+    averageMs,
+    p90Ms,
+  };
+}
+
+/**
+ * Saves benchmarking results in a local json file
+ *
+ * @param results the benchmarking results to save
+ * @returns the path to the created json file
+ */
+export async function saveResultsToDisk(results: BenchmarkingResults): Promise<string> {
+  const date = new Date();
+
+  const fileName = `${toSimpleDateString(date)}.json`;
+
+  const outputFile = nodePath.resolve(`./results/${fileName}`);
+
+  await nodeFsPromises.mkdir(nodePath.dirname(outputFile), { recursive: true });
+
+  const resultStr = JSON.stringify(results, null, 2);
+  await nodeFsPromises.writeFile(outputFile, resultStr);
+
+  return outputFile;
+}
+
+/**
+ * Takes a date and coverts it to a simple format that can be used as
+ * a filename (which is human readable and doesn't contain special
+ * characters)
+ *
+ * The format being: `YYYY-MM-DD_hh-mm-ss`
+ *
+ * @param date the date to convert
+ * @returns a string representing the date
+ */
+function toSimpleDateString(date: Date): string {
+  const isoString = date.toISOString();
+  const isoDate = isoString.split(".")[0]!;
+
+  return isoDate.replace("T", "_").replaceAll(":", "-");
+}

benchmarking/src/cloudflare.ts

@@ -0,0 +1,123 @@
+import nodeFsPromises from "node:fs/promises";
+import nodeFs from "node:fs";
+import nodePath from "node:path";
+import nodeChildProcess from "node:child_process";
+
+await ensureWranglerSetup();
+
+/**
+ * Collects name and absolute paths of apps (in this repository) that we want to benchmark
+ *
+ * @returns Array of objects containing the app's name and absolute path
+ */
+export async function collectAppPathsToBenchmark(): Promise<
+  {
+    name: string;
+    path: string;
+  }[]
+> {
+  const allExampleNames = await nodeFsPromises.readdir("../examples");
+
+  /**
+   * Example applications that we don't want to benchmark
+   *
+   * Currently we only want to skip the `vercel-commerce` example, and that's simply
+   * because it requires a shopify specific setup and secrets.
+   */
+  const exampleAppsNotToBenchmark = new Set(["vercel-commerce"]);
+
+  const examplePaths = allExampleNames
+    .filter((exampleName) => !exampleAppsNotToBenchmark.has(exampleName))
+    .map((exampleName) => ({
+      name: exampleName,
+      path: nodePath.resolve(`../examples/${exampleName}`),
+    }));
+
+  return examplePaths;
+}
+
+/**
+ * Builds an application using their "build:worker" script
+ * (an error is thrown if the application doesn't have such a script)
+ *
+ * @param dir Path to the application to build
+ */
+export async function buildApp(dir: string): Promise<void> {
+  const packageJsonPath = `${dir}/package.json`;
+  if (!nodeFs.existsSync(packageJsonPath)) {
+    throw new Error(`Error: package.json for app at "${dir}" not found`);
+  }
+
+  const packageJsonContent = JSON.parse(await nodeFsPromises.readFile(packageJsonPath, "utf8"));
+
+  const buildScript = "build:worker";
+
+  if (!packageJsonContent.scripts?.[buildScript]) {
+    throw new Error(`Error: package.json for app at "${dir}" does not include a "${buildScript}" script`);
+  }
+
+  const command = `pnpm ${buildScript}`;
+
+  return new Promise((resolve, reject) => {
+    nodeChildProcess.exec(command, { cwd: dir }, (error) => {
+      if (error) {
+        return reject(error);
+      }
+      return resolve();
+    });
+  });
+}
+
+/**
+ * Deploys a built application using wrangler
+ *
+ * @param dir Path to the application to build
+ * @returns the url of the deployed application
+ */
+export async function deployBuiltApp(dir: string): Promise<string> {
+  return new Promise<string>((resolve, reject) => {
+    nodeChildProcess.exec("pnpm exec wrangler deploy", { cwd: dir }, (error, stdout) => {
+      if (error) {
+        return reject(error);
+      }
+
+      const deploymentUrl = stdout.match(/\bhttps:\/\/(?:[a-zA-Z0-9.\-])*\.workers\.dev\b/)?.[0];
+
+      if (!deploymentUrl) {
+        return reject(new Error(`Could not obtain a deployment url for app at "${dir}"`));
+      }
+
+      return resolve(deploymentUrl);
+    });
+  });
+}
+
+/**
+ * Makes sure that everything is set up so that wrangler can actually deploy the applications.
+ * This means that:
+ *  - the user has logged in
+ *  - if they have more than one account they have set a CLOUDFLARE_ACCOUNT_ID env variable
+ */
+async function ensureWranglerSetup(): Promise<void> {
+  return new Promise((resolve, reject) => {
+    nodeChildProcess.exec("pnpm dlx wrangler whoami", (error, stdout) => {
+      if (error) {
+        return reject(error);
+      }
+
+      if (stdout.includes("You are not authenticated")) {
+        reject(new Error("Please log in using wrangler by running `pnpm dlx wrangler login`"));
+      }
+
+      if (!(process.env as Record<string, unknown>)["CLOUDFLARE_ACCOUNT_ID"]) {
+        reject(
+          new Error(
+            "Please set the CLOUDFLARE_ACCOUNT_ID environment variable to the id of the account you want to use to deploy the applications"
+          )
+        );
+      }
+
+      return resolve();
+    });
+  });
+}

benchmarking/src/index.ts

@@ -0,0 +1,42 @@
+import nodeTimesPromises from "node:timers/promises";
+import * as cloudflare from "./cloudflare";
+import { benchmarkApplicationResponseTime, BenchmarkingResults, saveResultsToDisk } from "./benchmarking";
+import { parallelRunWithSpinner } from "./utils";
+
+const appPathsToBenchmark = await cloudflare.collectAppPathsToBenchmark();
+
+const benchmarkingResults: BenchmarkingResults = await parallelRunWithSpinner(
+  "Benchmarking Apps",
+  appPathsToBenchmark.map(({ name, path }, i) => async () => {
+    await nodeTimesPromises.setTimeout(i * 1_000);
+    const fetchBenchmark = await benchmarkApplicationResponseTime({
+      build: async () => cloudflare.buildApp(path),
+      deploy: async () => cloudflare.deployBuiltApp(path),
+      fetch,
+    });
+
+    return {
+      name,
+      path,
+      fetchBenchmark,
+    };
+  })
+);
+
+console.log();
+
+const outputFile = await saveResultsToDisk(benchmarkingResults);
+
+console.log(`The benchmarking results have been written in ${outputFile}`);
+
+console.log("\n\nSummary: ");
+const summary = benchmarkingResults.map(({ name, fetchBenchmark }) => ({
+  name,
+  "average fetch duration (ms)": Math.round(fetchBenchmark.averageMs),
+  "90th percentile (ms)": Math.round(fetchBenchmark.p90Ms),
+}));
+console.table(summary);
+
+console.log();
+
+process.exit(0);

benchmarking/src/utils.ts

@@ -0,0 +1,61 @@
+import ora from "ora";
+
+/**
+ * Runs a list of operations in parallel while presenting a loading spinner with some text
+ *
+ * @param spinnerText The text to add to the spinner
+ * @param operations The operations to run
+ * @returns The operations results
+ */
+export async function parallelRunWithSpinner<T>(
+  spinnerText: string,
+  operations: (() => Promise<T>)[]
+): Promise<T[]> {
+  const spinner = ora({
+    discardStdin: false,
+    hideCursor: false,
+  }).start();
+
+  let doneCount = 0;
+
+  const updateSpinnerText = () => {
+    doneCount++;
+    spinner.text = `${spinnerText} (${doneCount}/${operations.length})`;
+  };
+
+  updateSpinnerText();
+
+  const results = await Promise.all(
+    operations.map(async (operation) => {
+      const result = await operation();
+      updateSpinnerText();
+      return result;
+    })
+  );
+
+  spinner.stop();
+
+  return results;
+}
+
+/**
+ * Gets a specific percentile for a given set of numbers
+ *
+ * @param data the data which percentile value needs to be computed
+ * @param percentile the requested percentile (a number between 0 and 100)
+ * @returns the computed percentile
+ */
+export function getPercentile(data: number[], percentile: number): number {
+  if (Number.isNaN(percentile) || percentile < 0 || percentile > 100) {
+    throw new Error(`A percentile needs to be between 0 and 100, found: ${percentile}`);
+  }
+
+  data = data.sort((a, b) => a - b);
+
+  const rank = (percentile / 100) * (data.length - 1);
+
+  const rankInt = Math.floor(rank);
+  const rankFract = rank - rankInt;
+
+  return Math.round(data[rankInt]! + rankFract * (data[rankInt + 1]! - data[rankInt]!));
+}