From e107877c0e04c9857467607773b95854472880f5 Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Wed, 10 Apr 2024 15:10:16 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20add=20benchmarker=20utility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- devTools/benchmarker/README.md | 11 ++ devTools/benchmarker/benchmarker.ts | 154 ++++++++++++++++++++++++++++ devTools/benchmarker/tsconfig.json | 8 ++ tsconfig.json | 3 + 4 files changed, 176 insertions(+) create mode 100644 devTools/benchmarker/README.md create mode 100644 devTools/benchmarker/benchmarker.ts create mode 100644 devTools/benchmarker/tsconfig.json diff --git a/devTools/benchmarker/README.md b/devTools/benchmarker/README.md new file mode 100644 index 00000000000..6382d31bf47 --- /dev/null +++ b/devTools/benchmarker/README.md @@ -0,0 +1,11 @@ +A simple benchmarking utility (elbow grease required) + +## Using this tool + +1. Add this project to the tsconfig of the project you would like to benchmark e.g. the [baker](baker/tsconfig.json) +2. Import the Benchmarker class somewhere into that project and instantiate it +3. Create if-blocks around sections of code you would like to benchmark e.g. `if (benchmark.flags.validateGdoc) { await this.validateGdoc()}` and the benchmarker tool will automatically add it to the list of flags to permutate through +4. Call `benchmarker.benchmark` with `{name, callback}` where `name` is a string and `callback` is the function that will call the code you're testing (all the references to `benchmark.flags` must be included in the callstack of this callback) +5. Run `yarn buildTsc` to build your code with the benchmarker +6. Run `node itsJustJavascript/path/to/benchmark/entry/point.js` and wait for the permutations to run +7. Review the results in `devTools/benchmarker/results` diff --git a/devTools/benchmarker/benchmarker.ts b/devTools/benchmarker/benchmarker.ts new file mode 100644 index 00000000000..2b23b210cbf --- /dev/null +++ b/devTools/benchmarker/benchmarker.ts @@ -0,0 +1,154 @@ +import fs from "fs/promises" + +async function exampleFunctionToProfile() { + let sum = 0 + if (benchmarker.flags.sum) { + for (let i = 0; i < 1000000000; i++) { + sum += i + } + } + + if (benchmarker.flags.promise) { + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + + return sum +} + +export class Benchmarker { + flags: Record + accessedProperties = new Set() + + constructor() { + const bm = this + this.flags = new Proxy( + {}, + { + get(target: any, prop: string, receiver: any) { + // don't add toJSON - it's used by JSON.stringify when writing the report + if (prop !== "toJSON") { + bm.accessedProperties.add(prop) + } + return Reflect.get(target, prop, receiver) + }, + } + ) + } + + async init({ + name, + callback, + }: { + name: string + callback: () => void | Promise + }) { + console.log("Running initial performance test (all flags disabled)") + const start = performance.now() + await callback() + const end = performance.now() + console.log("Time taken", end - start) + await this.writeReport({ + name, + start, + end, + flags: Object.fromEntries( + [...this.accessedProperties].map((key) => [key, false]) + ), + }) + console.log( + "Enabling all flags: ", + [...this.accessedProperties].join(", ") + ) + for (const key of this.accessedProperties) { + this.flags[key] = true + } + } + + async runPermutation({ + name, + flags, + callback, + }: { + name: string + flags: Record + callback: () => void | Promise + }) { + console.log(`Running test with flags: ${JSON.stringify(flags)}`) + for (const key of Object.keys(flags)) { + this.flags[key] = flags[key] + } + const start = performance.now() + await callback() + const end = performance.now() + console.log("Time taken", end - start) + await this.writeReport({ name, start, end, flags }) + } + + async writeReport({ + name, + end, + start, + flags, + }: { + name: string + start: number + end: number + flags: Record + }) { + await fs.mkdir("./devTools/benchmarker/results", { recursive: true }) + const serializedFlags = Object.entries(flags) + .map(([key, value]) => `${key}-${value}`) + .join("-") + await fs.writeFile( + `./devTools/benchmarker/results/${name}-${serializedFlags}.json`, + JSON.stringify({ time: end - start, flags }) + ) + } + + async benchmark({ + name, + callback, + }: { + name: string + callback: () => void | Promise + }) { + await this.init({ name, callback }) + + // generate permutations of flags + const flags = Object.keys(this.flags) + const permutations = this.generatePermutations(flags) + for (const permutation of permutations) { + await this.runPermutation({ + name, + flags: permutation, + callback, + }) + } + } + + generatePermutations(flags: string[]): Record[] { + const permutations: Record[] = [] + // start at 1 to skip the all-false permutation which we've already done in the init function + for (let i = 1; i < 2 ** flags.length; i++) { + const permutation: Record = {} + for (let j = 0; j < flags.length; j++) { + // the way this works is: + // 1 << j generates a number with a single bit set at position j + // i & (1 << j) checks if that bit is set in the number i + // if it is, we set the flag to true, otherwise false + permutation[flags[j]] = Boolean(i & (1 << j)) + } + permutations.push(permutation) + } + return permutations + } +} + +const benchmarker = new Benchmarker() + +benchmarker.benchmark({ + name: "profiling-example-function", + callback: async () => { + await exampleFunctionToProfile() + }, +}) diff --git a/devTools/benchmarker/tsconfig.json b/devTools/benchmarker/tsconfig.json new file mode 100644 index 00000000000..14e7d5dc0ad --- /dev/null +++ b/devTools/benchmarker/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfigs/tsconfig.base.json", + "compilerOptions": { + "outDir": "../../itsJustJavascript/devTools/benchmarker", + "rootDir": "." + }, + "references": [] +} diff --git a/tsconfig.json b/tsconfig.json index cf0e6095847..7b32840351f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -54,6 +54,9 @@ }, { "path": "./devTools/navigationTest" + }, + { + "path": "./devTools/benchmarker" } ] }