From 52bc5de4709e369912e1050134a329f421a2ac50 Mon Sep 17 00:00:00 2001 From: Greg Tatum Date: Thu, 7 Jan 2021 11:52:29 -0600 Subject: [PATCH] Add support for ndjson benchmark output --- src/config.ts | 4 +- src/extract.ts | 69 ++++++++++++++++++++++++++ src/write.ts | 13 +++-- test/data/extract/ndjson_output.ndjson | 2 + test/extract.ts | 22 ++++++++ 5 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 test/data/extract/ndjson_output.ndjson diff --git a/src/config.ts b/src/config.ts index 04a8c980a..e34f8e106 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,7 @@ import { promises as fs } from 'fs'; import * as os from 'os'; import * as path from 'path'; -export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2'; +export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2' | 'ndjson'; export interface Config { name: string; tool: ToolType; @@ -24,7 +24,7 @@ export interface Config { maxItemsInChart: number | null; } -export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2']; +export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2', 'ndjson']; const RE_UINT = /^\d+$/; function validateToolType(tool: string): asserts tool is ToolType { diff --git a/src/extract.ts b/src/extract.ts index 84a1e9b8b..35153591b 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -8,6 +8,7 @@ export interface BenchmarkResult { range?: string; unit: string; extra?: string; + biggerIsBetter?: boolean; } interface GitHubUser { @@ -415,6 +416,71 @@ function extractCatch2Result(output: string): BenchmarkResult[] { return ret; } +function ensureString(json: { [key: string]: unknown }, key: string): string { + const value = json[key]; + if (typeof value === 'string') { + return value; + } + throw new Error(`Expected the JSON to have a string at the key "${key}"`); +} + +function ensureNumber(json: { [key: string]: unknown }, key: string): number { + const value = json[key]; + if (typeof value === 'number') { + return value; + } + throw new Error(`Expected the JSON to have a number at the key "${key}"`); +} + +/** + * Process results that use the internal format for this tool, but using newline + * delimited json. This format is friendly for appending results onto a file. + * + * http://ndjson.org/ + */ +function extractNdjsonResult(output: string): BenchmarkResult[] { + // Split the newlines, process the json, and then validate the fields. + return output + .split(/\r?\n/g) + .filter(n => n) + .map(line => { + // Process the JSON. + let json; + try { + json = JSON.parse(line); + } catch (err) { + throw new Error( + `A line from the ndjson file could not be parsed.\nLine: ${line}\nError: ${err.message}`, + ); + } + if (!json || typeof json !== 'object') { + throw new Error(`A line form the ndjson was not an object. Line: ${line}`); + } + + const biggerIsBetter = json.biggerIsBetter; + if (typeof biggerIsBetter !== 'boolean') { + throw new Error( + `ndjson output must include a "biggerIsBetter" field as it cannot be inferred from the test suite.`, + ); + } + + // Validate the data provided in the entry. + const result: BenchmarkResult = { + name: ensureString(json, 'name'), + value: ensureNumber(json, 'value'), + unit: ensureString(json, 'unit'), + biggerIsBetter, + }; + if ('range' in json) { + result.range = ensureString(json, 'range'); + } + if ('extra' in json) { + result.extra = ensureString(json, 'extra'); + } + return result; + }); +} + export async function extractResult(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); const { tool } = config; @@ -439,6 +505,9 @@ export async function extractResult(config: Config): Promise { case 'catch2': benches = extractCatch2Result(output); break; + case 'ndjson': + benches = extractNdjsonResult(output); + break; default: throw new Error(`FATAL: Unexpected tool: '${tool}'`); } diff --git a/src/write.ts b/src/write.ts index 2c7834422..ae4a9fae5 100644 --- a/src/write.ts +++ b/src/write.ts @@ -56,7 +56,7 @@ async function addIndexHtmlIfNeeded(dir: string) { console.log('Created default index.html at', indexHtml); } -function biggerIsBetter(tool: ToolType): boolean { +function biggerIsBetter(tool: ToolType, result: BenchmarkResult): boolean { switch (tool) { case 'cargo': return false; @@ -70,6 +70,13 @@ function biggerIsBetter(tool: ToolType): boolean { return false; case 'catch2': return false; + case 'ndjson': { + const { biggerIsBetter } = result; + if (biggerIsBetter === undefined) { + throw new Error('ndjson is assumed to have a biggerIsBetter field.'); + } + return biggerIsBetter; + } } } @@ -90,7 +97,7 @@ function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number continue; } - const ratio = biggerIsBetter(curSuite.tool) + const ratio = biggerIsBetter(curSuite.tool, current) ? prev.value / current.value // e.g. current=100, prev=200 : current.value / prev.value; // e.g. current=200, prev=100 @@ -160,7 +167,7 @@ function buildComment(benchName: string, curSuite: Benchmark, prevSuite: Benchma const prev = prevSuite.benches.find(i => i.name === current.name); if (prev) { - const ratio = biggerIsBetter(curSuite.tool) + const ratio = biggerIsBetter(curSuite.tool, current) ? prev.value / current.value // e.g. current=100, prev=200 : current.value / prev.value; diff --git a/test/data/extract/ndjson_output.ndjson b/test/data/extract/ndjson_output.ndjson new file mode 100644 index 000000000..28ed9a42d --- /dev/null +++ b/test/data/extract/ndjson_output.ndjson @@ -0,0 +1,2 @@ +{ "extra": "100 samples\n76353 iterations", "name": "Fibonacci 10", "range": "± 0", "unit": "ns", "value": 0, "biggerIsBetter": true } +{ "extra": "100 samples\n75814 iterations", "name": "Fibonacci 20", "range": "± 0", "unit": "ns", "value": 1, "biggerIsBetter": true } diff --git a/test/extract.ts b/test/extract.ts index f09d5611f..60e50e675 100644 --- a/test/extract.ts +++ b/test/extract.ts @@ -220,6 +220,28 @@ describe('extractResult()', function() { }, ], }, + { + tool: 'ndjson', + file: 'ndjson_output.ndjson', + expected: [ + { + extra: '100 samples\n76353 iterations', + name: 'Fibonacci 10', + range: '± 0', + unit: 'ns', + value: 0, + biggerIsBetter: true, + }, + { + extra: '100 samples\n75814 iterations', + name: 'Fibonacci 20', + range: '± 0', + unit: 'ns', + value: 1, + biggerIsBetter: true, + }, + ], + }, ]; for (const test of normalCases) {