diff --git a/bin/cli.js b/bin/cli.js index 35d8b59..55fcb76 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -15,7 +15,7 @@ program .version(require('./get-version')) .usage('[options]') .option('-s, --schema [schema]', 'JSON string or a Path/Url pointing to an open rpc schema') - .option('-r, --reporters ', 'Use the specified reporter [console] [json] [empty]. Can be a comma separated list of reporters.') + .option('-r, --reporters ', 'Use the specified reporter [console] [console-rule] [console-streaming] [json] [empty]. Can be a comma separated list of reporters.') .option('-t, --transport ', 'Use the specified transport [http]') .option('--skip ', 'Methods to skip. Comma separated list of method names') .option('--only ', 'Methods to only run. Comma separated list of method names') diff --git a/src/coverage.test.ts b/src/coverage.test.ts index f55fe3b..ac8ffe9 100644 --- a/src/coverage.test.ts +++ b/src/coverage.test.ts @@ -1,7 +1,10 @@ -import coverage, { ExampleCall, IOptions } from "./coverage"; +import coverage, { Call, IOptions } from "./coverage"; import { OpenrpcDocument } from "@open-rpc/meta-schema"; import EmptyReporter from "./reporters/emptyReporter"; import ConsoleReporter from "./reporters/console"; +import Rule from "./rules/rule"; +import ExamplesRule from "./rules/examples-rule"; +import JsonSchemaFakerRule from "./rules/json-schema-faker-rule"; const mockSchema = { openrpc: "1.0.0", @@ -64,7 +67,7 @@ const mockSchema = { { name: "barParam2", value: "bar", - } + }, ], result: { name: "fooResult", @@ -116,6 +119,53 @@ const mockSchema = { } as OpenrpcDocument; describe("coverage", () => { + describe("rules", () => { + it("can call multiple rules with different async or sync lifecycle functions", async () => { + const reporter = new EmptyReporter(); + const transport = () => Promise.resolve({}); + const openrpcDocument = mockSchema; + const exampleRule = new ExamplesRule({ skip: ["foo"], only: ["baz"] }); + const jsonSchemaFakerRule = new JsonSchemaFakerRule({ skip: ["baz"], only: [] }); + const jsonSchemaFakerRule2 = new JsonSchemaFakerRule({ + skip: [], + only: ["foo", "bar"], + }); + class MyCustomRule implements Rule { + getTitle(): string { + return "My custom rule"; + } + getCalls(openrpcDocument: OpenrpcDocument, method: any) { + return []; + } + async validateCall(call: Call) { + return call; + } + } + const myCustomRule = new MyCustomRule(); + + const getCallsSpy = jest.spyOn(exampleRule, "getCalls"); + const getCallsCustomSpy = jest.spyOn( + myCustomRule, + "getCalls" + ); + const getCallsJsonSchemaFakerSpy = jest.spyOn( + jsonSchemaFakerRule, + "getCalls" + ); + const options = { + reporters: [reporter], + rules: [exampleRule, myCustomRule, jsonSchemaFakerRule, jsonSchemaFakerRule2], + transport, + openrpcDocument, + skip: [], + only: [], + }; + await coverage(options); + expect(getCallsSpy).toHaveBeenCalled(); + expect(getCallsCustomSpy).toHaveBeenCalled(); + expect(getCallsJsonSchemaFakerSpy).toHaveBeenCalled(); + }); + }); describe("reporter", () => { it("can call the reporter", (done) => { class CustomReporter { @@ -140,8 +190,8 @@ describe("coverage", () => { onBegin() {} onTestBegin() {} onTestEnd() {} - onEnd(options: IOptions, exampleCalls: ExampleCall[]) { - expect(exampleCalls[0].result).toBe(true); + onEnd(options: IOptions, calls: Call[]) { + expect(calls[0].result).toBe(true); done(); } } @@ -159,12 +209,12 @@ describe("coverage", () => { }); describe("coverage tests", () => { it("throws an error when there are no methods", async () => { - const reporter = new class CustomReporter { + const reporter = new (class CustomReporter { onBegin() {} onTestBegin() {} onTestEnd() {} onEnd() {} - }; + })(); const spy = jest.spyOn(reporter, "onTestBegin"); const transport = () => Promise.resolve({}); const openrpcDocument = mockSchema; @@ -172,11 +222,11 @@ describe("coverage", () => { reporters: [reporter], transport, openrpcDocument, - skip: ['foo', 'bar', 'baz'], + skip: ["foo", "bar", "baz"], only: [], }; - await expect(coverage(options)).rejects.toThrow('No methods to test'); + await expect(coverage(options)).rejects.toThrow("No methods to test"); }); it("can get to expectedResult checking with no servers", async () => { const reporter = new class CustomReporter { @@ -187,32 +237,34 @@ describe("coverage", () => { }; const spy = jest.spyOn(reporter, "onTestBegin"); const transport = () => Promise.resolve({}); - const openrpcDocument = {...mockSchema}; + const openrpcDocument = { ...mockSchema }; openrpcDocument.servers = undefined; const options = { reporters: [reporter], transport, openrpcDocument, skip: [], - only: ['baz'], + only: ["baz"], }; await expect(coverage(options)).resolves.toBeDefined(); }); }); describe("transport", () => { - it("can call the transport", (done) => { - const transport = () => { - done(); - return Promise.resolve({}); - }; - coverage({ + it("can call the transport", async () => { + const transport = jest.fn(); + await coverage({ reporters: [new EmptyReporter()], transport, - openrpcDocument: mockSchema, + openrpcDocument: { + openrpc: "1.2.6", + info: { title: "f", version: "0.0.0" }, + methods: [mockSchema.methods[0]], + }, skip: [], only: [], }); + await expect(transport).toHaveBeenCalled(); }); }); describe("reporter more tests", () => { @@ -234,13 +286,13 @@ describe("coverage", () => { await coverage(options); expect(reporter.onBegin).toHaveBeenCalled(); }); - it("onTestBegin is called", async () => { - const reporter = new class CustomReporter { + it("onTestBegin is called", async () => { + const reporter = new (class CustomReporter { onBegin() {} onTestBegin() {} onTestEnd() {} onEnd() {} - }; + })(); const spy = jest.spyOn(reporter, "onTestBegin"); const transport = () => Promise.resolve({}); const openrpcDocument = mockSchema; @@ -253,7 +305,7 @@ describe("coverage", () => { }; await coverage(options); - expect(spy).toHaveBeenCalledTimes(12); + expect(spy).toHaveBeenCalledTimes(3); }); it("can handle multiple reporters", async () => { const reporter = new EmptyReporter(); @@ -281,14 +333,14 @@ describe("coverage", () => { await coverage(options); expect(onBeginSpy).toHaveBeenCalledTimes(1); - expect(onTestBeginSpy).toHaveBeenCalledTimes(12); - expect(onTestEndSpy).toHaveBeenCalledTimes(12); + expect(onTestBeginSpy).toHaveBeenCalledTimes(3); + expect(onTestEndSpy).toHaveBeenCalledTimes(3); expect(onEndSpy).toHaveBeenCalledTimes(1); expect(onBeginSpy2).toHaveBeenCalledTimes(1); - expect(onTestBeginSpy2).toHaveBeenCalledTimes(12); - expect(onTestEndSpy2).toHaveBeenCalledTimes(12); + expect(onTestBeginSpy2).toHaveBeenCalledTimes(3); + expect(onTestEndSpy2).toHaveBeenCalledTimes(3); expect(onEndSpy2).toHaveBeenCalledTimes(1); - }) + }); }); }); diff --git a/src/coverage.ts b/src/coverage.ts index 1d0e979..3599f13 100644 --- a/src/coverage.ts +++ b/src/coverage.ts @@ -1,31 +1,23 @@ import { OpenrpcDocument, - ExamplePairingObject, - ExampleObject, JSONSchema, - ContentDescriptorObject, - Servers, - MethodObjectParams, MethodObject, } from "@open-rpc/meta-schema"; -const jsf = require("json-schema-faker"); // tslint:disable-line -import Ajv from "ajv"; -import { isEqual } from "lodash"; import Reporter from "./reporters/emptyReporter"; - -const getFakeParams = (params: any[]): any[] => { - return params.map((p) => jsf.generate(p.schema)); -}; +import JsonSchemaFakerRule from "./rules/json-schema-faker-rule"; +import ExamplesRule from "./rules/examples-rule"; +import Rule from "./rules/rule"; export interface IOptions { openrpcDocument: OpenrpcDocument; skip: string[]; only: string[]; + rules?: Rule[]; transport(url: string, method: string, params: any[]): PromiseLike; reporters: Reporter[]; } -export interface ExampleCall { +export interface Call { methodName: string; params: any[]; url: string; @@ -35,19 +27,29 @@ export interface ExampleCall { resultSchema: JSONSchema; expectedResult?: any; requestError?: any; + error?: { + message: string; + code: number; + data: any; + }; title: string; + rule?: Rule; + timings?: { + startTime?: number; + endTime?: number; + getCallsStart?: number; + getCallsEnd?: number; + beforeRequestStart?: number; + beforeRequestEnd?: number; + afterRequestStart?: number; + afterRequestEnd?: number; + afterResponseStart?: number; + afterResponseEnd?: number; + validateCallStart?: number; + validateCallEnd?: number; + } } -const paramsToObj = ( - params: any[], - methodParams: ContentDescriptorObject[] -): any => { - return params.reduce((acc, val, i) => { - acc[methodParams[i].name] = val; - return acc; - }, {}); -}; - export default async (options: IOptions) => { const filteredMethods = (options.openrpcDocument.methods as MethodObject[]) .filter(({ name }) => !options.skip.includes(name)) @@ -59,93 +61,106 @@ export default async (options: IOptions) => { throw new Error("No methods to test"); } - const exampleCalls: ExampleCall[] = []; - - const servers: Servers = options.openrpcDocument.servers || [ - { url: "http://localhost:3333" }, - ]; - - servers.forEach(({ url }) => { - filteredMethods.forEach((method) => { - if (method.examples === undefined || method.examples.length === 0) { - for (let i = 0; i < 10; i++) { - const p = getFakeParams(method.params); - // handle object or array case - const params = - method.paramStructure === "by-name" - ? paramsToObj(p, method.params as ContentDescriptorObject[]) - : p; - exampleCalls.push({ - title: method.name + " > json-schema-faker params and expect result schema to match [" + i + "]", - methodName: method.name, - params, - url, - resultSchema: (method.result as ContentDescriptorObject).schema, - }); - } - return; - } + let calls: Call[] = []; - (method.examples as ExamplePairingObject[]).forEach((ex) => { - const p = (ex.params as ExampleObject[]).map((e) => e.value); - const params = - method.paramStructure === "by-name" - ? paramsToObj(p, method.params as ContentDescriptorObject[]) - : p; - exampleCalls.push({ - title: method.name + " > example params and expect result to match: " + ex.name, - methodName: method.name, - params, - url, - resultSchema: (method.result as ContentDescriptorObject).schema, - expectedResult: (ex.result as ExampleObject).value, - }); - }); - }); - }); + let rules: Rule[] = [new JsonSchemaFakerRule(), new ExamplesRule()]; + if (options.rules && options.rules.length > 0) { + rules = options.rules; + } for (const reporter of options.reporters) { - reporter.onBegin(options, exampleCalls); + reporter.onBegin(options, calls); + } + + for (const rule of rules) { + await Promise.resolve(rule.onBegin?.(options)); } - for (const exampleCall of exampleCalls) { + // getCalls could be async or sync + const callsPromises = await Promise.all(filteredMethods.map((method) => + Promise.all( + rules.map(async (rule) => { + const _calls = await Promise.resolve(rule.getCalls(options.openrpcDocument, method)) + _calls.forEach((call) => { + // this adds the rule after the fact, it's a bit of a hack + call.rule = rule; + }); + return _calls; + } + ) + ) + )); + calls.push(...callsPromises.flat().flat()); + + for (const call of calls) { + const startTime = Date.now(); + if (call.timings === undefined) { + call.timings = { + startTime, + getCallsStart: startTime, + getCallsEnd: Date.now(), + } + } for (const reporter of options.reporters) { - reporter.onTestBegin(options, exampleCall); + reporter.onTestBegin(options, call); } - try { - const callResult = await options.transport( - exampleCall.url, - exampleCall.methodName, - exampleCall.params - ); - exampleCall.result = callResult.result; + if (call.timings) { + call.timings.beforeRequestStart = Date.now(); + } + // lifecycle methods could be async or sync + await Promise.resolve(call.rule?.beforeRequest?.(options, call)); - if (exampleCall.expectedResult) { - exampleCall.valid = isEqual( - exampleCall.expectedResult, - exampleCall.result - ); - } else { - const ajv = new Ajv(); - ajv.validate(exampleCall.resultSchema, exampleCall.result); - if (ajv.errors && ajv.errors.length > 0) { - exampleCall.valid = false; - exampleCall.reason = JSON.stringify(ajv.errors); - } else { - exampleCall.valid = true; - } - } + // transport is async but the await needs to happen later + // so that afterRequest is run immediately after the request is made + const callResultPromise = options.transport( + call.url, + call.methodName, + call.params + ); + if (call.timings) { + call.timings.afterRequestStart = Date.now(); + } + await Promise.resolve(call.rule?.afterRequest?.(options, call)); + if (call.timings) { + call.timings.afterRequestEnd = Date.now(); + } + try { + const callResult = await callResultPromise; + call.result = callResult.result; + call.error = callResult.error; } catch (e) { - exampleCall.valid = false; - exampleCall.requestError = e; + call.valid = false; + call.requestError = e; + } + if (call.requestError === undefined) { + if (call.timings) { + call.timings.validateCallStart = Date.now(); + } + await Promise.resolve(call.rule?.validateCall(call)); + if (call.timings) { + call.timings.validateCallEnd = Date.now(); + } + } + + if (call.timings) { + call.timings.afterResponseStart = Date.now(); + } + await Promise.resolve(call.rule?.afterResponse?.(options, call)); + if (call.timings) { + call.timings.afterResponseEnd = Date.now(); + call.timings.endTime = Date.now(); } for (const reporter of options.reporters) { - reporter.onTestEnd(options, exampleCall); + reporter.onTestEnd(options, call); } } + for (const rule of rules) { + await Promise.resolve(rule.onEnd?.(options, calls)); + } + for (const reporter of options.reporters) { - reporter.onEnd(options, exampleCalls); + reporter.onEnd(options, calls); } - return exampleCalls; + return calls; }; diff --git a/src/index.ts b/src/index.ts index 980e4a0..5e7f08a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,22 +5,28 @@ import { ITransport } from "./transports/ITransport"; import ConsoleReporter from "./reporters/console"; import JsonReporter from "./reporters/json"; import EmptyReporter from "./reporters/emptyReporter"; +import Rule from "./rules/rule"; +import ConsoleRuleReporter from "./reporters/console-rule"; +import ConsoleStreamingReporter from "./reporters/console-streaming"; const reporters = { console: ConsoleReporter, json: JsonReporter, empty: EmptyReporter, + "console-streaming": ConsoleStreamingReporter, + "console-rule": ConsoleRuleReporter, }; const transports = { http: HTTPTransport, }; -type ReporterString = "console" | "json" | "empty"; +type ReporterString = "console" | "json" | "empty" | "console-streaming" | "console-rule"; interface IOptions { openrpcDocument: OpenrpcDocument; skip?: string[]; only?: string[]; reporters: ReporterString[]; + rules?: Rule[]; transport: "http" | ITransport; } @@ -32,6 +38,7 @@ export default async (options: IOptions) => { } return coverage({ reporters: reporterInstances, + rules: options.rules || [], openrpcDocument: options.openrpcDocument, skip: options.skip || [], only: options.only || [], diff --git a/src/reporters/console-rule.ts b/src/reporters/console-rule.ts new file mode 100644 index 0000000..20ba229 --- /dev/null +++ b/src/reporters/console-rule.ts @@ -0,0 +1,216 @@ +import colors from "colors"; +import { JSONSchemaObject } from "@open-rpc/meta-schema"; +import { Call, IOptions } from "../coverage"; +import _ from "lodash"; +import Reporter from "./reporter"; + +const getExpectedString = (ex: Call) => { + let resSchemaID; + if (ex.resultSchema === true) { + resSchemaID = "true"; + } else if (ex.resultSchema === false) { + resSchemaID = "false"; + } else { + const s = ex.resultSchema as JSONSchemaObject; + resSchemaID = s.title ? s.title + s.type : s.type; + } + const exRes = + typeof ex.expectedResult === "string" + ? ex.expectedResult + : JSON.stringify(ex.expectedResult); + return ex.expectedResult ? exRes : resSchemaID; +}; + +class ConsoleRuleReporter implements Reporter { + private metrics: { + success: number; + error: number; + }; + + constructor() { + this.metrics = { + success: 0, + error: 0, + }; + } + + onBegin(options: IOptions, calls: Call[]) {} + onTestBegin(options: IOptions, call: Call) {} + onTestEnd(options: IOptions, call: Call) {} + onEnd(options: IOptions, calls: Call[]) { + const metrics = { + success: 0, + error: 0, + }; + _.chain(calls) + .groupBy((call) => call.rule?.getTitle()) + .forEach((callsForRule, ruleName) => { + const hasInvalid = callsForRule.reduce( + (m, { valid }) => m || !valid, + false + ); + + console.log("\n"); + if (hasInvalid) { + console.log(colors.bgRed(colors.yellow(ruleName + ":"))); + } else { + console.log(colors.bgGreen(colors.black(ruleName + ":"))); + } + callsForRule.forEach((ex) => { + if (ex.valid) { + this.metrics.success++; + const expected = getExpectedString(ex); + const timeDiff = (ex.timings!.endTime! - ex.timings!.startTime!); + const threshold = 1000; + const timeDiffColor = timeDiff <= threshold ? 'green' : 'yellow'; // Fast tests are green, slow tests are red + console.log( + " ", + colors.bold(colors.green("✓")), + colors.magenta("-"), + colors.blue(`${ex.methodName}(${JSON.stringify(ex.params)})`), + colors.italic.bold.dim(ex.title), + colors[timeDiffColor].dim(((ex.timings!.endTime! - ex.timings!.startTime!) / 1000).toString() + "s"), + ); + } else { + this.metrics.error++; + const expected = getExpectedString(ex); + console.log( + " ", + colors.bold(colors.red("X")), + colors.magenta("-"), + colors.bgRed(colors.blue(`${ex.methodName}(${JSON.stringify(ex.params)})`)), + colors.gray(ex.title), + colors.gray(((ex.timings!.endTime! - ex.timings!.startTime!) / 1000).toString() + "s") + ); + console.log( + colors.magenta("\t\t\t \->"), + colors.white.underline("Expected result:"), + colors.white(expected), + ); + + if (ex.requestError) { + console.log( + colors.magenta("\t\t\t \->"), + colors.white.underline("instead got an error: "), + colors.red(ex.requestError) + ); + } else { + console.log( + colors.magenta("\t\t\t \->"), + colors.white.underline("instead received: "), + colors.red(JSON.stringify(ex.result !== undefined ? ex.result : ex.error)) + ); + + if (ex.reason) { + console.log( + colors.magenta("\t\t\t\t \->"), + colors.yellow.underline("Reason for being invalid: "), + colors.yellow(ex.reason) + ); + } + } + } + }); + }) + .value(); + + // get method call coverage + const passingCallsByMethod = calls.reduce((m, call) => { + if (call.valid === false) { + return m; + } + if (!m[call.methodName]) { + m[call.methodName] = []; + } + m[call.methodName].push(call); + return m; + }, {} as { [key: string]: Call[] }); + + const passingMethods = Object.keys(passingCallsByMethod); + + const failedCallsByMethod = calls.reduce((m, call) => { + this.metrics[call.valid ? "success" : "error"]++; + if (call.valid === true) { + return m; + } + if (!m[call.methodName]) { + m[call.methodName] = []; + } + m[call.methodName].push(call); + return m; + }, {} as { [key: string]: Call[] }); + + const totalPassingMethods = Object.keys(passingCallsByMethod).length; + const openrpcMethods = options.openrpcDocument.methods.length; + + const failedMethods = Object.keys(failedCallsByMethod); + + //take into account failed methods + const missingMethods: string[] = options.openrpcDocument.methods + .map(({ name }) => name) + .filter((methodName) => !passingMethods.includes(methodName)) + .filter((methodName) => !failedMethods.includes(methodName)); + + console.log("=========="); + if (options.skip && options.skip.length > 0) { + console.log("Skipped:"); + } + console.log("Methods"); + console.log( + ` Coverage: ${totalPassingMethods}/${openrpcMethods} (${( + (totalPassingMethods / openrpcMethods) * + 100 + ).toFixed(2)}%)` + ); + console.log( + " Passing: ", + colors.green( + passingMethods + .map( + (m) => + colors.green(m) + + colors.magenta(" - ") + + colors.blue(`${passingCallsByMethod[m].length}`) + ) + .join(", ") + ) + ); + if (failedMethods.length > 0) { + console.log( + " Failed:", + colors.red( + failedMethods + .map( + (m) => + colors.red(m) + + colors.magenta(" - ") + + colors.blue(`${failedCallsByMethod[m].length}`) + ) + .join(", ") + ) + ); + } + console.log(" Missing: ", colors.gray(missingMethods.join(", "))); + console.log( + "Total Time: ", + colors.green( + ( + calls.reduce( + (m, { timings }) => m + (timings!.endTime! - timings!.startTime!), + 0 + ) / 1000 + ).toString() + "s" + ) + ); + console.log( + "Call Success: ", + colors.green(this.metrics.success.toString()) + ); + console.log("Call Failed: ", colors.red(this.metrics.error.toString())); + console.log("=========="); + + process.exit(metrics.error > 0 ? 1 : 0); + } +} + +export default ConsoleRuleReporter; diff --git a/src/reporters/console-streaming.ts b/src/reporters/console-streaming.ts new file mode 100644 index 0000000..53d141b --- /dev/null +++ b/src/reporters/console-streaming.ts @@ -0,0 +1,148 @@ +import colors from "colors"; +import { + JSONSchemaObject, +} from "@open-rpc/meta-schema"; +import {Call, IOptions} from "../coverage"; +import _ from "lodash"; +import Reporter from "./reporter"; + +const getExpectedString = (ex: Call) => { + let resSchemaID; + if (ex.resultSchema === true) { resSchemaID = "true"; } + else if (ex.resultSchema === false) { resSchemaID = "false"; } + else { + const s = (ex.resultSchema as JSONSchemaObject); + resSchemaID = s.title ? s.title + s.type : s.type + } + const exRes = typeof ex.expectedResult === "string" ? ex.expectedResult : JSON.stringify(ex.expectedResult); + return ex.expectedResult ? exRes : resSchemaID; +}; + +class ConsoleStreamingReporter implements Reporter { + private metrics: { + success: number; + error: number; + } + + constructor() { + this.metrics = { + success: 0, + error: 0 + }; + } + + onBegin(options: IOptions, calls: Call[]) {} + onTestBegin(options: IOptions, call: Call) {} + onTestEnd(options: IOptions, call: Call) { + const ex = call; + + if (ex.valid) { + this.metrics.success++; + const expected = getExpectedString(ex); + const timeDiff = (ex.timings!.endTime! - ex.timings!.startTime!); + const threshold = 1000; + const timeDiffColor = timeDiff <= threshold ? 'green' : 'yellow'; // Fast tests are green, slow tests are red + console.log( + " ", + colors.bold(colors.green("✓")), + colors.magenta("-"), + colors.blue(`${ex.methodName}(${JSON.stringify(ex.params)})`), + colors.italic.bold.dim(ex.title), + colors[timeDiffColor].dim(((ex.timings!.endTime! - ex.timings!.startTime!) / 1000).toString() + "s"), + ); + } else { + this.metrics.error++; + const expected = getExpectedString(ex); + console.log( + " ", + colors.bold(colors.red("X")), + colors.magenta("-"), + colors.bgRed(colors.blue(`${ex.methodName}(${JSON.stringify(ex.params)})`)), + colors.gray(ex.title), + colors.gray(((ex.timings!.endTime! - ex.timings!.startTime!) / 1000).toString() + "s") + ); + console.log( + colors.magenta("\t\t\t \->"), + colors.white.underline("Expected result:"), + colors.white(expected), + ); + + if (ex.requestError) { + console.log( + colors.magenta("\t\t\t \->"), + colors.white.underline("instead got an error: "), + colors.red(ex.requestError) + ); + } else { + console.log( + colors.magenta("\t\t\t \->"), + colors.white.underline("instead received: "), + colors.red(JSON.stringify(ex.result !== undefined ? ex.result : ex.error)) + ); + + if (ex.reason) { + console.log( + colors.magenta("\t\t\t\t \->"), + colors.yellow.underline("Reason for being invalid: "), + colors.yellow(ex.reason) + ); + } + } + } + } + onEnd(options: IOptions, calls: Call[]) { + // get method call coverage + const passingCallsByMethod = calls.reduce((m, call) => { + if (call.valid === false) { + return m; + } + if (!m[call.methodName]) { + m[call.methodName] = []; + } + m[call.methodName].push(call); + return m; + }, {} as {[key: string]: Call[]}); + + const passingMethods = Object.keys(passingCallsByMethod); + + const failedCallsByMethod = calls.reduce((m, call) => { + if (call.valid === true) { + return m; + } + if (!m[call.methodName]) { + m[call.methodName] = []; + } + m[call.methodName].push(call); + return m; + }, {} as {[key: string]: Call[]}); + + const totalPassingMethods = Object.keys(passingCallsByMethod).length; + const openrpcMethods = options.openrpcDocument.methods.length; + + const failedMethods = Object.keys(failedCallsByMethod); + + //take into account failed methods + const missingMethods: string[] = options.openrpcDocument.methods + .map(({name}) => name) + .filter((methodName) => !passingMethods.includes(methodName)) + .filter((methodName) => !failedMethods.includes(methodName)); + + console.log("=========="); + if (options.skip && options.skip.length > 0) { + console.log("Skipped:") + } + console.log("Methods"); + console.log(` Coverage: ${totalPassingMethods}/${openrpcMethods} (${((totalPassingMethods / openrpcMethods) * 100).toFixed(2)}%)`); + console.log(" Passing: ", colors.green(passingMethods.map((m) => colors.green(m) + colors.magenta(' - ') + colors.blue(`${passingCallsByMethod[m].length}`)).join(", "))); + if (failedMethods.length > 0) { + console.log(" Failed:", colors.red(failedMethods.map((m) => colors.red(m) + colors.magenta(' - ') + colors.blue(`${failedCallsByMethod[m].length}`)).join(", "))); + } + console.log(" Missing: ", colors.gray(missingMethods.join(", "))); + console.log("Total Time: ", colors.green((calls.reduce((m, {timings}) => m + (timings!.endTime! - timings!.startTime!), 0) / 1000).toString() + "s")); + console.log("Call Success: ", colors.green(this.metrics.success.toString())); + console.log("Call Failed: ", colors.red(this.metrics.error.toString())); + console.log("=========="); + } +} + +export default ConsoleStreamingReporter; diff --git a/src/reporters/console.ts b/src/reporters/console.ts index 7584a98..34dc68a 100644 --- a/src/reporters/console.ts +++ b/src/reporters/console.ts @@ -2,11 +2,11 @@ import colors from "colors"; import { JSONSchemaObject, } from "@open-rpc/meta-schema"; -import {ExampleCall, IOptions} from "../coverage"; +import {Call, IOptions} from "../coverage"; import _ from "lodash"; import Reporter from "./reporter"; -const getExpectedString = (ex: ExampleCall) => { +const getExpectedString = (ex: Call) => { let resSchemaID; if (ex.resultSchema === true) { resSchemaID = "true"; } else if (ex.resultSchema === false) { resSchemaID = "false"; } @@ -19,19 +19,32 @@ const getExpectedString = (ex: ExampleCall) => { }; class ConsoleReporter implements Reporter { + private metrics: { + success: number; + error: number; + } + + constructor() { + this.metrics = { + success: 0, + error: 0 + }; + } - onBegin(options: IOptions, exampleCalls: ExampleCall[]) {} - onTestBegin(options: IOptions, exampleCall: ExampleCall) {} - onTestEnd(options: IOptions, exampleCall: ExampleCall) {} - onEnd(options: IOptions, exampleCalls: ExampleCall[]) { + onBegin(options: IOptions, calls: Call[]) {} + onTestBegin(options: IOptions, call: Call) {} + onTestEnd(options: IOptions, call: Call) { + + } + onEnd(options: IOptions, calls: Call[]) { const metrics = { success: 0, error: 0 }; - _.chain(exampleCalls) + _.chain(calls) .groupBy("methodName") - .forEach((exampleCallsForMethod, methodName) => { - const hasInvalid = exampleCallsForMethod.reduce((m, {valid}) => m || !valid, false); + .forEach((callsForMethod, methodName) => { + const hasInvalid = callsForMethod.reduce((m, {valid}) => m || !valid, false); if (hasInvalid) { console.log(colors.bgRed(colors.yellow(methodName + ":"))); @@ -39,7 +52,7 @@ class ConsoleReporter implements Reporter { console.log(colors.bgGreen(colors.black(methodName + ":"))); } - exampleCallsForMethod.forEach((ex) => { + callsForMethod.forEach((ex) => { if (ex.valid) { metrics.success++; const expected = getExpectedString(ex); @@ -48,6 +61,7 @@ class ConsoleReporter implements Reporter { colors.bold(colors.green("✓")), colors.magenta("-"), colors.blue(`${methodName}(${JSON.stringify(ex.params)})`), + colors.white(((ex.timings!.endTime! - ex.timings!.startTime!) / 1000).toString() + "s"), ); } else { metrics.error++; @@ -56,7 +70,8 @@ class ConsoleReporter implements Reporter { "\t", colors.bold(colors.red("X")), colors.magenta("-"), - colors.bgRed(colors.blue(`${methodName}(${JSON.stringify(ex.params)})`)) + colors.bgRed(colors.blue(`${methodName}(${JSON.stringify(ex.params)})`)), + colors.white(((ex.timings!.endTime! - ex.timings!.startTime!) / 1000).toString() + "s") ); console.log( colors.magenta("\t\t\t \->"), @@ -74,7 +89,7 @@ class ConsoleReporter implements Reporter { console.log( colors.magenta("\t\t\t \->"), colors.white.underline("instead received: "), - colors.red(JSON.stringify(ex.result)) + colors.red(JSON.stringify(ex)), ); if (ex.reason) { @@ -92,6 +107,7 @@ class ConsoleReporter implements Reporter { .value(); console.log("=========="); + console.log("Total Time: ", colors.green((calls.reduce((m, {timings}) => m + (timings!.endTime! - timings!.startTime!), 0) / 1000).toString() + "s")); console.log("Success: ", colors.green(metrics.success.toString())); console.log("Errors: ", colors.red(metrics.error.toString())); console.log("=========="); diff --git a/src/reporters/emptyReporter.ts b/src/reporters/emptyReporter.ts index ea61989..a3a34bf 100644 --- a/src/reporters/emptyReporter.ts +++ b/src/reporters/emptyReporter.ts @@ -1,22 +1,23 @@ -import { ExampleCall, IOptions } from '../coverage'; +import { Call, IOptions } from '../coverage'; import Reporter from './reporter'; class EmptyReporter implements Reporter { - onBegin(options: IOptions, exampleCalls: ExampleCall[]) { - console.log(`Starting the run with ${exampleCalls.length} tests`); + onBegin(options: IOptions, calls: Call[]) { + console.log(`Starting the run with ${calls.length} tests`); } - onTestBegin(options: IOptions, exampleCall: ExampleCall) { + onTestBegin(options: IOptions, call: Call) { + console.log(`started test ${call.title}`); } - onTestEnd(options: IOptions, exampleCall: ExampleCall) { - console.log(`Finished test ${exampleCall.title}: ${exampleCall.valid ? "success" : "error"}`); + onTestEnd(options: IOptions, call: Call) { + console.log(`Finished test ${call.title}: ${call.valid ? "success" : "error"}`); } - onEnd(options: IOptions, exampleCalls: ExampleCall[]) { - const failed = exampleCalls.filter((ec) => !ec.valid); - const passed = exampleCalls.filter((ec) => ec.valid); - console.log(`Finished the running ${exampleCalls.length} tests: ${failed.length} failed, ${passed.length} passed`); + onEnd(options: IOptions, calls: Call[]) { + const failed = calls.filter((ec) => !ec.valid); + const passed = calls.filter((ec) => ec.valid); + console.log(`Finished the running ${calls.length} tests: ${failed.length} failed, ${passed.length} passed`); } } diff --git a/src/reporters/json.ts b/src/reporters/json.ts index cf972cf..4d4bd9a 100644 --- a/src/reporters/json.ts +++ b/src/reporters/json.ts @@ -1,18 +1,18 @@ -import { ExampleCall, IOptions } from '../coverage'; +import { Call, IOptions } from '../coverage'; import Reporter from './reporter'; class JsonReporter implements Reporter { - onBegin(options: IOptions, exampleCalls: ExampleCall[]) {} - onTestBegin(options: IOptions, exampleCall: ExampleCall) {} + onBegin(options: IOptions, calls: Call[]) {} + onTestBegin(options: IOptions, call: Call) {} - onTestEnd(options: IOptions, exampleCall: ExampleCall) {} + onTestEnd(options: IOptions, call: Call) {} - onEnd(options: IOptions, exampleCalls: ExampleCall[]) { - const failed = exampleCalls.filter((ec) => !ec.valid); + onEnd(options: IOptions, calls: Call[]) { + const failed = calls.filter((ec) => !ec.valid); - const passed = exampleCalls.filter((ec) => ec.valid); + const passed = calls.filter((ec) => ec.valid); - console.log(JSON.stringify(exampleCalls, undefined, 4)); + console.log(JSON.stringify(calls, undefined, 4)); } } diff --git a/src/reporters/reporter.ts b/src/reporters/reporter.ts index 15ca9e6..cbdbe8e 100644 --- a/src/reporters/reporter.ts +++ b/src/reporters/reporter.ts @@ -1,10 +1,10 @@ -import { ExampleCall, IOptions } from "../coverage"; +import { Call, IOptions } from "../coverage"; interface Reporter { - onBegin(options: IOptions, exampleCalls: ExampleCall[]): void; - onTestBegin(options: IOptions, exampleCall: ExampleCall): void; - onTestEnd(options: IOptions, exampleCall: ExampleCall): void; - onEnd(options: IOptions, exampleCalls: ExampleCall[]): void; + onBegin(options: IOptions, calls: Call[]): void; + onTestBegin(options: IOptions, call: Call): void; + onTestEnd(options: IOptions, call: Call): void; + onEnd(options: IOptions, calls: Call[]): void; } export default Reporter; diff --git a/src/rules/examples-rule.ts b/src/rules/examples-rule.ts new file mode 100644 index 0000000..537a8cb --- /dev/null +++ b/src/rules/examples-rule.ts @@ -0,0 +1,62 @@ +import { ContentDescriptorObject, ExampleObject, ExamplePairingObject, MethodObject, OpenrpcDocument } from "@open-rpc/meta-schema"; +import { Call, IOptions } from "../coverage"; +import { isEqual } from "lodash"; +import Rule from "./rule"; +import paramsToObj from "../utils/params-to-obj"; + +interface RulesOptions { + skip: string[]; + only: string[]; +} + +class ExamplesRule implements Rule { + private skip?: string[]; + private only?: string[]; + constructor(options?: RulesOptions) { + this.skip = options?.skip; + this.only = options?.only; + } + getTitle() { + return "Generate params from examples and expect results to match" + } + getCalls(openrpcDocument: OpenrpcDocument, method: MethodObject): Call[] { + if (this.skip && this.skip.includes(method.name)) { + return []; + } + if (this.only && this.only.length > 0 && !this.only.includes(method.name)) { + return []; + } + const calls: Call[] = []; + if (method.examples) { + (method.examples as ExamplePairingObject[]).forEach((ex) => { + const p = (ex.params as ExampleObject[]).map((e) => e.value); + const params = + method.paramStructure === "by-name" + ? paramsToObj(p, method.params as ContentDescriptorObject[]) + : p; + calls.push({ + title: + this.getTitle() + " " + + ex.name, + methodName: method.name, + params, + url: openrpcDocument.servers?.[0].url || "", + resultSchema: (method.result as ContentDescriptorObject).schema, + expectedResult: (ex.result as ExampleObject).value, + }); + }); + } + return calls; + } + validateCall(call: Call): Call { + if (call.expectedResult !== undefined && call.result !== undefined) { + call.valid = isEqual( + call.expectedResult, + call.result + ); + } + return call; + } +} + +export default ExamplesRule; diff --git a/src/rules/json-schema-faker-rule.test.ts b/src/rules/json-schema-faker-rule.test.ts new file mode 100644 index 0000000..aac4d79 --- /dev/null +++ b/src/rules/json-schema-faker-rule.test.ts @@ -0,0 +1,54 @@ + +import JsonSchemaFakerRule from "./json-schema-faker-rule"; + +describe("JsonSchemaFakerRule", () => { + it("should validate example calls", () => { + const rule = new JsonSchemaFakerRule(); + const openrpcDocument = { + openrpc: "1.0.0", + info: { + title: "my api", + version: "0.0.0-development", + }, + servers: [ + { + name: "my api", + url: "http://localhost:3333", + }, + ], + methods: [ + { + name: "foo", + params: [], + paramStructure: "by-name", + result: { + name: "fooResult", + schema: { + type: "boolean", + }, + }, + }, + ], + } as any; + const calls = rule.getCalls(openrpcDocument, openrpcDocument.methods[0]); + calls[0].result = true; + const result = rule.validateCall(calls[0]); + expect(result.valid).toBe(true); + }); + it("should handle errors within ajv when validating", () => { + const rule = new JsonSchemaFakerRule(); + const Call = { + title: 'test call', + methodName: "foo", + params: [], + url: "http://localhost:3333", + resultSchema: { + type: "boolean", + unevaluatedProperties: false, + }, + }; + const result = rule.validateCall(Call); + expect(result.valid).toBe(false); + expect(result.reason).toMatch('unknown keyword: "unevaluatedProperties"'); + }); +}); diff --git a/src/rules/json-schema-faker-rule.ts b/src/rules/json-schema-faker-rule.ts new file mode 100644 index 0000000..15eb3e8 --- /dev/null +++ b/src/rules/json-schema-faker-rule.ts @@ -0,0 +1,81 @@ +import { ContentDescriptorObject, MethodObject, OpenrpcDocument } from "@open-rpc/meta-schema"; +import { Call, IOptions } from "../coverage"; +import Ajv from "ajv"; +import Rule from "./rule"; +import paramsToObj from "../utils/params-to-obj"; + +const jsf = require("json-schema-faker"); // tslint:disable-line + +const getFakeParams = (params: any[]): any[] => { + return params.map((p) => jsf.generate(p.schema)); +}; + +interface RulesOptions { + skip: string[]; + only: string[]; + numCalls?: number; +} +class JsonSchemaFakerRule implements Rule { + private skip?: string[]; + private only?: string[]; + private numCalls: number; + constructor(options?: RulesOptions) { + this.skip = options?.skip; + this.only = options?.only; + this.numCalls = options?.numCalls || 10; + } + getTitle() { + return "Generate params from json-schema-faker and expect results to match"; + } + getCalls(openrpcDocument: OpenrpcDocument, method: MethodObject): Call[] { + if (this.skip && this.skip.includes(method.name)) { + return []; + } + if (this.only && this.only.length > 0 && !this.only.includes(method.name)) { + return []; + } + const url = openrpcDocument.servers ? openrpcDocument.servers[0].url : ""; + const calls: Call[] = []; + if (method.examples === undefined || method.examples.length === 0) { + let callNumForParams = this.numCalls; + if (method.params.length === 0) { + callNumForParams = 1; + } + + for (let i = 0; i < callNumForParams; i++) { + const p = getFakeParams(method.params); + // handle object or array case + const params = + method.paramStructure === "by-name" + ? paramsToObj(p, method.params as ContentDescriptorObject[]) + : p; + calls.push({ + title: this.getTitle() + "[" + i + "]", + methodName: method.name, + params, + url, + resultSchema: (method.result as ContentDescriptorObject).schema, + }); + } + } + return calls; + } + validateCall(call: Call): Call { + try { + const ajv = new Ajv(); + ajv.validate(call.resultSchema, call.result); + if (ajv.errors && ajv.errors.length > 0) { + call.valid = false; + call.reason = JSON.stringify(ajv.errors); + } else { + call.valid = true; + } + } catch (e: any) { + call.valid = false; + call.reason = e.message; + } + return call; + } +} + +export default JsonSchemaFakerRule; diff --git a/src/rules/rule.ts b/src/rules/rule.ts new file mode 100644 index 0000000..6495392 --- /dev/null +++ b/src/rules/rule.ts @@ -0,0 +1,16 @@ +import { MethodObject, OpenrpcDocument } from "@open-rpc/meta-schema"; +import { Call, IOptions } from "../coverage"; + +interface Rule { + onBegin?(options: IOptions): Promise | void; + getTitle(): string; + getCalls(openrpcDocument: OpenrpcDocument, method: MethodObject): Call[] | Promise; + validateCall(call: Call): Promise | Call; + onEnd?(options: IOptions, calls: Call[]): void; + // call lifecycle + beforeRequest?(options: IOptions, call: Call): Promise | void; + afterRequest?(options: IOptions, call: Call): Promise | void; + afterResponse?(options: IOptions, call: Call): Promise | void; +} + +export default Rule; diff --git a/src/utils/params-to-obj.ts b/src/utils/params-to-obj.ts new file mode 100644 index 0000000..9ab53a0 --- /dev/null +++ b/src/utils/params-to-obj.ts @@ -0,0 +1,13 @@ +import { ContentDescriptorObject } from "@open-rpc/meta-schema"; + +const paramsToObj = ( + params: any[], + methodParams: ContentDescriptorObject[] +): any => { + return params.reduce((acc, val, i) => { + acc[methodParams[i].name] = val; + return acc; + }, {}); +}; + +export default paramsToObj;