From 1f08b889ef2605cbb0c2f5ff5257c4e5c392d7d8 Mon Sep 17 00:00:00 2001 From: DudaGod Date: Wed, 10 Jul 2024 18:05:08 +0300 Subject: [PATCH 01/15] feat: look for configs in format testplane.config.{ts,js,cts,cjs} --- src/config/defaults.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/config/defaults.js b/src/config/defaults.js index c44ca5b6e..a046ab4a7 100644 --- a/src/config/defaults.js +++ b/src/config/defaults.js @@ -113,4 +113,13 @@ module.exports = { passive: false, }; -module.exports.configPaths = [".testplane.conf.ts", ".testplane.conf.js", ".hermione.conf.ts", ".hermione.conf.js"]; +module.exports.configPaths = [ + ".testplane.conf.ts", + ".testplane.conf.js", + "testplane.config.ts", + "testplane.config.js", + "testplane.config.cts", + "testplane.config.cjs", + ".hermione.conf.ts", + ".hermione.conf.js", +]; From 8c82cd4468eddd7a2a7dafac119fd4f840fdacd4 Mon Sep 17 00:00:00 2001 From: y-infra Date: Wed, 10 Jul 2024 18:11:31 +0000 Subject: [PATCH 02/15] chore(release): 8.17.0 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ffedbf1d..dc3322da3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.17.0](https://github.com/gemini-testing/testplane/compare/v8.16.0...v8.17.0) (2024-07-10) + + +### Features + +* look for configs in format testplane.config.{ts,js,cts,cjs} ([1f08b88](https://github.com/gemini-testing/testplane/commit/1f08b889ef2605cbb0c2f5ff5257c4e5c392d7d8)) + ## [8.16.0](https://github.com/gemini-testing/testplane/compare/v8.15.0...v8.16.0) (2024-07-10) diff --git a/package-lock.json b/package-lock.json index edb9857c1..141c371e3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testplane", - "version": "8.16.0", + "version": "8.17.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "testplane", - "version": "8.16.0", + "version": "8.17.0", "license": "MIT", "dependencies": { "@babel/code-frame": "7.24.2", diff --git a/package.json b/package.json index f4fc43022..d133f9242 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testplane", - "version": "8.16.0", + "version": "8.17.0", "description": "Tests framework based on mocha and wdio", "main": "build/src/index.js", "files": [ From abb14a08b724f88c0c292ef323ed82680157ff9d Mon Sep 17 00:00:00 2001 From: shadowusr <58862284+shadowusr@users.noreply.github.com> Date: Tue, 16 Jul 2024 19:13:40 +0300 Subject: [PATCH 03/15] fix: move escape-string-regexp to deps (#974) --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 141c371e3..9ddfd5f9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "debug": "2.6.9", "devtools": "8.21.0", "error-stack-parser": "2.1.4", + "escape-string-regexp": "1.0.5", "expect-webdriverio": "3.5.3", "fastq": "1.13.0", "fs-extra": "5.0.0", @@ -93,7 +94,6 @@ "copyfiles": "2.4.1", "doctoc": "2.2.0", "esbuild": "0.18.20", - "escape-string-regexp": "1.0.5", "eslint": "8.25.0", "eslint-config-gemini-testing": "2.8.0", "eslint-config-prettier": "8.7.0", diff --git a/package.json b/package.json index d133f9242..47657ea71 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "debug": "2.6.9", "devtools": "8.21.0", "error-stack-parser": "2.1.4", + "escape-string-regexp": "1.0.5", "expect-webdriverio": "3.5.3", "fastq": "1.13.0", "fs-extra": "5.0.0", @@ -130,7 +131,6 @@ "copyfiles": "2.4.1", "doctoc": "2.2.0", "esbuild": "0.18.20", - "escape-string-regexp": "1.0.5", "eslint": "8.25.0", "eslint-config-gemini-testing": "2.8.0", "eslint-config-prettier": "8.7.0", From 23e8b036cf0a703454fd0fbb8c35fe2b3aca0c27 Mon Sep 17 00:00:00 2001 From: y-infra Date: Tue, 16 Jul 2024 16:14:35 +0000 Subject: [PATCH 04/15] chore(release): 8.17.1 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc3322da3..ea0758f1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.17.1](https://github.com/gemini-testing/testplane/compare/v8.17.0...v8.17.1) (2024-07-16) + + +### Bug Fixes + +* move escape-string-regexp to deps ([#974](https://github.com/gemini-testing/testplane/issues/974)) ([abb14a0](https://github.com/gemini-testing/testplane/commit/abb14a08b724f88c0c292ef323ed82680157ff9d)) + ## [8.17.0](https://github.com/gemini-testing/testplane/compare/v8.16.0...v8.17.0) (2024-07-10) diff --git a/package-lock.json b/package-lock.json index 9ddfd5f9a..5f5f29ca2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testplane", - "version": "8.17.0", + "version": "8.17.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "testplane", - "version": "8.17.0", + "version": "8.17.1", "license": "MIT", "dependencies": { "@babel/code-frame": "7.24.2", diff --git a/package.json b/package.json index 47657ea71..e95ca633e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testplane", - "version": "8.17.0", + "version": "8.17.1", "description": "Tests framework based on mocha and wdio", "main": "build/src/index.js", "files": [ From 3c9ce6becccf1f8940dbd9c92204b15c4efa184a Mon Sep 17 00:00:00 2001 From: Roman Kuznetsov Date: Mon, 15 Jul 2024 03:45:24 +0300 Subject: [PATCH 05/15] fix: support error snippets nested browser commands --- src/browser/stacktrace/index.ts | 6 +- src/browser/stacktrace/utils.ts | 111 ++++++++++++++++++++++++--- src/error-snippets/frames.ts | 40 +--------- src/error-snippets/source-maps.ts | 3 +- src/error-snippets/utils.ts | 19 ++--- src/utils/fs.ts | 13 ++++ test/src/browser/stacktrace/index.ts | 22 ++---- test/src/browser/stacktrace/utils.ts | 65 ++++++++++------ test/src/error-snippets/utils.ts | 8 +- 9 files changed, 175 insertions(+), 112 deletions(-) diff --git a/src/browser/stacktrace/index.ts b/src/browser/stacktrace/index.ts index 2fd4daa9e..2dc53e355 100644 --- a/src/browser/stacktrace/index.ts +++ b/src/browser/stacktrace/index.ts @@ -1,4 +1,4 @@ -import { ShallowStackFrames, applyStackFrames, captureRawStackFrames } from "./utils"; +import { ShallowStackFrames, applyStackTraceIfBetter, captureRawStackFrames } from "./utils"; import { getBrowserCommands, getElementCommands } from "../history/commands"; import { runWithHooks } from "../history/utils"; @@ -15,7 +15,7 @@ export const runWithStacktraceHooks = ({ }): ReturnType => { const frames = captureRawStackFrames(stackFilterFunc || runWithStacktraceHooks); - if (stackFrames.isNested(frames)) { + if (stackFrames.areInternal(frames)) { return fn(); } @@ -25,7 +25,7 @@ export const runWithStacktraceHooks = ({ before: () => stackFrames.enter(key, frames), fn, after: () => stackFrames.leave(key), - error: (err: Error) => applyStackFrames(err, frames), + error: (err: Error) => applyStackTraceIfBetter(err, frames), }); }; diff --git a/src/browser/stacktrace/utils.ts b/src/browser/stacktrace/utils.ts index a42a6ef06..20a2be9d8 100644 --- a/src/browser/stacktrace/utils.ts +++ b/src/browser/stacktrace/utils.ts @@ -1,6 +1,8 @@ +import _ from "lodash"; import ErrorStackParser from "error-stack-parser"; import type { SetRequired } from "type-fest"; import logger from "../../utils/logger"; +import { softFileURLToPath } from "../../utils/fs"; import { WDIO_IGNORED_STACK_FUNCTIONS, WDIO_STACK_TRACE_LIMIT } from "./constants"; export type RawStackFrames = string; @@ -48,7 +50,7 @@ const getErrorRawStackFrames = (e: ErrorWithStack): RawStackFrames => { return e.stack.slice(errorMessageEndsStackIndex + 1); }; -export function captureRawStackFrames(filterFunc?: (...args: unknown[]) => unknown): RawStackFrames { +export const captureRawStackFrames = (filterFunc?: (...args: unknown[]) => unknown): RawStackFrames => { const savedStackTraceLimit = Error.stackTraceLimit; const targetObj = {} as { stack: RawStackFrames }; @@ -59,19 +61,90 @@ export function captureRawStackFrames(filterFunc?: (...args: unknown[]) => unkno const rawFramesPosition = targetObj.stack.indexOf("\n") + 1; // crop out error message return targetObj.stack.slice(rawFramesPosition); -} +}; + +/** + * @description + * Rank values: + * + * 0: Can't extract code snippet; useless + * + * 1: WebdriverIO internals: Better than nothing + * + * 2: Project internals: Better than WebdriverIO internals, but worse, than user code part + * + * 3: User code: Best choice + */ +export const FRAME_RELEVANCE: Record boolean }> = { + repl: { value: 0, matcher: fileName => /^REPL\d+$/.test(fileName) }, + nodeInternals: { value: 0, matcher: fileName => /^node:[a-zA-Z\-_]/.test(fileName) }, + wdioInternals: { value: 1, matcher: fileName => fileName.includes("/node_modules/webdriverio/") }, + projectInternals: { value: 2, matcher: fileName => fileName.includes("/node_modules/") }, + userCode: { value: 3, matcher: () => true }, +} as const; + +export const getFrameRelevance = (frame: StackFrame): number => { + if ([frame.fileName, frame.lineNumber, frame.columnNumber].some(_.isUndefined)) { + return 0; + } + + const fileName: string = softFileURLToPath(frame.fileName!); + + for (const factor in FRAME_RELEVANCE) { + if (FRAME_RELEVANCE[factor].matcher(fileName)) { + return FRAME_RELEVANCE[factor].value; + } + } + + return 0; +}; -export function applyStackFrames(error: Error, frames: RawStackFrames): Error { +const getStackTraceRelevance = (error: Error): number => { + const framesParsed = ErrorStackParser.parse(error); + + return framesParsed.reduce((maxRelevance, frame) => { + return Math.max(maxRelevance, getFrameRelevance(frame)); + }, 0); +}; + +const createErrorWithStack = (stack: RawStackFrames, errorMessage = ""): Error => { + const newError = new Error(errorMessage); + + newError.stack = getErrorTitle(newError) + "\n" + stack; + + return newError; +}; + +const applyStackTrace = (error: Error, stack: RawStackFrames): Error => { if (!error || !error.message) { return error; } - error.stack = getErrorTitle(error) + "\n" + frames; + error.stack = getErrorTitle(error) + "\n" + stack; return error; -} +}; + +export const applyStackTraceIfBetter = (error: Error, stack: RawStackFrames): Error => { + if (!error || !error.message) { + return error; + } + + try { + const newStackTraceRelevance = getStackTraceRelevance(createErrorWithStack(stack)); + const currentStackTraceRelevance = getStackTraceRelevance(error); + + if (newStackTraceRelevance > currentStackTraceRelevance) { + applyStackTrace(error, stack); + } + } catch (err) { + logger.warn("Couldn't compare error stack traces"); + } + + return error; +}; -export function filterExtraWdioFrames(error: Error): Error { +export const filterExtraWdioFrames = (error: Error): Error => { if (!error || !error.message || !error.stack) { return error; } @@ -103,13 +176,13 @@ export function filterExtraWdioFrames(error: Error): Error { const framesFiltered = rawFramesArr.filter((_, i) => shouldIncludeFrame(framesParsed[i])).join("\n"); - return applyStackFrames(error, framesFiltered); + return applyStackTrace(error, framesFiltered); } catch (filterError) { logger.warn("Couldn't filter out wdio frames", filterError); return error; } -} +}; export class ShallowStackFrames { private _framesMap: Map; @@ -132,13 +205,29 @@ export class ShallowStackFrames { this._framesMap.delete(key); } - isNested(childFrames: RawStackFrames): boolean { + private _getParentStackFrame(childFrames: RawStackFrames): RawStackFrames | null { for (const parentFrames of this._framesMap.values()) { if (childFrames.length !== parentFrames.length && childFrames.endsWith(parentFrames)) { - return true; + return parentFrames; } } - return false; + return null; + } + + areInternal(childFrames: RawStackFrames): boolean { + const parentStackFrame = this._getParentStackFrame(childFrames); + + if (!parentStackFrame) { + return false; + } + + const isNodeModulesFrame = (frame: string): boolean => frame.includes("/node_modules/"); + const isNodeInternalFrame = (frame: string): boolean => frame.includes(" (node:"); + + const extraFrames = childFrames.slice(0, childFrames.length - parentStackFrame.length); + const extraFramesArray = extraFrames.split("\n"); + + return extraFramesArray.every(frame => !frame || isNodeModulesFrame(frame) || isNodeInternalFrame(frame)); } } diff --git a/src/error-snippets/frames.ts b/src/error-snippets/frames.ts index 1507126e6..12e6efa0c 100644 --- a/src/error-snippets/frames.ts +++ b/src/error-snippets/frames.ts @@ -1,44 +1,8 @@ -import _ from "lodash"; import ErrorStackParser from "error-stack-parser"; import logger from "../utils/logger"; -import { softFileURLToPath } from "./utils"; +import { getFrameRelevance } from "../browser/stacktrace/utils"; import type { ResolvedFrame, SufficientStackFrame } from "./types"; - -/** - * @description - * Rank values: - * - * 0: Can't extract code snippet; useless - * - * 1: WebdriverIO internals: Better than nothing - * - * 2: Project internals: Better than WebdriverIO internals, but worse, than user code part - * - * 3: User code: Best choice - */ -const FRAME_REELVANCE: Record boolean }> = { - repl: { value: 0, matcher: fileName => /^REPL\d+$/.test(fileName) }, - nodeInternals: { value: 0, matcher: fileName => /^node:[a-zA-Z\-_]/.test(fileName) }, - wdioInternals: { value: 1, matcher: fileName => fileName.includes("/node_modules/webdriverio/") }, - projectInternals: { value: 2, matcher: fileName => fileName.includes("/node_modules/") }, - userCode: { value: 3, matcher: () => true }, -} as const; - -const getFrameRelevance = (frame: StackFrame): number => { - if ([frame.fileName, frame.lineNumber, frame.columnNumber].some(_.isUndefined)) { - return 0; - } - - const fileName: string = softFileURLToPath(frame.fileName!); - - for (const factor in FRAME_REELVANCE) { - if (FRAME_REELVANCE[factor].matcher(fileName)) { - return FRAME_REELVANCE[factor].value; - } - } - - return 0; -}; +import { softFileURLToPath } from "../utils/fs"; export const findRelevantStackFrame = (error: Error): SufficientStackFrame | null => { try { diff --git a/src/error-snippets/source-maps.ts b/src/error-snippets/source-maps.ts index e2162acfa..5604803b3 100644 --- a/src/error-snippets/source-maps.ts +++ b/src/error-snippets/source-maps.ts @@ -1,7 +1,8 @@ import { SourceMapConsumer, type BasicSourceMapConsumer } from "source-map"; import url from "url"; import { SOURCE_MAP_URL_COMMENT } from "./constants"; -import { softFileURLToPath, getSourceCodeFile } from "./utils"; +import { getSourceCodeFile } from "./utils"; +import { softFileURLToPath } from "../utils/fs"; import type { SufficientStackFrame, ResolvedFrame } from "./types"; export const extractSourceMaps = async ( diff --git a/src/error-snippets/utils.ts b/src/error-snippets/utils.ts index dc8608da2..2f9cfd0e4 100644 --- a/src/error-snippets/utils.ts +++ b/src/error-snippets/utils.ts @@ -1,11 +1,11 @@ import path from "path"; -import { fileURLToPath } from "url"; import fs from "fs-extra"; import { codeFrameColumns } from "@babel/code-frame"; import { getErrorTitle } from "../browser/stacktrace/utils"; import { SNIPPET_LINES_ABOVE, SNIPPET_LINES_BELOW, SOURCE_MAP_URL_COMMENT } from "./constants"; import { AssertViewError } from "../browser/commands/assert-view/errors/assert-view-error"; import { BaseStateError } from "../browser/commands/assert-view/errors/base-state-error"; +import { softFileURLToPath } from "../utils/fs"; interface FormatFileNameHeaderOpts { line: number; @@ -29,23 +29,16 @@ export const shouldNotAddCodeSnippet = (err: Error): boolean => { return isScreenshotError; }; -export const softFileURLToPath = (fileName: string): string => { - if (!fileName.startsWith("file://")) { - return fileName; - } - - try { - return fileURLToPath(fileName); - } catch (_) { - return fileName; - } +const trimAsyncPrefix = (fileName: string): string => { + const asyncPrefix = "async "; + return fileName.startsWith(asyncPrefix) ? fileName.slice(asyncPrefix.length) : fileName; }; export const formatFileNameHeader = (fileName: string, opts: FormatFileNameHeaderOpts): string => { const lineNumberWidth = String(opts.line - opts.linesAbove).length; const offsetWidth = String(opts.line + opts.linesBelow).length; - const filePath = softFileURLToPath(fileName); + const filePath = softFileURLToPath(trimAsyncPrefix(fileName)); const relativeFileName = path.isAbsolute(filePath) ? path.relative(process.cwd(), filePath) : filePath; const lineNumberOffset = ".".repeat(lineNumberWidth).padStart(offsetWidth); const offset = ` ${lineNumberOffset} |`; @@ -78,7 +71,7 @@ export const formatErrorSnippet = (error: Error, { file, source, location }: For }; export const getSourceCodeFile = async (fileName: string): Promise => { - const filePath = softFileURLToPath(fileName); + const filePath = softFileURLToPath(trimAsyncPrefix(fileName)); if (path.isAbsolute(filePath)) { return fs.readFile(filePath, "utf8"); diff --git a/src/utils/fs.ts b/src/utils/fs.ts index 79d678b6a..c9faa2715 100644 --- a/src/utils/fs.ts +++ b/src/utils/fs.ts @@ -1,4 +1,5 @@ import fs from "fs"; +import { fileURLToPath } from "url"; export const exists = async (path: string): Promise => { try { @@ -8,3 +9,15 @@ export const exists = async (path: string): Promise => { return false; } }; + +export const softFileURLToPath = (fileName: string): string => { + if (!fileName.startsWith("file://")) { + return fileName; + } + + try { + return fileURLToPath(fileName); + } catch (_) { + return fileName; + } +}; diff --git a/test/src/browser/stacktrace/index.ts b/test/src/browser/stacktrace/index.ts index 59a317568..84c0b4b58 100644 --- a/test/src/browser/stacktrace/index.ts +++ b/test/src/browser/stacktrace/index.ts @@ -27,7 +27,7 @@ describe("stacktrace", () => { runWithStacktraceHooks = proxyquire("../../../../src/browser/stacktrace", { "./utils": { captureRawStackFrames: captureRawStackFramesSpy, - applyStackFrames: applyStackFramesStub, + applyStackTraceIfBetter: applyStackFramesStub, }, }).runWithStacktraceHooks; @@ -36,7 +36,7 @@ describe("stacktrace", () => { sandbox.spy(stackFrames, "enter"); sandbox.spy(stackFrames, "leave"); sandbox.spy(stackFrames, "getKey"); - sandbox.spy(stackFrames, "isNested"); + sandbox.spy(stackFrames, "areInternal"); }); afterEach(() => sandbox.restore()); @@ -73,22 +73,12 @@ describe("stacktrace", () => { assert.calledOnce(stackFrames.enter); }); - it("should enter stack trace once with nested calls", () => { - const fn = sandbox.stub().callsFake(() => { - const nestedFirst = sandbox.stub().callsFake(() => { - const nestedSecond = sandbox.stub().callsFake(() => { - return runWithStacktraceHooks_(sandbox.stub()); - }); + it("should enter stack trace once if frames are irrelevant", () => { + stackFrames.areInternal = sandbox.stub().returns(true); - return runWithStacktraceHooks_(nestedSecond); - }); + runWithStacktraceHooks_(() => {}); - return runWithStacktraceHooks_(nestedFirst); - }); - - runWithStacktraceHooks_(fn); - - assert.calledOnce(stackFrames.enter); + assert.notCalled(stackFrames.enter); }); it("should leave stack trace after function resolved", async () => { diff --git a/test/src/browser/stacktrace/utils.ts b/test/src/browser/stacktrace/utils.ts index bb4a01cf3..63d294efd 100644 --- a/test/src/browser/stacktrace/utils.ts +++ b/test/src/browser/stacktrace/utils.ts @@ -1,13 +1,13 @@ import { ShallowStackFrames, - applyStackFrames, + applyStackTraceIfBetter, captureRawStackFrames, filterExtraWdioFrames, } from "../../../../src/browser/stacktrace/utils"; type AnyFunc = (...args: any[]) => unknown; // eslint-disable-line @typescript-eslint/no-explicit-any -describe("utils/stacktrace", () => { +describe("stacktrace/utils", () => { describe("captureRawStackFrames", () => { it("should only return frames", () => { const frames = captureRawStackFrames(); @@ -29,32 +29,30 @@ describe("utils/stacktrace", () => { }); }); - describe("applyStackFrames", () => { + describe("applyStackTraceIfBetter", () => { it("should work with multiline error messages", () => { const error = new Error("my\nmulti-line\nerror\nmessage"); - const frames = "foo\nbar"; + error.stack = "Error: " + error.message + "\n"; - applyStackFrames(error, frames); - - const expectedStack = ["Error: my\nmulti-line\nerror\nmessage", "foo", "bar"].join("\n"); - - assert.equal(error.stack, expectedStack); - }); - - it("should work with error-like objects", () => { - const error = { message: "foo" } as Error; - const frames = "bar"; + const frames = [ + "at Context. (test/src/browser/stacktrace/utils.ts:43:20)", + "at processImmediate (node:internal/timers:471:21)", + ].join("\n"); - applyStackFrames(error, frames); + applyStackTraceIfBetter(error, frames); - const expectedStack = ["Error: foo", "bar"].join("\n"); + const expectedStack = [ + "Error: my\nmulti-line\nerror\nmessage", + "at Context. (test/src/browser/stacktrace/utils.ts:43:20)", + "at processImmediate (node:internal/timers:471:21)", + ].join("\n"); assert.equal(error.stack, expectedStack); }); it("should not throw on bad input", () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - assert.doesNotThrow(() => applyStackFrames("foo" as any, 1 as any)); + assert.doesNotThrow(() => applyStackTraceIfBetter("foo" as any, 1 as any)); }); }); @@ -73,7 +71,7 @@ describe("utils/stacktrace", () => { "at async Socket. (http://localhost:4001/node_modules/testplane/build/src/runner/browser-env/vite/browser-modules/mocha/index.js?v=80fca7b2:54:17)", ].join("\n"); - applyStackFrames(error, errorStack); + error.stack = `${error.name}: ${error.message}\n${errorStack}`; filterExtraWdioFrames(error); const expectedStack = [ @@ -106,14 +104,14 @@ describe("utils/stacktrace", () => { }); }); - describe("isNested", () => { + describe("areInternal", () => { it("should return 'false' on different frames", () => { const key = stackFrames.getKey(); const parentFrames = "f\no\no"; stackFrames.enter(key, parentFrames); - assert.isFalse(stackFrames.isNested("b\na\nr")); + assert.isFalse(stackFrames.areInternal("b\na\nr")); stackFrames.leave(key); }); @@ -124,18 +122,37 @@ describe("utils/stacktrace", () => { stackFrames.enter(key, parentFrames); - assert.isFalse(stackFrames.isNested("f\no\no")); + assert.isFalse(stackFrames.areInternal("f\no\no")); + + stackFrames.leave(key); + }); + + it("should return 'true' on nested frames if those are internal frames", () => { + const key = stackFrames.getKey(); + const parentFrames = `b\na\nr`; + const childFrames = [ + "at async Element.wrapCommandFn (file:///project_folder/node_modules/webdriverio/node_modules/@wdio/utils/build/shim.js:81:29)", + "at processTicksAndRejections (node:internal/process/task_queues:95:5)", + "b", + "a", + "r", + ].join("\n"); + + stackFrames.enter(key, parentFrames); + + assert.isTrue(stackFrames.areInternal(childFrames)); stackFrames.leave(key); }); - it("should return 'true' on nested frames", () => { + it("should return 'false' on nested frames if those are not internal frames", () => { const key = stackFrames.getKey(); - const parentFrames = "b\na\nr"; + const parentFrames = `b\na\nr`; + const childFrames = ["f", "o", "o", "b", "a", "r"].join("\n"); stackFrames.enter(key, parentFrames); - assert.isTrue(stackFrames.isNested("f\no\no\nb\na\nr")); + assert.isFalse(stackFrames.areInternal(childFrames)); stackFrames.leave(key); }); diff --git a/test/src/error-snippets/utils.ts b/test/src/error-snippets/utils.ts index 22f2bf564..494f5ae48 100644 --- a/test/src/error-snippets/utils.ts +++ b/test/src/error-snippets/utils.ts @@ -2,12 +2,8 @@ import path from "path"; import sinon from "sinon"; import url from "url"; import fs from "fs-extra"; -import { - softFileURLToPath, - formatFileNameHeader, - getSourceCodeFile, - formatErrorSnippet, -} from "../../../src/error-snippets/utils"; +import { formatFileNameHeader, getSourceCodeFile, formatErrorSnippet } from "../../../src/error-snippets/utils"; +import { softFileURLToPath } from "../../../src/utils/fs"; import type { codeFrameColumns } from "@babel/code-frame"; const codeFrame = require("@babel/code-frame"); // eslint-disable-line @typescript-eslint/no-var-requires From 406a2d4cc0bef646de18f2ce4eae2cd87ccb6e8a Mon Sep 17 00:00:00 2001 From: y-infra Date: Wed, 17 Jul 2024 11:53:13 +0000 Subject: [PATCH 06/15] chore(release): 8.17.2 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea0758f1b..728179556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.17.2](https://github.com/gemini-testing/testplane/compare/v8.17.1...v8.17.2) (2024-07-17) + + +### Bug Fixes + +* support error snippets nested browser commands ([3c9ce6b](https://github.com/gemini-testing/testplane/commit/3c9ce6becccf1f8940dbd9c92204b15c4efa184a)) + ### [8.17.1](https://github.com/gemini-testing/testplane/compare/v8.17.0...v8.17.1) (2024-07-16) diff --git a/package-lock.json b/package-lock.json index 5f5f29ca2..47ba14aa6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testplane", - "version": "8.17.1", + "version": "8.17.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "testplane", - "version": "8.17.1", + "version": "8.17.2", "license": "MIT", "dependencies": { "@babel/code-frame": "7.24.2", diff --git a/package.json b/package.json index e95ca633e..05359ef3c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testplane", - "version": "8.17.1", + "version": "8.17.2", "description": "Tests framework based on mocha and wdio", "main": "build/src/index.js", "files": [ From c8ecd601713088e9cd09ae7435d942603d0240f0 Mon Sep 17 00:00:00 2001 From: DudaGod Date: Mon, 22 Jul 2024 13:34:50 +0300 Subject: [PATCH 07/15] feat: add "pid" to log info with test result --- src/reporters/utils/helpers.js | 3 ++- test/src/reporters/flat.js | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/reporters/utils/helpers.js b/src/reporters/utils/helpers.js index 45b595a6b..8496997d1 100644 --- a/src/reporters/utils/helpers.js +++ b/src/reporters/utils/helpers.js @@ -23,10 +23,11 @@ exports.formatTestInfo = test => { const suiteName = test.fullTitle().replace(test.title, ""); const sessionId = test.sessionId ? `:${test.sessionId}` : ""; const reason = test.pending && ` reason: ${chalk.red(getSkipReason(test) || "no comment")}`; + const pid = test.meta?.pid ? `, pid:${test.meta.pid}` : ""; return ( ` ${suiteName}${chalk.underline(test.title)} [${chalk.yellow(test.browserId)}` + - `${sessionId}] - ${chalk.cyan(test.duration || 0)}ms${reason || ""}` + `${sessionId}${pid}] - ${chalk.cyan(test.duration || 0)}ms${reason || ""}` ); }; diff --git a/test/src/reporters/flat.js b/test/src/reporters/flat.js index aeb6e9c07..946eec32f 100644 --- a/test/src/reporters/flat.js +++ b/test/src/reporters/flat.js @@ -102,15 +102,26 @@ describe("Flat reporter", () => { TEST_FAIL: "failed", }; - it("should correctly do the rendering", async () => { - test = mkTestStub_({ sessionId: "test_session" }); + it("should render session id", async () => { + test = mkTestStub_({ sessionId: "500100" }); await createFlatReporter(); emit(RunnerEvents.TEST_PASS, test); const result = getDeserializedResult(informer.log.firstCall.args[0]); - assert.equal(result, "suite test [chrome:test_session] - 100500ms"); + assert.equal(result, "suite test [chrome:500100] - 100500ms"); + }); + + it("should render pid", async () => { + test = mkTestStub_({ meta: { pid: "12345" } }); + + await createFlatReporter(); + emit(RunnerEvents.TEST_PASS, test); + + const result = getDeserializedResult(informer.log.firstCall.args[0]); + + assert.equal(result, "suite test [chrome, pid:12345] - 100500ms"); }); describe("skipped tests report", () => { From c63b0be081efc5658527fe17fe974a2293a35877 Mon Sep 17 00:00:00 2001 From: y-infra Date: Mon, 22 Jul 2024 15:33:29 +0000 Subject: [PATCH 08/15] chore(release): 8.18.0 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 728179556..704770318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.18.0](https://github.com/gemini-testing/testplane/compare/v8.17.2...v8.18.0) (2024-07-22) + + +### Features + +* add "pid" to log info with test result ([c8ecd60](https://github.com/gemini-testing/testplane/commit/c8ecd601713088e9cd09ae7435d942603d0240f0)) + ### [8.17.2](https://github.com/gemini-testing/testplane/compare/v8.17.1...v8.17.2) (2024-07-17) diff --git a/package-lock.json b/package-lock.json index 47ba14aa6..9c7ae680b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testplane", - "version": "8.17.2", + "version": "8.18.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "testplane", - "version": "8.17.2", + "version": "8.18.0", "license": "MIT", "dependencies": { "@babel/code-frame": "7.24.2", diff --git a/package.json b/package.json index 05359ef3c..5f93c6bb3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testplane", - "version": "8.17.2", + "version": "8.18.0", "description": "Tests framework based on mocha and wdio", "main": "build/src/index.js", "files": [ From 3624e01f5856a074edadcd897e8be54c0b16f934 Mon Sep 17 00:00:00 2001 From: Kabir-Ivan <110986400+Kabir-Ivan@users.noreply.github.com> Date: Wed, 24 Jul 2024 16:55:38 +0300 Subject: [PATCH 09/15] fix: correctly require plugins with yarn+pnp (#977) --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9c7ae680b..d68819bb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,7 +36,7 @@ "micromatch": "4.0.5", "mocha": "10.2.0", "modern-node-polyfills": "1.0.0", - "plugins-loader": "1.3.1", + "plugins-loader": "1.3.2", "png-validator": "1.1.0", "sharp": "0.32.6", "sizzle": "2.3.6", @@ -13426,9 +13426,9 @@ } }, "node_modules/plugins-loader": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.1.tgz", - "integrity": "sha512-WqZU4HLzvPiGaovYojXbIMd3wxoZ4OtN500M9RblbUU2CNto0VuTda1fIzo+nje+K0IFG0YoRqM1E9Xee5rpRQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.2.tgz", + "integrity": "sha512-LPYpqoP99V4g4dkrlAwS72V5EHXD1hWcx2HzA9mlJ13RyugR6TaPLSX37hNHwQxG0YTkWmoiWPhMnYqnsEawZQ==", "dependencies": { "lodash": "^4.16.4" } @@ -27361,9 +27361,9 @@ "dev": true }, "plugins-loader": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.1.tgz", - "integrity": "sha512-WqZU4HLzvPiGaovYojXbIMd3wxoZ4OtN500M9RblbUU2CNto0VuTda1fIzo+nje+K0IFG0YoRqM1E9Xee5rpRQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.2.tgz", + "integrity": "sha512-LPYpqoP99V4g4dkrlAwS72V5EHXD1hWcx2HzA9mlJ13RyugR6TaPLSX37hNHwQxG0YTkWmoiWPhMnYqnsEawZQ==", "requires": { "lodash": "^4.16.4" } diff --git a/package.json b/package.json index 5f93c6bb3..f465d338a 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "micromatch": "4.0.5", "mocha": "10.2.0", "modern-node-polyfills": "1.0.0", - "plugins-loader": "1.3.1", + "plugins-loader": "1.3.2", "png-validator": "1.1.0", "sharp": "0.32.6", "sizzle": "2.3.6", From 3a41d6ea027e06f0948aaa91bceb6c801d04e68f Mon Sep 17 00:00:00 2001 From: y-infra Date: Wed, 24 Jul 2024 13:56:53 +0000 Subject: [PATCH 10/15] chore(release): 8.18.1 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 704770318..fd065c085 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.18.1](https://github.com/gemini-testing/testplane/compare/v8.18.0...v8.18.1) (2024-07-24) + + +### Bug Fixes + +* correctly require plugins with yarn+pnp ([#977](https://github.com/gemini-testing/testplane/issues/977)) ([3624e01](https://github.com/gemini-testing/testplane/commit/3624e01f5856a074edadcd897e8be54c0b16f934)) + ## [8.18.0](https://github.com/gemini-testing/testplane/compare/v8.17.2...v8.18.0) (2024-07-22) diff --git a/package-lock.json b/package-lock.json index d68819bb7..e6d0f2c15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testplane", - "version": "8.18.0", + "version": "8.18.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "testplane", - "version": "8.18.0", + "version": "8.18.1", "license": "MIT", "dependencies": { "@babel/code-frame": "7.24.2", diff --git a/package.json b/package.json index f465d338a..a17be4bd9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testplane", - "version": "8.18.0", + "version": "8.18.1", "description": "Tests framework based on mocha and wdio", "main": "build/src/index.js", "files": [ From 36aa51657c0807f1d3474084b336b02270bba9d7 Mon Sep 17 00:00:00 2001 From: Kabir-Ivan <110986400+Kabir-Ivan@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:06:54 +0300 Subject: [PATCH 11/15] feat: add config option to rerun failed tests (#973) --- docs/config.md | 12 +++ src/config/defaults.js | 5 + src/config/index.ts | 1 + src/config/options.js | 29 ++++++ src/config/types.ts | 6 ++ src/test-reader/test-parser.js | 31 +++++++ src/testplane.ts | 32 ++++++- test/src/config/options.js | 137 ++++++++++++++++++++++++++++ test/src/test-reader/test-parser.js | 114 ++++++++++++++++++++++- test/src/testplane.js | 65 +++++++++++-- test/utils.js | 6 ++ 11 files changed, 427 insertions(+), 11 deletions(-) diff --git a/docs/config.md b/docs/config.md index 7114edc10..666ab5aec 100644 --- a/docs/config.md +++ b/docs/config.md @@ -73,6 +73,7 @@ - [List of useful plugins](#list-of-useful-plugins) - [prepareBrowser](#preparebrowser) - [prepareEnvironment](#prepareenvironment) +- [lastFailed](#lastfailed) @@ -731,3 +732,14 @@ Full list of parameters: - waitServerTimeout (optional) `Number` - timeout to wait for server to be ready (ms). 60_000 by default - probeRequestTimeout (optional) `Number` - one request timeout (ms), after which request will be aborted. 10_000 by default - probeRequestInterval (optional) `Number` - interval between ready probe requests (ms). 1_000 by default + +### lastFailed +Allows you to run only tests that failed on the last run. Disabled by default - it means run all tests, but the file with the failed tests is always written. + +```js +lastFailed: { + only: true, // true means run only failed, false - run all (default: false) + input: ['.testplane/failed.json', '.testplane/failed2.json'], // File/files to read failed tests list from (default: '.testplane/failed.json') + output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json') +} +``` \ No newline at end of file diff --git a/src/config/defaults.js b/src/config/defaults.js index a046ab4a7..8e87ad725 100644 --- a/src/config/defaults.js +++ b/src/config/defaults.js @@ -94,6 +94,11 @@ module.exports = { headless: null, isolation: null, testRunEnv: NODEJS_TEST_RUN_ENV, + lastFailed: { + only: false, + output: ".testplane/failed.json", + input: ".testplane/failed.json", + }, devServer: { command: null, cwd: null, diff --git a/src/config/index.ts b/src/config/index.ts index 2f7432ff3..3edd1fdcc 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -72,6 +72,7 @@ export class Config { const browserOptions = _.extend({}, browser, { id: id, system: this.system, + lastFailed: this.lastFailed, }); return new BrowserConfig(browserOptions); diff --git a/src/config/options.js b/src/config/options.js index 1c57df8c5..201120b33 100644 --- a/src/config/options.js +++ b/src/config/options.js @@ -107,6 +107,35 @@ const rootSection = section( plugins: options.anyObject(), + lastFailed: section({ + only: options.boolean("lastFailed.only"), + input: option({ + defaultValue: defaults.lastFailed.input, + validate: value => { + if (!_.isString(value) && !_.isArray(value)) { + throw new Error('"lastFailed.input" must be a string or an array'); + } + if (!_.isArray(value) && !value.endsWith(".json")) { + throw new Error('"lastFailed.input" must have .json extension'); + } + if (_.isArray(value) && value.filter(v => !v.endsWith(".json")).length) { + throw new Error('"lastFailed.input" elements must have .json extension'); + } + }, + }), + output: option({ + defaultValue: defaults.lastFailed.output, + validate: value => { + if (!_.isString(value)) { + throw new Error('"lastFailed.output" must be a string'); + } + if (!value.endsWith(".json")) { + throw new Error('"lastFailed.output" must have .json extension'); + } + }, + }), + }), + sets: map( section({ files: option({ diff --git a/src/config/types.ts b/src/config/types.ts index fe91cce0b..547affe87 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -140,6 +140,12 @@ export interface CommonConfig { headless: "old" | "new" | boolean | null; isolation: boolean; + lastFailed: { + only: boolean; + input: string | Array; + output: string; + }; + openAndWaitOpts: { timeout?: number; waitNetworkIdle: boolean; diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 4401a8cd2..5cce762f9 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -14,9 +14,15 @@ const { MasterEvents } = require("../events"); const _ = require("lodash"); const clearRequire = require("clear-require"); const path = require("path"); +const fs = require("fs-extra"); +const logger = require("../utils/logger"); +const { getShortMD5 } = require("../utils/crypto"); + +const getFailedTestId = test => getShortMD5(`${test.fullTitle}${test.browserId}${test.browserVersion}`); class TestParser extends EventEmitter { #opts; + #failedTests; #buildInstructions; /** @@ -27,6 +33,7 @@ class TestParser extends EventEmitter { super(); this.#opts = opts; + this.#failedTests = new Set(); this.#buildInstructions = new InstructionsList(); } @@ -66,6 +73,24 @@ class TestParser extends EventEmitter { const esmDecorator = f => f + `?rand=${rand}`; await readFiles(files, { esmDecorator, config: mochaOpts, eventBus }); + if (config.lastFailed.only) { + try { + this.#failedTests = new Set(); + const inputPaths = _.isArray(config.lastFailed.input) + ? config.lastFailed.input + : config.lastFailed.input.split(",").map(v => v.trim()); + for (const inputPath of inputPaths) { + for (const test of await fs.readJSON(inputPath)) { + this.#failedTests.add(getFailedTestId(test)); + } + } + } catch { + logger.warn( + `Could not read failed tests data at ${config.lastFailed.input}. Running all tests instead`, + ); + } + } + revertTransformHook(); } @@ -113,6 +138,12 @@ class TestParser extends EventEmitter { treeBuilder.addTestFilter(test => grep.test(test.fullTitle())); } + if (config.lastFailed && config.lastFailed.only && this.#failedTests.size) { + treeBuilder.addTestFilter(({ fullTitle, ...rest }) => { + return this.#failedTests.has(getFailedTestId({ fullTitle: fullTitle(), ...rest })); + }); + } + const rootSuite = treeBuilder.applyFilters().getRootSuite(); const tests = rootSuite.getTests(); diff --git a/src/testplane.ts b/src/testplane.ts index 2307f0ddc..7fc00cf5a 100644 --- a/src/testplane.ts +++ b/src/testplane.ts @@ -1,5 +1,6 @@ import { CommanderStatic } from "@gemini-testing/commander"; import _ from "lodash"; +import fs from "fs-extra"; import { Stats as RunnerStats } from "./stats"; import { BaseTestplane } from "./base-testplane"; import { MainRunner as NodejsEnvRunner } from "./runner"; @@ -16,7 +17,7 @@ import logger from "./utils/logger"; import { isRunInNodeJsEnv } from "./utils/config"; import { initDevServer } from "./dev-server"; import { ConfigInput } from "./config/types"; -import { MasterEventHandler, Test } from "./types"; +import { MasterEventHandler, Test, TestResult } from "./types"; interface RunOpts { browsers: string[]; @@ -37,9 +38,16 @@ interface RunOpts { devtools: boolean; } +export type FailedListItem = { + browserVersion?: string; + browserId?: string; + fullTitle: string; +}; + interface ReadTestsOpts extends Pick { silent: boolean; ignore: string | string[]; + failed: FailedListItem[]; } export interface Testplane { @@ -50,12 +58,14 @@ export interface Testplane { export class Testplane extends BaseTestplane { protected failed: boolean; + protected failedList: FailedListItem[]; protected runner: NodejsEnvRunner | BrowserEnvRunner | null; constructor(config?: string | ConfigInput) { super(config); this.failed = false; + this.failedList = []; this.runner = null; } @@ -101,7 +111,13 @@ export class Testplane extends BaseTestplane { ); this.runner = runner; - this.on(MasterEvents.TEST_FAIL, () => this._fail()).on(MasterEvents.ERROR, (err: Error) => this.halt(err)); + this.on(MasterEvents.TEST_FAIL, res => { + this._fail(); + this._addFailedTest(res); + }); + this.on(MasterEvents.ERROR, (err: Error) => this.halt(err)); + + this.on(MasterEvents.RUNNER_END, async () => await this._saveFailed()); await initReporters(reporters, this); @@ -119,6 +135,10 @@ export class Testplane extends BaseTestplane { return !this.isFailed(); } + protected async _saveFailed(): Promise { + await fs.outputJSON(this._config.lastFailed.output, this.failedList); // No spaces because users usually don't need to read it + } + protected async _readTests( testPaths: string[] | TestCollection, opts: Partial, @@ -169,6 +189,14 @@ export class Testplane extends BaseTestplane { this.failed = true; } + protected _addFailedTest(result: TestResult): void { + this.failedList.push({ + fullTitle: result.fullTitle(), + browserId: result.browserId, + browserVersion: result.browserVersion, + }); + } + isWorker(): boolean { return false; } diff --git a/test/src/config/options.js b/test/src/config/options.js index 4ab17b9d4..360ba8aea 100644 --- a/test/src/config/options.js +++ b/test/src/config/options.js @@ -463,6 +463,143 @@ describe("config options", () => { }); }); + describe("lastFailed", () => { + describe("only", () => { + it("should throw error if only is not a boolean", () => { + const readConfig = { + lastFailed: { + only: "String", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.only" must be a boolean'); + }); + }); + + describe("input", () => { + it("should throw error if input is not a string", () => { + const readConfig = { + lastFailed: { + input: false, + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" must be a string or an array'); + }); + + it("should throw error if input is a string without .json at the end", () => { + const readConfig = { + lastFailed: { + input: "string", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" must have .json extension'); + }); + + it("should not throw error if input is a string with .json at the end", () => { + const readConfig = { + lastFailed: { + input: "string.json", + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); + + it("should throw error if input is an array that contains a string without .json at the end", () => { + const readConfig = { + lastFailed: { + input: ["string.json", "string"], + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.input" elements must have .json extension'); + }); + + it("should not throw error if input is an array that contains only strings with .json at the end", () => { + const readConfig = { + lastFailed: { + input: ["string.json"], + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); + }); + + describe("output", () => { + it("should throw error if output is not a string", () => { + const readConfig = { + lastFailed: { + output: false, + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.output" must be a string'); + }); + + it("should throw error if output is a string without .json at the end", () => { + const readConfig = { + lastFailed: { + output: "string", + }, + }; + + Config.read.returns(readConfig); + + assert.throws(() => createConfig(), Error, '"lastFailed.output" must have .json extension'); + }); + + it("should not throw error if output is a string with .json at the end", () => { + const readConfig = { + lastFailed: { + output: "string.json", + }, + }; + + Config.read.returns(readConfig); + + assert.doesNotThrow(() => createConfig()); + }); + }); + + it("should set default lastFailed option if it does not set in config file", () => { + const config = createConfig(); + + assert.deepEqual(config.lastFailed, defaults.lastFailed); + }); + + it("should override lastFailed option", () => { + const newValue = { + input: "some-path.json", + output: "some-path.json", + only: true, + }; + const readConfig = { lastFailed: newValue }; + + Config.read.returns(readConfig); + + const config = createConfig(); + + assert.deepEqual(config.lastFailed, newValue); + }); + }); + describe("prepareEnvironment", () => { it("should throw error if prepareEnvironment is not a null or function", () => { const readConfig = { prepareEnvironment: "String" }; diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index eb3f7d6b0..d52ac9540 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -14,6 +14,7 @@ const proxyquire = require("proxyquire").noCallThru(); const path = require("path"); const { EventEmitter } = require("events"); const _ = require("lodash"); +const fs = require("fs-extra"); const { NEW_BUILD_INSTRUCTION } = TestReaderEvents; @@ -40,6 +41,7 @@ describe("test-reader/test-parser", () => { "./test-transformer": { setupTransformHook }, }).TestParser; + sandbox.stub(fs, "readJSON").resolves([]); sandbox.stub(InstructionsList.prototype, "push").returnsThis(); sandbox.stub(InstructionsList.prototype, "exec").returns(new Suite()); }); @@ -320,6 +322,68 @@ describe("test-reader/test-parser", () => { }); }); + describe("failed tests", () => { + it("should read if config.lastFailed.only is set", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "file.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "file.json"); + }); + + it("should read from one file if config.lastFailed.input is a string", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + }); + + it("should read from multiple files if config.lastFailed.input is a string with commas", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json, failed2.json", + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + assert.calledWith(fs.readJSON, "failed2.json"); + }); + + it("should read from multiple files if config.lastFailed.input is an array", async () => { + const config = makeConfigStub({ + lastFailed: { + only: true, + input: ["failed.json", "failed2.json"], + }, + }); + + await loadFiles_({ config }); + + assert.calledWith(fs.readJSON, "failed.json"); + assert.calledWith(fs.readJSON, "failed2.json"); + }); + + it("should not read if config.lastFailed.only is not set", async () => { + await loadFiles_(); + + assert.notCalled(fs.readJSON); + }); + }); + describe("read files", () => { it("should read passed files", async () => { const files = ["foo/bar", "baz/qux"]; @@ -475,13 +539,14 @@ describe("test-reader/test-parser", () => { }); describe("parse", () => { - const parse_ = async ({ files, browserId, config, grep } = {}) => { + const parse_ = async ({ files, browserId, config, grep } = {}, loadFilesConfig) => { + loadFilesConfig = loadFilesConfig || makeConfigStub(); config = _.defaults(config, { desiredCapabilities: {}, }); const parser = new TestParser(); - await parser.loadFiles([], makeConfigStub()); + await parser.loadFiles([], loadFilesConfig); return parser.parse(files || [], { browserId, config, grep }); }; @@ -494,6 +559,51 @@ describe("test-reader/test-parser", () => { sandbox.stub(Suite.prototype, "getTests").returns([]); }); + describe("addTestFilter", () => { + it("should not call if config.lastFailed.only is not set", async () => { + await parse_(); + + assert.notCalled(TreeBuilder.prototype.addTestFilter); + }); + + it("should call addTestFilter if config.lastFailed.only is set", async () => { + const tests = [ + { + fullTitle: () => "title", + browserId: "chrome", + browserVersion: "1", + }, + { + fullTitle: () => "title2", + browserId: "chrome", + browserVersion: "1", + }, + ]; + + fs.readJSON.resolves([ + { + fullTitle: tests[0].fullTitle(), + browserId: tests[0].browserId, + browserVersion: tests[0].browserVersion, + }, + ]); + + const config = makeConfigStub({ + lastFailed: { + only: true, + input: "failed.json", + }, + }); + + await parse_({ config }, config); + + const filter = TreeBuilder.prototype.addTestFilter.lastCall.args[0]; + + assert.equal(filter(tests[0]), true); + assert.equal(filter(tests[1]), false); + }); + }); + it("should execute build instructions", async () => { await parse_(); diff --git a/test/src/testplane.js b/test/src/testplane.js index f298e30a4..8c4f826fd 100644 --- a/test/src/testplane.js +++ b/test/src/testplane.js @@ -1,6 +1,7 @@ "use strict"; const _ = require("lodash"); +const fs = require("fs-extra"); const { EventEmitter } = require("events"); const pluginsLoader = require("plugins-loader"); const Promise = require("bluebird"); @@ -49,6 +50,8 @@ describe("testplane", () => { sandbox.stub(RuntimeConfig, "getInstance").returns({ extend: sandbox.stub() }); sandbox.stub(TestReader.prototype, "read").resolves(); sandbox.stub(RunnerStats, "create"); + sandbox.stub(fs, "readJSON").resolves([]); + sandbox.stub(fs, "outputJSON").resolves(); initReporters = sandbox.stub().resolves(); signalHandler = new AsyncEmitter(); @@ -217,7 +220,10 @@ describe("testplane", () => { describe("repl mode", () => { it("should not reset test timeout to 0 if run not in repl", async () => { mkNodejsEnvRunner_(); - const testplane = mkTestplane_({ system: { mochaOpts: { timeout: 100500 } } }); + const testplane = mkTestplane_({ + lastFailed: { only: false }, + system: { mochaOpts: { timeout: 100500 } }, + }); await testplane.run([], { replMode: { enabled: false } }); @@ -226,7 +232,10 @@ describe("testplane", () => { it("should reset test timeout to 0 if run in repl", async () => { mkNodejsEnvRunner_(); - const testplane = mkTestplane_({ system: { mochaOpts: { timeout: 100500 } } }); + const testplane = mkTestplane_({ + lastFailed: { only: false }, + system: { mochaOpts: { timeout: 100500 } }, + }); await testplane.run([], { replMode: { enabled: true } }); @@ -318,7 +327,12 @@ describe("testplane", () => { await runTestplane(testPaths, { browsers, grep, sets, replMode }); - assert.calledOnceWith(Testplane.prototype.readTests, testPaths, { browsers, grep, sets, replMode }); + assert.calledOnceWith(Testplane.prototype.readTests, testPaths, { + browsers, + grep, + sets, + replMode, + }); }); it("should accept test collection as first parameter", async () => { @@ -384,7 +398,12 @@ describe("testplane", () => { }); it('should return "false" if there are failed tests', () => { - mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.TEST_FAIL)); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; + mkNodejsEnvRunner_(runner => runner.emit(RunnerEvents.TEST_FAIL, results)); return runTestplane().then(success => assert.isFalse(success)); }); @@ -397,6 +416,27 @@ describe("testplane", () => { return testplane.run().then(() => assert.calledOnceWith(testplane.halt, err)); }); + + it("should save failed tests", async () => { + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; + mkNodejsEnvRunner_(runner => { + runner.emit(RunnerEvents.TEST_FAIL, results), runner.emit(RunnerEvents.RUNNER_END); + }); + + await runTestplane(); + + assert.calledWith(fs.outputJSON, "some-other-path", [ + { + fullTitle: results.fullTitle(), + browserId: results.browserId, + browserVersion: "1", + }, + ]); + }); }); describe("should passthrough", () => { @@ -467,6 +507,11 @@ describe("testplane", () => { it("all runner events with passed event data", () => { const runner = mkNodejsEnvRunner_(); const testplane = mkTestplane_(); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; const omitEvents = ["EXIT", "NEW_BROWSER", "UPDATE_REFERENCE"]; return testplane.run().then(() => { @@ -474,9 +519,9 @@ describe("testplane", () => { const spy = sinon.spy().named(`${name} handler`); testplane.on(event, spy); - runner.emit(event, "some-data"); + runner.emit(event, results); - assert.calledWith(spy, "some-data"); + assert.calledWith(spy, results); }); }); }); @@ -761,8 +806,14 @@ describe("testplane", () => { it('should return "true" after some test fail', () => { const testplane = mkTestplane_(); + const results = { + fullTitle: () => "Title", + browserId: "chrome", + browserVersion: "1", + }; + mkNodejsEnvRunner_(runner => { - runner.emit(RunnerEvents.TEST_FAIL); + runner.emit(RunnerEvents.TEST_FAIL, results); assert.isTrue(testplane.isFailed()); }); diff --git a/test/utils.js b/test/utils.js index 79748fba4..581469f2f 100644 --- a/test/utils.js +++ b/test/utils.js @@ -30,6 +30,11 @@ function makeConfigStub(opts = {}) { testRunEnv: NODEJS_TEST_RUN_ENV, }, sets: {}, + lastFailed: { + only: false, + input: "some-path", + output: "some-other-path", + }, }); const config = { @@ -38,6 +43,7 @@ function makeConfigStub(opts = {}) { system: opts.system, sets: opts.sets, configPath: opts.configPath, + lastFailed: opts.lastFailed, }; opts.browsers.forEach(browserId => { From cd2763d0894b2c395bf0332e72dddc2e59dae5ef Mon Sep 17 00:00:00 2001 From: Kabir-Ivan <110986400+Kabir-Ivan@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:08:48 +0300 Subject: [PATCH 12/15] fix: correctly require plugins with yarn+pnp (#978) --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index e6d0f2c15..12fdc8249 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,7 +36,7 @@ "micromatch": "4.0.5", "mocha": "10.2.0", "modern-node-polyfills": "1.0.0", - "plugins-loader": "1.3.2", + "plugins-loader": "1.3.3", "png-validator": "1.1.0", "sharp": "0.32.6", "sizzle": "2.3.6", @@ -13426,9 +13426,9 @@ } }, "node_modules/plugins-loader": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.2.tgz", - "integrity": "sha512-LPYpqoP99V4g4dkrlAwS72V5EHXD1hWcx2HzA9mlJ13RyugR6TaPLSX37hNHwQxG0YTkWmoiWPhMnYqnsEawZQ==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.3.tgz", + "integrity": "sha512-ByG2rWfPgLCZnv82c+Whth4jpUDklzrrAMheGKCRTZOnrR4t+rasnonjdVYqdDgbJ3EaGSQiujI96fyxiGAp0g==", "dependencies": { "lodash": "^4.16.4" } @@ -27361,9 +27361,9 @@ "dev": true }, "plugins-loader": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.2.tgz", - "integrity": "sha512-LPYpqoP99V4g4dkrlAwS72V5EHXD1hWcx2HzA9mlJ13RyugR6TaPLSX37hNHwQxG0YTkWmoiWPhMnYqnsEawZQ==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.3.3.tgz", + "integrity": "sha512-ByG2rWfPgLCZnv82c+Whth4jpUDklzrrAMheGKCRTZOnrR4t+rasnonjdVYqdDgbJ3EaGSQiujI96fyxiGAp0g==", "requires": { "lodash": "^4.16.4" } diff --git a/package.json b/package.json index a17be4bd9..7d1be82c9 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ "micromatch": "4.0.5", "mocha": "10.2.0", "modern-node-polyfills": "1.0.0", - "plugins-loader": "1.3.2", + "plugins-loader": "1.3.3", "png-validator": "1.1.0", "sharp": "0.32.6", "sizzle": "2.3.6", From 7060c36fc3ad21ec68be93b91aa5f3613b6d6141 Mon Sep 17 00:00:00 2001 From: y-infra Date: Mon, 29 Jul 2024 12:03:42 +0000 Subject: [PATCH 13/15] chore(release): 8.19.0 --- CHANGELOG.md | 12 ++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd065c085..0d59800d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [8.19.0](https://github.com/gemini-testing/testplane/compare/v8.18.1...v8.19.0) (2024-07-29) + + +### Features + +* add config option to rerun failed tests ([#973](https://github.com/gemini-testing/testplane/issues/973)) ([36aa516](https://github.com/gemini-testing/testplane/commit/36aa51657c0807f1d3474084b336b02270bba9d7)) + + +### Bug Fixes + +* correctly require plugins with yarn+pnp ([#978](https://github.com/gemini-testing/testplane/issues/978)) ([cd2763d](https://github.com/gemini-testing/testplane/commit/cd2763d0894b2c395bf0332e72dddc2e59dae5ef)) + ### [8.18.1](https://github.com/gemini-testing/testplane/compare/v8.18.0...v8.18.1) (2024-07-24) diff --git a/package-lock.json b/package-lock.json index 12fdc8249..6897849a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testplane", - "version": "8.18.1", + "version": "8.19.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "testplane", - "version": "8.18.1", + "version": "8.19.0", "license": "MIT", "dependencies": { "@babel/code-frame": "7.24.2", diff --git a/package.json b/package.json index 7d1be82c9..845460e63 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testplane", - "version": "8.18.1", + "version": "8.19.0", "description": "Tests framework based on mocha and wdio", "main": "build/src/index.js", "files": [ From b89c5b18e9ee64a5af7679e35592c3048f57fa06 Mon Sep 17 00:00:00 2001 From: Kabir-Ivan <110986400+Kabir-Ivan@users.noreply.github.com> Date: Tue, 30 Jul 2024 11:15:30 +0300 Subject: [PATCH 14/15] fix: fix failed tests rerun (#980) --- src/test-reader/test-parser.js | 10 ++++++++-- test/src/test-reader/test-parser.js | 12 ++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/test-reader/test-parser.js b/src/test-reader/test-parser.js index 5cce762f9..857253dcf 100644 --- a/src/test-reader/test-parser.js +++ b/src/test-reader/test-parser.js @@ -139,8 +139,14 @@ class TestParser extends EventEmitter { } if (config.lastFailed && config.lastFailed.only && this.#failedTests.size) { - treeBuilder.addTestFilter(({ fullTitle, ...rest }) => { - return this.#failedTests.has(getFailedTestId({ fullTitle: fullTitle(), ...rest })); + treeBuilder.addTestFilter(test => { + return this.#failedTests.has( + getFailedTestId({ + fullTitle: test.fullTitle(), + browserId: test.browserId, + browserVersion: test.browserVersion, + }), + ); }); } diff --git a/test/src/test-reader/test-parser.js b/test/src/test-reader/test-parser.js index d52ac9540..f901190e2 100644 --- a/test/src/test-reader/test-parser.js +++ b/test/src/test-reader/test-parser.js @@ -568,16 +568,16 @@ describe("test-reader/test-parser", () => { it("should call addTestFilter if config.lastFailed.only is set", async () => { const tests = [ - { - fullTitle: () => "title", + new Test({ + title: "title", browserId: "chrome", browserVersion: "1", - }, - { - fullTitle: () => "title2", + }), + new Test({ + title: "title2", browserId: "chrome", browserVersion: "1", - }, + }), ]; fs.readJSON.resolves([ From e78a2c257d6427876e5d61cce835fb1cc8f6ffc6 Mon Sep 17 00:00:00 2001 From: y-infra Date: Tue, 30 Jul 2024 08:34:41 +0000 Subject: [PATCH 15/15] chore(release): 8.19.1 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d59800d2..949ed45d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.19.1](https://github.com/gemini-testing/testplane/compare/v8.19.0...v8.19.1) (2024-07-30) + + +### Bug Fixes + +* fix failed tests rerun ([#980](https://github.com/gemini-testing/testplane/issues/980)) ([b89c5b1](https://github.com/gemini-testing/testplane/commit/b89c5b18e9ee64a5af7679e35592c3048f57fa06)) + ## [8.19.0](https://github.com/gemini-testing/testplane/compare/v8.18.1...v8.19.0) (2024-07-29) diff --git a/package-lock.json b/package-lock.json index 6897849a3..c2d9bee33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "testplane", - "version": "8.19.0", + "version": "8.19.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "testplane", - "version": "8.19.0", + "version": "8.19.1", "license": "MIT", "dependencies": { "@babel/code-frame": "7.24.2", diff --git a/package.json b/package.json index 845460e63..3d85d9f05 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "testplane", - "version": "8.19.0", + "version": "8.19.1", "description": "Tests framework based on mocha and wdio", "main": "build/src/index.js", "files": [