Merge branch 'master' into sipayRT.site-url-for-npm

CHANGELOG.md

@@ -2,6 +2,60 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.19.1](https://github.com/gemini-testing/testplane/compare/v8.19.0...v8.19.1) (2024-07-30)
+
+
+### Bug Fixes
+
+* fix failed tests rerun ([#980](https://github.com/gemini-testing/testplane/issues/980)) ([b89c5b1](https://github.com/gemini-testing/testplane/commit/b89c5b18e9ee64a5af7679e35592c3048f57fa06))
+
+## [8.19.0](https://github.com/gemini-testing/testplane/compare/v8.18.1...v8.19.0) (2024-07-29)
+
+
+### Features
+
+* add config option to rerun failed tests ([#973](https://github.com/gemini-testing/testplane/issues/973)) ([36aa516](https://github.com/gemini-testing/testplane/commit/36aa51657c0807f1d3474084b336b02270bba9d7))
+
+
+### Bug Fixes
+
+* correctly require plugins with yarn+pnp ([#978](https://github.com/gemini-testing/testplane/issues/978)) ([cd2763d](https://github.com/gemini-testing/testplane/commit/cd2763d0894b2c395bf0332e72dddc2e59dae5ef))
+
+### [8.18.1](https://github.com/gemini-testing/testplane/compare/v8.18.0...v8.18.1) (2024-07-24)
+
+
+### Bug Fixes
+
+* correctly require plugins with yarn+pnp ([#977](https://github.com/gemini-testing/testplane/issues/977)) ([3624e01](https://github.com/gemini-testing/testplane/commit/3624e01f5856a074edadcd897e8be54c0b16f934))
+
+## [8.18.0](https://github.com/gemini-testing/testplane/compare/v8.17.2...v8.18.0) (2024-07-22)
+
+
+### Features
+
+* add "pid" to log info with test result ([c8ecd60](https://github.com/gemini-testing/testplane/commit/c8ecd601713088e9cd09ae7435d942603d0240f0))
+
+### [8.17.2](https://github.com/gemini-testing/testplane/compare/v8.17.1...v8.17.2) (2024-07-17)
+
+
+### Bug Fixes
+
+* support error snippets nested browser commands ([3c9ce6b](https://github.com/gemini-testing/testplane/commit/3c9ce6becccf1f8940dbd9c92204b15c4efa184a))
+
+### [8.17.1](https://github.com/gemini-testing/testplane/compare/v8.17.0...v8.17.1) (2024-07-16)
+
+
+### Bug Fixes
+
+* move escape-string-regexp to deps ([#974](https://github.com/gemini-testing/testplane/issues/974)) ([abb14a0](https://github.com/gemini-testing/testplane/commit/abb14a08b724f88c0c292ef323ed82680157ff9d))
+
+## [8.17.0](https://github.com/gemini-testing/testplane/compare/v8.16.0...v8.17.0) (2024-07-10)
+
+
+### Features
+
+* look for configs in format testplane.config.{ts,js,cts,cjs} ([1f08b88](https://github.com/gemini-testing/testplane/commit/1f08b889ef2605cbb0c2f5ff5257c4e5c392d7d8))
+
 ## [8.16.0](https://github.com/gemini-testing/testplane/compare/v8.15.0...v8.16.0) (2024-07-10)
 
 

docs/config.md

@@ -73,6 +73,7 @@
   - [List of useful plugins](#list-of-useful-plugins)
 - [prepareBrowser](#preparebrowser)
 - [prepareEnvironment](#prepareenvironment)
+- [lastFailed](#lastfailed)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -731,3 +732,14 @@ Full list of parameters:
      - waitServerTimeout (optional) `Number` - timeout to wait for server to be ready (ms). 60_000 by default
      - probeRequestTimeout (optional) `Number` - one request timeout (ms), after which request will be aborted. 10_000 by default
      - probeRequestInterval (optional) `Number` - interval between ready probe requests (ms). 1_000 by default
+
+### lastFailed
+Allows you to run only tests that failed on the last run. Disabled by default - it means run all tests, but the file with the failed tests is always written.
+
+```js
+lastFailed: {
+    only: true, // true means run only failed, false - run all (default: false)
+    input: ['.testplane/failed.json', '.testplane/failed2.json'], // File/files to read failed tests list from (default: '.testplane/failed.json')
+    output: '.testplane/failed.json', // File to write failed tests list to (default: '.testplane/failed.json')
+}
+```

package.json

@@ -1,6 +1,6 @@
 {
   "name": "testplane",
-  "version": "8.16.0",
+  "version": "8.19.1",
   "description": "Tests framework based on mocha and wdio",
   "main": "build/src/index.js",
   "files": [
@@ -65,6 +65,7 @@
     "debug": "2.6.9",
     "devtools": "8.21.0",
     "error-stack-parser": "2.1.4",
+    "escape-string-regexp": "1.0.5",
     "expect-webdriverio": "3.5.3",
     "fastq": "1.13.0",
     "fs-extra": "5.0.0",
@@ -77,7 +78,7 @@
     "micromatch": "4.0.5",
     "mocha": "10.2.0",
     "modern-node-polyfills": "1.0.0",
-    "plugins-loader": "1.3.1",
+    "plugins-loader": "1.3.3",
     "png-validator": "1.1.0",
     "sharp": "0.32.6",
     "sizzle": "2.3.6",
@@ -131,7 +132,6 @@
     "copyfiles": "2.4.1",
     "doctoc": "2.2.0",
     "esbuild": "0.18.20",
-    "escape-string-regexp": "1.0.5",
     "eslint": "8.25.0",
     "eslint-config-gemini-testing": "2.8.0",
     "eslint-config-prettier": "8.7.0",

src/browser/stacktrace/index.ts

@@ -1,4 +1,4 @@
-import { ShallowStackFrames, applyStackFrames, captureRawStackFrames } from "./utils";
+import { ShallowStackFrames, applyStackTraceIfBetter, captureRawStackFrames } from "./utils";
 import { getBrowserCommands, getElementCommands } from "../history/commands";
 import { runWithHooks } from "../history/utils";
 
@@ -15,7 +15,7 @@ export const runWithStacktraceHooks = ({
 }): ReturnType<typeof fn> => {
     const frames = captureRawStackFrames(stackFilterFunc || runWithStacktraceHooks);
 
-    if (stackFrames.isNested(frames)) {
+    if (stackFrames.areInternal(frames)) {
         return fn();
     }
 
@@ -25,7 +25,7 @@ export const runWithStacktraceHooks = ({
         before: () => stackFrames.enter(key, frames),
         fn,
         after: () => stackFrames.leave(key),
-        error: (err: Error) => applyStackFrames(err, frames),
+        error: (err: Error) => applyStackTraceIfBetter(err, frames),
     });
 };
 

src/browser/stacktrace/utils.ts

@@ -1,6 +1,8 @@
+import _ from "lodash";
 import ErrorStackParser from "error-stack-parser";
 import type { SetRequired } from "type-fest";
 import logger from "../../utils/logger";
+import { softFileURLToPath } from "../../utils/fs";
 import { WDIO_IGNORED_STACK_FUNCTIONS, WDIO_STACK_TRACE_LIMIT } from "./constants";
 
 export type RawStackFrames = string;
@@ -48,7 +50,7 @@ const getErrorRawStackFrames = (e: ErrorWithStack): RawStackFrames => {
     return e.stack.slice(errorMessageEndsStackIndex + 1);
 };
 
-export function captureRawStackFrames(filterFunc?: (...args: unknown[]) => unknown): RawStackFrames {
+export const captureRawStackFrames = (filterFunc?: (...args: unknown[]) => unknown): RawStackFrames => {
     const savedStackTraceLimit = Error.stackTraceLimit;
     const targetObj = {} as { stack: RawStackFrames };
 
@@ -59,19 +61,90 @@ export function captureRawStackFrames(filterFunc?: (...args: unknown[]) => unkno
     const rawFramesPosition = targetObj.stack.indexOf("\n") + 1; // crop out error message
 
     return targetObj.stack.slice(rawFramesPosition);
-}
+};
+
+/**
+ * @description
+ * Rank values:
+ *
+ * 0: Can't extract code snippet; useless
+ *
+ * 1: WebdriverIO internals: Better than nothing
+ *
+ * 2: Project internals: Better than WebdriverIO internals, but worse, than user code part
+ *
+ * 3: User code: Best choice
+ */
+export const FRAME_RELEVANCE: Record<string, { value: number; matcher: (fileName: string) => boolean }> = {
+    repl: { value: 0, matcher: fileName => /^REPL\d+$/.test(fileName) },
+    nodeInternals: { value: 0, matcher: fileName => /^node:[a-zA-Z\-_]/.test(fileName) },
+    wdioInternals: { value: 1, matcher: fileName => fileName.includes("/node_modules/webdriverio/") },
+    projectInternals: { value: 2, matcher: fileName => fileName.includes("/node_modules/") },
+    userCode: { value: 3, matcher: () => true },
+} as const;
+
+export const getFrameRelevance = (frame: StackFrame): number => {
+    if ([frame.fileName, frame.lineNumber, frame.columnNumber].some(_.isUndefined)) {
+        return 0;
+    }
+
+    const fileName: string = softFileURLToPath(frame.fileName!);
+
+    for (const factor in FRAME_RELEVANCE) {
+        if (FRAME_RELEVANCE[factor].matcher(fileName)) {
+            return FRAME_RELEVANCE[factor].value;
+        }
+    }
+
+    return 0;
+};
 
-export function applyStackFrames(error: Error, frames: RawStackFrames): Error {
+const getStackTraceRelevance = (error: Error): number => {
+    const framesParsed = ErrorStackParser.parse(error);
+
+    return framesParsed.reduce((maxRelevance, frame) => {
+        return Math.max(maxRelevance, getFrameRelevance(frame));
+    }, 0);
+};
+
+const createErrorWithStack = (stack: RawStackFrames, errorMessage = ""): Error => {
+    const newError = new Error(errorMessage);
+
+    newError.stack = getErrorTitle(newError) + "\n" + stack;
+
+    return newError;
+};
+
+const applyStackTrace = (error: Error, stack: RawStackFrames): Error => {
     if (!error || !error.message) {
         return error;
     }
 
-    error.stack = getErrorTitle(error) + "\n" + frames;
+    error.stack = getErrorTitle(error) + "\n" + stack;
 
     return error;
-}
+};
+
+export const applyStackTraceIfBetter = (error: Error, stack: RawStackFrames): Error => {
+    if (!error || !error.message) {
+        return error;
+    }
+
+    try {
+        const newStackTraceRelevance = getStackTraceRelevance(createErrorWithStack(stack));
+        const currentStackTraceRelevance = getStackTraceRelevance(error);
+
+        if (newStackTraceRelevance > currentStackTraceRelevance) {
+            applyStackTrace(error, stack);
+        }
+    } catch (err) {
+        logger.warn("Couldn't compare error stack traces");
+    }
+
+    return error;
+};
 
-export function filterExtraWdioFrames(error: Error): Error {
+export const filterExtraWdioFrames = (error: Error): Error => {
     if (!error || !error.message || !error.stack) {
         return error;
     }
@@ -103,13 +176,13 @@ export function filterExtraWdioFrames(error: Error): Error {
 
         const framesFiltered = rawFramesArr.filter((_, i) => shouldIncludeFrame(framesParsed[i])).join("\n");
 
-        return applyStackFrames(error, framesFiltered);
+        return applyStackTrace(error, framesFiltered);
     } catch (filterError) {
         logger.warn("Couldn't filter out wdio frames", filterError);
 
         return error;
     }
-}
+};
 
 export class ShallowStackFrames {
     private _framesMap: Map<string, string>;
@@ -132,13 +205,29 @@ export class ShallowStackFrames {
         this._framesMap.delete(key);
     }
 
-    isNested(childFrames: RawStackFrames): boolean {
+    private _getParentStackFrame(childFrames: RawStackFrames): RawStackFrames | null {
         for (const parentFrames of this._framesMap.values()) {
             if (childFrames.length !== parentFrames.length && childFrames.endsWith(parentFrames)) {
-                return true;
+                return parentFrames;
             }
         }
 
-        return false;
+        return null;
+    }
+
+    areInternal(childFrames: RawStackFrames): boolean {
+        const parentStackFrame = this._getParentStackFrame(childFrames);
+
+        if (!parentStackFrame) {
+            return false;
+        }
+
+        const isNodeModulesFrame = (frame: string): boolean => frame.includes("/node_modules/");
+        const isNodeInternalFrame = (frame: string): boolean => frame.includes(" (node:");
+
+        const extraFrames = childFrames.slice(0, childFrames.length - parentStackFrame.length);
+        const extraFramesArray = extraFrames.split("\n");
+
+        return extraFramesArray.every(frame => !frame || isNodeModulesFrame(frame) || isNodeInternalFrame(frame));
     }
 }

src/config/defaults.js

@@ -94,6 +94,11 @@ module.exports = {
     headless: null,
     isolation: null,
     testRunEnv: NODEJS_TEST_RUN_ENV,
+    lastFailed: {
+        only: false,
+        output: ".testplane/failed.json",
+        input: ".testplane/failed.json",
+    },
     devServer: {
         command: null,
         cwd: null,
@@ -113,4 +118,13 @@ module.exports = {
     passive: false,
 };
 
-module.exports.configPaths = [".testplane.conf.ts", ".testplane.conf.js", ".hermione.conf.ts", ".hermione.conf.js"];
+module.exports.configPaths = [
+    ".testplane.conf.ts",
+    ".testplane.conf.js",
+    "testplane.config.ts",
+    "testplane.config.js",
+    "testplane.config.cts",
+    "testplane.config.cjs",
+    ".hermione.conf.ts",
+    ".hermione.conf.js",
+];