feat: support generating static report for playwright (#505)

* test: add e2e tests for pwt

* feat: add playwright test-adapter

* feat: add playwright integration

* fix: update server host in GUI e2e tests

* fix: further advancements in static pwt report generation

* chore: rename test-adapter unit test to hermione

* test: implement unit tests for pwt test-adapter and e2e pwt fixture

* fix: add error name by default

* fix: review issues fixes, use pwt delimiter instead of space

* fix: display common page screenshot in pwt report when possible

* fix: various review issue fixes, p-queue, pwt fixture config, pwt separator

* fix: bring back old attempt computing

* test: fixing tests

* fix: review fixes, test fixes, fixing expand retries

* chore: update package-lock

* fix: get rid of replaceAll

* ci: install pwt browsers

* test: fix e2e

.circleci/config.yml

@@ -23,6 +23,9 @@ jobs:
             cd test/func/docker/browser-utils &&
             npm ci &&
             npm run install-chromium
+      - run:
+          name: Install Chromium for Playwright
+          command: npx playwright install chromium
 
       - run:
           name: Download Selenium

.eslintignore

@@ -11,3 +11,4 @@ hot
 /test/func/packages/*/plugin.js
 /hermione-report
 tmp
+**/playwright-report

.gitignore

@@ -22,6 +22,7 @@ sqlite.db
 .nyc_output
 tmp
 
+**/playwright-report
 hermione-report
 test/func/**/report
 test/func/**/report-backup

lib/common-utils.ts

@@ -100,9 +100,7 @@ export const isNoRefImageError = (error?: unknown): error is NoRefImageError =>
 };
 
 export const hasNoRefImageErrors = ({assertViewResults = []}: {assertViewResults?: AssertViewResult[]}): boolean => {
-    const noRefImageErrors = assertViewResults.filter((assertViewResult: AssertViewResult) => isNoRefImageError(assertViewResult));
-
-    return !isEmpty(noRefImageErrors);
+    return assertViewResults.some((assertViewResult: AssertViewResult) => isNoRefImageError(assertViewResult));
 };
 
 const hasFailedImages = (result: {imagesInfo?: ImageInfoFull[]}): boolean => {
@@ -130,7 +128,7 @@ export const hasDiff = (assertViewResults: AssertViewResult[]): boolean => {
     return assertViewResults.some((result) => isImageDiffError(result as {name?: string}));
 };
 
-export const isBase64Image = (image: ImageData | ImageBase64 | undefined): image is ImageBase64 => {
+export const isBase64Image = (image: ImageData | ImageBase64 | null | undefined): image is ImageBase64 => {
     return Boolean((image as ImageBase64 | undefined)?.base64);
 };
 

lib/constants/index.ts

@@ -2,8 +2,11 @@ export * from './browser';
 export * from './database';
 export * from './defaults';
 export * from './diff-modes';
+export * from './group-tests';
 export * from './paths';
+export * from './tests';
 export * from './plugin-events';
 export * from './save-formats';
 export * from './test-statuses';
+export * from './tool-names';
 export * from './view-modes';

lib/constants/tests.ts

@@ -0,0 +1 @@
+export const PWT_TITLE_DELIMITER = ' › ';

lib/constants/tool-names.ts

@@ -0,0 +1,4 @@
+export enum ToolName {
+    Hermione = 'hermione',
+    Playwright = 'playwright'
+}

lib/db-utils/server.ts

@@ -9,7 +9,7 @@ import NestedError from 'nested-error-stacks';
 import {StaticTestsTreeBuilder} from '../tests-tree-builder/static';
 import * as commonSqliteUtils from './common';
 import {isUrl, fetchFile, normalizeUrls, logger} from '../common-utils';
-import {DATABASE_URLS_JSON_NAME, DB_COLUMNS, LOCAL_DATABASE_NAME, TestStatus} from '../constants';
+import {DATABASE_URLS_JSON_NAME, DB_COLUMNS, LOCAL_DATABASE_NAME, TestStatus, ToolName} from '../constants';
 import {DbLoadResult, HandleDatabasesOptions} from './common';
 import {DbUrlsJsonData, RawSuitesRow, ReporterConfig} from '../types';
 import {Tree} from '../tests-tree-builder/base';
@@ -57,10 +57,10 @@ export async function mergeDatabases(srcDbPaths: string[], reportPath: string):
     }
 }
 
-export function getTestsTreeFromDatabase(dbPath: string): Tree {
+export function getTestsTreeFromDatabase(toolName: ToolName, dbPath: string): Tree {
     try {
         const db = new Database(dbPath, {readonly: true, fileMustExist: true});
-        const testsTreeBuilder = StaticTestsTreeBuilder.create();
+        const testsTreeBuilder = StaticTestsTreeBuilder.create({toolName});
 
         const suitesRows = (db.prepare(commonSqliteUtils.selectAllSuitesQuery())
             .raw()

lib/errors/index.ts

@@ -3,6 +3,7 @@ import {DiffOptions, ImageData} from '../types';
 import {ValueOf} from 'type-fest';
 
 export const ErrorName = {
+    GENERAL_ERROR: 'Error',
     IMAGE_DIFF: 'ImageDiffError',
     NO_REF_IMAGE: 'NoRefImageError',
     ASSERT_VIEW: 'AssertViewError'
@@ -20,13 +21,14 @@ export interface ImageDiffError {
     refImg: ImageData;
     diffClusters: CoordBounds[];
     diffBuffer?: ArrayBuffer;
+    diffImg?: ImageData;
 }
 
 export interface NoRefImageError {
     name: ErrorNames['NO_REF_IMAGE'];
     stateName: string;
     message: string;
-    stack: string;
+    stack?: string;
     currImg: ImageData;
-    refImg: ImageData;
+    refImg?: ImageData;
 }

lib/gui/tool-runner/index.js

@@ -19,6 +19,7 @@ const {getShortMD5} = require('../../common-utils');
 const {formatId, mkFullTitle, mergeDatabasesForReuse, filterByEqualDiffSizes} = require('./utils');
 const {getTestsTreeFromDatabase} = require('../../db-utils/server');
 const {formatTestResult} = require('../../server-utils');
+const {ToolName} = require('../../constants');
 
 module.exports = class ToolRunner {
     static create(paths, hermione, configs) {
@@ -280,7 +281,7 @@ module.exports = class ToolRunner {
         const dbPath = path.resolve(this._reportPath, LOCAL_DATABASE_NAME);
 
         if (await fs.pathExists(dbPath)) {
-            return getTestsTreeFromDatabase(dbPath);
+            return getTestsTreeFromDatabase(ToolName.Hermione, dbPath);
         }
 
         logger.warn(chalk.yellow(`Nothing to reuse in ${this._reportPath}: can not load data from ${DATABASE_URLS_JSON_NAME}`));

lib/image-handler.ts

@@ -15,10 +15,18 @@ import {
     ImageInfoFail,
     ImageInfoFull,
     ImagesSaver,
-    ImageSize
+    ImageInfoPageSuccess
 } from './types';
 import {ERROR, FAIL, PluginEvents, SUCCESS, TestStatus, UPDATED} from './constants';
-import {getError, getShortMD5, isImageDiffError, isNoRefImageError, logger, mkTestId} from './common-utils';
+import {
+    getError,
+    getShortMD5,
+    isBase64Image,
+    isImageDiffError,
+    isNoRefImageError,
+    logger,
+    mkTestId
+} from './common-utils';
 import {ImageDiffError} from './errors';
 import {cacheExpectedPaths, cacheAllImages, cacheDiffImages} from './image-cache';
 import {ReporterTestResult} from './test-adapter';
@@ -50,23 +58,38 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter {
         return _.get(_.find(assertViewResults, {stateName}), 'currImg');
     }
 
+    static getDiffImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined {
+        return _.get(_.find(assertViewResults, {stateName}), 'diffImg');
+    }
+
     static getRefImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined {
         return _.get(_.find(assertViewResults, {stateName}), 'refImg');
     }
 
-    static getScreenshot(testResult: ReporterTestResultPlain): ImageBase64 | undefined {
-        return testResult.error?.screenshot;
+    static getScreenshot(testResult: ReporterTestResultPlain): ImageBase64 | ImageData | null | undefined {
+        return testResult.screenshot;
     }
 
     getImagesFor(testResult: ReporterTestResultPlain, assertViewStatus: TestStatus, stateName?: string): ImageInfo | undefined {
         const refImg = ImageHandler.getRefImg(testResult.assertViewResults, stateName);
         const currImg = ImageHandler.getCurrImg(testResult.assertViewResults, stateName);
-        const errImg = ImageHandler.getScreenshot(testResult);
+
+        const pageImg = ImageHandler.getScreenshot(testResult);
 
         const {path: refPath} = this._getExpectedPath(testResult, stateName);
         const currPath = utils.getCurrentPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName});
         const diffPath = utils.getDiffPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName});
 
+        // Handling whole page common screenshots
+        if (!stateName && pageImg) {
+            return {
+                actualImg: {
+                    path: this._getImgFromStorage(currPath),
+                    size: pageImg.size
+                }
+            };
+        }
+
         if ((assertViewStatus === SUCCESS || assertViewStatus === UPDATED) && refImg) {
             const result: ImageInfo = {
                 expectedImg: {path: this._getImgFromStorage(refPath), size: refImg.size}
@@ -98,11 +121,11 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter {
             };
         }
 
-        if (assertViewStatus === ERROR) {
+        if (assertViewStatus === ERROR && currImg) {
             return {
                 actualImg: {
-                    path: testResult.state?.name ? this._getImgFromStorage(currPath) : '',
-                    size: (currImg?.size || errImg?.size) as ImageSize
+                    path: this._getImgFromStorage(currPath),
+                    size: currImg.size
                 }
             };
         }
@@ -112,7 +135,7 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter {
 
     getImagesInfo(testResult: ReporterTestResultPlain): ImageInfoFull[] {
         const imagesInfo: ImageInfoFull[] = testResult.assertViewResults?.map((assertResult): ImageInfoFull => {
-            let status: TestStatus, error: {message: string; stack: string;} | undefined;
+            let status: TestStatus, error: {message: string; stack?: string;} | undefined;
 
             if (testResult.isUpdated === true) {
                 status = UPDATED;
@@ -134,14 +157,21 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter {
             ) as ImageInfoFull;
         }) ?? [];
 
-        // common screenshot on test fail
+        // Common page screenshot
         if (ImageHandler.getScreenshot(testResult)) {
-            const errorImage = _.extend(
-                {status: ERROR, error: getError(testResult.error)},
-                this.getImagesFor(testResult, ERROR)
-            ) as ImageInfoError;
+            const error = getError(testResult.error);
 
-            imagesInfo.push(errorImage);
+            if (!_.isEmpty(error)) {
+                imagesInfo.push(_.extend(
+                    {status: ERROR, error},
+                    this.getImagesFor(testResult, ERROR)
+                ) as ImageInfoError);
+            } else {
+                imagesInfo.push(_.extend(
+                    {status: SUCCESS},
+                    this.getImagesFor(testResult, SUCCESS)
+                ) as ImageInfoPageSuccess);
+            }
         }
 
         return imagesInfo;
@@ -158,20 +188,22 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter {
             const destCurrPath = utils.getCurrentPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName});
             const srcCurrPath = ImageHandler.getCurrImg(testResult.assertViewResults, stateName)?.path;
 
-            const dstCurrPath = utils.getDiffPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName});
-            const srcDiffPath = path.resolve(tmp.tmpdir, dstCurrPath);
+            const destDiffPath = utils.getDiffPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName});
+            const srcDiffPath = ImageHandler.getDiffImg(assertViewResults, stateName)?.path ?? path.resolve(tmp.tmpdir, destDiffPath);
             const actions: unknown[] = [];
 
             if (!(assertResult instanceof Error)) {
                 actions.push(this._saveImg(srcRefPath, destRefPath));
             }
 
             if (isImageDiffError(assertResult)) {
-                await this._saveDiffInWorker(assertResult, srcDiffPath, worker);
+                if (!assertResult.diffImg) {
+                    await this._saveDiffInWorker(assertResult, srcDiffPath, worker);
+                }
 
                 actions.push(
                     this._saveImg(srcCurrPath, destCurrPath),
-                    this._saveImg(srcDiffPath, dstCurrPath)
+                    this._saveImg(srcDiffPath, destDiffPath)
                 );
 
                 if (!reusedReference) {
@@ -187,7 +219,7 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter {
         }));
 
         if (ImageHandler.getScreenshot(testResult)) {
-            await this._saveErrorScreenshot(testResult);
+            await this._savePageScreenshot(testResult);
         }
 
         await this.emitAsync(PluginEvents.TEST_SCREENSHOTS_SAVED, {
@@ -289,18 +321,24 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter {
         cacheDiffImages.set(hash, destPath);
     }
 
-    private async _saveErrorScreenshot(testResult: ReporterTestResultPlain): Promise<void> {
+    private async _savePageScreenshot(testResult: ReporterTestResultPlain): Promise<void> {
         const screenshot = ImageHandler.getScreenshot(testResult);
-        if (!screenshot?.base64) {
+        if (!(screenshot as ImageBase64)?.base64 && !(screenshot as ImageData)?.path) {
             logger.warn('Cannot save screenshot on reject');
 
             return Promise.resolve();
         }
 
         const currPath = utils.getCurrentPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir});
-        const localPath = path.resolve(tmp.tmpdir, currPath);
-        await utils.makeDirFor(localPath);
-        await fs.writeFile(localPath, new Buffer(screenshot.base64, 'base64'), 'base64');
+        let localPath: string;
+
+        if (isBase64Image(screenshot)) {
+            localPath = path.resolve(tmp.tmpdir, currPath);
+            await utils.makeDirFor(localPath);
+            await fs.writeFile(localPath, new Buffer(screenshot.base64, 'base64'), 'base64');
+        } else {
+            localPath = screenshot?.path as string;
+        }
 
         await this._saveImg(localPath, currPath);
     }

lib/image-store.ts

@@ -27,6 +27,6 @@ export class SqliteImageStore implements ImageStore {
         }, suitePathString, browserName);
 
         const imagesInfo: ImageInfoFull[] = imagesInfoResult && JSON.parse(imagesInfoResult[DB_COLUMNS.IMAGES_INFO as keyof Pick<LabeledSuitesRow, 'imagesInfo'>]) || [];
-        return imagesInfo.find(info => info.stateName === stateName);
+        return imagesInfo.find(info => (info as {stateName?: string}).stateName === stateName);
     }
 }

lib/plugin-adapter.ts

@@ -8,6 +8,7 @@ import * as utils from './server-utils';
 import {cliCommands} from './cli-commands';
 import {HtmlReporter} from './plugin-api';
 import {HtmlReporterApi, ReporterConfig, ReporterOptions} from './types';
+import {ToolName} from './constants';
 
 type PrepareFn = (hermione: Hermione & HtmlReporterApi, reportBuilder: StaticReportBuilder, config: ReporterConfig) => Promise<void>;
 
@@ -33,7 +34,7 @@ export class PluginAdapter {
     }
 
     addApi(): this {
-        this._hermione.htmlReporter = HtmlReporter.create(this._config);
+        this._hermione.htmlReporter = HtmlReporter.create(this._config, {toolName: ToolName.Hermione});
         return this;
     }
 

lib/plugin-api.ts

@@ -1,31 +1,44 @@
 import EventsEmitter2 from 'eventemitter2';
-import {PluginEvents} from './constants';
-import {downloadDatabases, mergeDatabases, getTestsTreeFromDatabase} from './db-utils/server';
+import {PluginEvents, ToolName} from './constants';
+import {downloadDatabases, getTestsTreeFromDatabase, mergeDatabases} from './db-utils/server';
 import {LocalImagesSaver} from './local-images-saver';
 import {version} from '../package.json';
 import {ImagesSaver, ReporterConfig, ReportsSaver} from './types';
 
 interface HtmlReporterValues {
+    toolName: ToolName;
     extraItems: Record<string, string>;
     metaInfoExtenders: Record<string, string>;
     imagesSaver: ImagesSaver;
     reportsSaver: ReportsSaver | null;
 }
 
+interface ReporterOptions {
+    toolName: ToolName;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type ParametersExceptFirst<F> = F extends (arg0: any, ...rest: infer R) => any ? R : never;
+
 export class HtmlReporter extends EventsEmitter2 {
     protected _config: ReporterConfig;
     protected _values: HtmlReporterValues;
     protected _version: string;
 
-    static create<T extends HtmlReporter>(this: new (config: ReporterConfig) => T, config: ReporterConfig): T {
-        return new this(config);
+    static create<T extends HtmlReporter>(
+        this: new (config: ReporterConfig, options?: Partial<ReporterOptions>) => T,
+        config: ReporterConfig,
+        options?: Partial<ReporterOptions>
+    ): T {
+        return new this(config, options);
     }
 
-    constructor(config: ReporterConfig) {
+    constructor(config: ReporterConfig, {toolName}: Partial<ReporterOptions> = {}) {
         super();
 
         this._config = config;
         this._values = {
+            toolName: toolName ?? ToolName.Hermione,
             extraItems: {},
             metaInfoExtenders: {},
             imagesSaver: LocalImagesSaver,
@@ -91,7 +104,7 @@ export class HtmlReporter extends EventsEmitter2 {
         return mergeDatabases(...args);
     }
 
-    getTestsTreeFromDatabase(...args: Parameters<typeof getTestsTreeFromDatabase>): ReturnType<typeof getTestsTreeFromDatabase> {
-        return getTestsTreeFromDatabase(...args);
+    getTestsTreeFromDatabase(...args: ParametersExceptFirst<typeof getTestsTreeFromDatabase>): ReturnType<typeof getTestsTreeFromDatabase> {
+        return getTestsTreeFromDatabase(this.values.toolName, ...args);
     }
 }