From f75a4b918792ee7d279656c62a8c6139ac95b202 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Fri, 15 Dec 2023 16:31:44 +0300 Subject: [PATCH 1/7] refactor: get rid of other test results formats and unnecessary methods --- hermione.ts | 17 ++- lib/common-utils.ts | 37 +++++- lib/constants/database.ts | 5 +- lib/db-utils/server.ts | 4 +- lib/gui/tool-runner/index.ts | 8 +- lib/gui/tool-runner/report-subscriber.ts | 12 +- lib/report-builder/gui.ts | 99 +++++--------- lib/report-builder/static.ts | 95 +------------ lib/server-utils.ts | 12 +- lib/sqlite-client.ts | 20 +-- lib/test-adapter/hermione.ts | 2 - lib/test-adapter/index.ts | 5 +- lib/test-adapter/reporter.ts | 15 +-- lib/test-adapter/sqlite.ts | 162 +++++++++++++++++++++++ lib/test-adapter/transformers/tree.ts | 27 ++++ lib/test-adapter/utils/index.ts | 15 ++- lib/tests-tree-builder/base.ts | 39 +++--- lib/tests-tree-builder/gui.ts | 8 +- lib/tests-tree-builder/static.ts | 44 ++---- lib/types.ts | 72 +++++----- playwright.ts | 17 +-- 21 files changed, 385 insertions(+), 330 deletions(-) create mode 100644 lib/test-adapter/sqlite.ts create mode 100644 lib/test-adapter/transformers/tree.ts diff --git a/hermione.ts b/hermione.ts index 385c3a700..f3788405e 100644 --- a/hermione.ts +++ b/hermione.ts @@ -93,7 +93,8 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport hermione.on(hermione.events.TEST_PASS, testResult => { promises.push(queue.add(async () => { const formattedResult = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder); - await reportBuilder.addSuccess(formattedResult); + + await reportBuilder.addTestResult(formattedResult); }).catch(reject)); }); @@ -103,8 +104,10 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); - await reportBuilder.addFail(formattedResult); - }).catch(reject)); + await reportBuilder.addTestResult(formattedResult); + }).catch((e) => { + reject(e); + })); }); hermione.on(hermione.events.TEST_FAIL, testResult => { @@ -113,15 +116,17 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); - await reportBuilder.addFail(formattedResult); - }).catch(reject)); + await reportBuilder.addTestResult(formattedResult); + }).catch((e) => { + reject(e); + })); }); hermione.on(hermione.events.TEST_PENDING, testResult => { promises.push(queue.add(async () => { const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder); - await reportBuilder.addSkipped(formattedResult); + await reportBuilder.addTestResult(formattedResult); }).catch(reject)); }); diff --git a/lib/common-utils.ts b/lib/common-utils.ts index 9fdd4628b..1988b43e4 100644 --- a/lib/common-utils.ts +++ b/lib/common-utils.ts @@ -1,13 +1,26 @@ import crypto from 'crypto'; -import {pick, isEmpty} from 'lodash'; +import {isEmpty, pick} from 'lodash'; import url from 'url'; import axios, {AxiosRequestConfig} from 'axios'; -import {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED, TestStatus} from './constants'; - -import {UNCHECKED, INDETERMINATE, CHECKED} from './constants/checked-statuses'; -import {ImageData, ImageBase64, ImageInfoFull, TestError, ImageInfoFail} from './types'; +import { + ERROR, + FAIL, + HERMIONE_TITLE_DELIMITER, + IDLE, PWT_TITLE_DELIMITER, + QUEUED, + RUNNING, + SKIPPED, + SUCCESS, + TestStatus, + ToolName, + UPDATED +} from './constants'; + +import {CHECKED, INDETERMINATE, UNCHECKED} from './constants/checked-statuses'; +import {ImageBase64, ImageData, ImageInfoFail, ImageInfoFull, TestError} from './types'; import {ErrorName, ImageDiffError, NoRefImageError} from './errors'; import {ReporterTestResult} from './test-adapter'; + export const getShortMD5 = (str: string): string => { return crypto.createHash('md5').update(str, 'ascii').digest('hex').substr(0, 7); }; @@ -217,3 +230,17 @@ export const isCheckboxChecked = (status: number): boolean => Number(status) === export const isCheckboxIndeterminate = (status: number): boolean => Number(status) === INDETERMINATE; export const isCheckboxUnchecked = (status: number): boolean => Number(status) === UNCHECKED; export const getToggledCheckboxState = (status: number): number => isCheckboxChecked(status) ? UNCHECKED : CHECKED; + +export const getTitleDelimiter = (toolName: ToolName): string => { + if (toolName === ToolName.Hermione) { + return HERMIONE_TITLE_DELIMITER; + } else if (toolName === ToolName.Playwright) { + return PWT_TITLE_DELIMITER; + } else { + return HERMIONE_TITLE_DELIMITER; + } +}; + +export function getDetailsFileName(testId: string, browserId: string, attempt: number): string { + return `${testId}-${browserId}_${Number(attempt) + 1}_${Date.now()}.json`; +} diff --git a/lib/constants/database.ts b/lib/constants/database.ts index cc9cd381b..3a71ea866 100644 --- a/lib/constants/database.ts +++ b/lib/constants/database.ts @@ -1,4 +1,4 @@ -import {ValueOf} from 'type-fest'; +import {Mutable, NameToIndexMap} from '../types'; // TODO: change to enums export const DB_TYPES = {int: 'INT', text: 'TEXT'} as const; @@ -40,7 +40,8 @@ export const DB_MAX_AVAILABLE_PAGE_SIZE = 65536; // helps to speed up queries export const DB_SUITES_TABLE_NAME = 'suites'; export const LOCAL_DATABASE_NAME = 'sqlite.db'; export const DATABASE_URLS_JSON_NAME = 'databaseUrls.json'; + export const DB_COLUMN_INDEXES = SUITES_TABLE_COLUMNS.reduce((acc: Record, {name}, index) => { acc[name] = index; return acc; -}, {}) as { [K in ValueOf]: number }; +}, {}) as NameToIndexMap>; diff --git a/lib/db-utils/server.ts b/lib/db-utils/server.ts index a93b65382..c2d8623d0 100644 --- a/lib/db-utils/server.ts +++ b/lib/db-utils/server.ts @@ -57,10 +57,10 @@ export async function mergeDatabases(srcDbPaths: string[], reportPath: string): } } -export function getTestsTreeFromDatabase(toolName: ToolName, dbPath: string): Tree { +export function getTestsTreeFromDatabase(toolName: ToolName, dbPath: string, baseHost: string): Tree { try { const db = new Database(dbPath, {readonly: true, fileMustExist: true}); - const testsTreeBuilder = StaticTestsTreeBuilder.create({toolName}); + const testsTreeBuilder = StaticTestsTreeBuilder.create({toolName, baseHost}); const suitesRows = (db.prepare(commonSqliteUtils.selectAllSuitesQuery()) .raw() diff --git a/lib/gui/tool-runner/index.ts b/lib/gui/tool-runner/index.ts index 6117dbcc0..5eed1cd92 100644 --- a/lib/gui/tool-runner/index.ts +++ b/lib/gui/tool-runner/index.ts @@ -187,7 +187,7 @@ export class ToolRunner { const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT, reportBuilder); - const formattedResult = await reportBuilder.addUpdated(formattedResultWithoutAttempt); + const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); updateResult.attempt = formattedResult.attempt; @@ -316,9 +316,9 @@ export class ToolRunner { this._tests[testId] = _.extend(test, {browserId}); if (test.pending) { - queue.add(async () => reportBuilder.addSkipped(formatTestResultUnsafe(test, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder))); + queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder))); } else { - queue.add(async () => reportBuilder.addIdle(formatTestResultUnsafe(test, IDLE, UNKNOWN_ATTEMPT, reportBuilder))); + queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, IDLE, UNKNOWN_ATTEMPT, reportBuilder))); } }); @@ -394,7 +394,7 @@ export class ToolRunner { const dbPath = path.resolve(this._reportPath, LOCAL_DATABASE_NAME); if (await fs.pathExists(dbPath)) { - return getTestsTreeFromDatabase(ToolName.Hermione, dbPath); + return getTestsTreeFromDatabase(ToolName.Hermione, dbPath, this._pluginConfig.baseHost); } logger.warn(chalk.yellow(`Nothing to reuse in ${this._reportPath}: can not load data from ${DATABASE_URLS_JSON_NAME}`)); diff --git a/lib/gui/tool-runner/report-subscriber.ts b/lib/gui/tool-runner/report-subscriber.ts index a307aaa1c..5edd00145 100644 --- a/lib/gui/tool-runner/report-subscriber.ts +++ b/lib/gui/tool-runner/report-subscriber.ts @@ -33,7 +33,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo queue.add(async () => { const formattedResultWithoutAttempt = formatTestResult(data as HermioneTestResult, RUNNING, UNKNOWN_ATTEMPT, reportBuilder); - const formattedResult = await reportBuilder.addRunning(formattedResultWithoutAttempt); + const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); return client.emit(ClientEvents.BEGIN_STATE, testBranch); @@ -44,7 +44,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo queue.add(async () => { const formattedResultWithoutAttempt = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder); - const formattedResult = await reportBuilder.addSuccess(formattedResultWithoutAttempt); + const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); @@ -57,7 +57,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); - const formattedResult = await reportBuilder.addRetry(formattedResultWithoutAttempt); + const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); @@ -70,9 +70,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); - const formattedResult = status === TestStatus.FAIL - ? await reportBuilder.addFail(formattedResultWithoutAttempt) - : await reportBuilder.addError(formattedResultWithoutAttempt); + const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); @@ -83,7 +81,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo queue.add(async () => { const formattedResultWithoutAttempt = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder); - const formattedResult = await reportBuilder.addSkipped(formattedResultWithoutAttempt); + const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); diff --git a/lib/report-builder/gui.ts b/lib/report-builder/gui.ts index 074406f41..a491be61f 100644 --- a/lib/report-builder/gui.ts +++ b/lib/report-builder/gui.ts @@ -1,16 +1,13 @@ -import * as _ from 'lodash'; -import {StaticReportBuilder} from './static'; +import _ from 'lodash'; +import {StaticReportBuilder, StaticReportBuilderOptions} from './static'; import {GuiTestsTreeBuilder, TestBranch, TestEqualDiffsData, TestRefUpdateData} from '../tests-tree-builder/gui'; -import { - IDLE, RUNNING, UPDATED, TestStatus, DB_COLUMNS, ToolName, HERMIONE_TITLE_DELIMITER -} from '../constants'; +import {UPDATED, DB_COLUMNS, ToolName, HERMIONE_TITLE_DELIMITER, SKIPPED} from '../constants'; import {ConfigForStaticFile, getConfigForStaticFile} from '../server-utils'; import {ReporterTestResult} from '../test-adapter'; -import {PreparedTestResult} from '../sqlite-client'; import {Tree, TreeImage} from '../tests-tree-builder/base'; -import {ImageInfoWithState, ReporterConfig} from '../types'; +import {ImageInfoFull, ImageInfoWithState, ReporterConfig} from '../types'; import {isUpdatedStatus} from '../common-utils'; -import {HtmlReporterValues} from '../plugin-api'; +import {HtmlReporter, HtmlReporterValues} from '../plugin-api'; import {SkipItem} from '../tests-tree-builder/static'; import {copyAndUpdate} from '../test-adapter/utils'; @@ -36,38 +33,13 @@ export class GuiReportBuilder extends StaticReportBuilder { private _skips: SkipItem[]; private _apiValues?: HtmlReporterValues; - constructor(...args: ConstructorParameters) { - super(...args); + constructor(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, options: StaticReportBuilderOptions) { + super(htmlReporter, pluginConfig, options); - this._testsTree = GuiTestsTreeBuilder.create({toolName: ToolName.Hermione}); + this._testsTree = GuiTestsTreeBuilder.create({toolName: ToolName.Hermione, baseHost: pluginConfig.baseHost}); this._skips = []; } - async addIdle(result: ReporterTestResult): Promise { - return this._addTestResult(result, {status: IDLE}); - } - - async addRunning(result: ReporterTestResult): Promise { - return this._addTestResult(result, {status: RUNNING}); - } - - override async addSkipped(result: ReporterTestResult): Promise { - const formattedResult = await super.addSkipped(result); - const { - fullName: suite, - skipReason: comment, - browserId: browser - } = formattedResult; - - this._skips.push({suite, browser, comment}); - - return formattedResult; - } - - async addUpdated(result: ReporterTestResult): Promise { - return this._addTestResult(result, {status: UPDATED}); - } - setApiValues(values: HtmlReporterValues): this { this._apiValues = values; return this; @@ -114,8 +86,7 @@ export class GuiReportBuilder extends StaticReportBuilder { undoAcceptImage(testResultWithoutAttempt: ReporterTestResult, stateName: string): UndoAcceptImageResult | null { const attempt = this._testAttemptManager.getCurrentAttempt(testResultWithoutAttempt); - const imagesInfoFormatter = this.imageHandler; - const testResult = copyAndUpdate(testResultWithoutAttempt, {attempt}, {imagesInfoFormatter}); + const testResult = copyAndUpdate(testResultWithoutAttempt, {attempt}); const resultId = testResult.id; const suitePath = testResult.testPath; @@ -150,7 +121,7 @@ export class GuiReportBuilder extends StaticReportBuilder { updatedImage = this._testsTree.updateImageInfo(imageId, previousImage); } - const newResult = copyAndUpdate(testResult, {attempt: this._testAttemptManager.getCurrentAttempt(testResult)}, {imagesInfoFormatter}); + const newResult = copyAndUpdate(testResult, {attempt: this._testAttemptManager.getCurrentAttempt(testResult)}); this._deleteTestResultFromDb({where: [ `${DB_COLUMNS.SUITE_PATH} = ?`, @@ -163,43 +134,43 @@ export class GuiReportBuilder extends StaticReportBuilder { return {updatedImage, removedResult, previousExpectedPath, shouldRemoveReference, shouldRevertReference, newResult}; } - protected override async _addTestResult(formattedResultOriginal: ReporterTestResult, props: {status: TestStatus} & Partial): Promise { - const formattedResult = await super._addTestResult(formattedResultOriginal, props); + override async addTestResult(formattedResultOriginal: ReporterTestResult): Promise { + const formattedResult = await super.addTestResult(formattedResultOriginal); + + if (formattedResult.status === SKIPPED) { + const { + fullName: suite, + skipReason: comment, + browserId: browser + } = formattedResult; - const testResult = this._createTestResult(formattedResult, { - ...props, - timestamp: formattedResult.timestamp ?? 0, - attempt: formattedResult.attempt - }); + this._skips.push({suite, browser, comment}); + } - this._extendTestWithImagePaths(testResult, formattedResult); + const formattedResultWithImagePaths = this._extendTestWithImagePaths(formattedResult); - this._testsTree.addTestResult(testResult, formattedResult); + this._testsTree.addTestResult(formattedResultWithImagePaths); - return formattedResult; + return formattedResultWithImagePaths; } - private _extendTestWithImagePaths(testResult: PreparedTestResult, formattedResult: ReporterTestResult): void { - const newImagesInfo = formattedResult.imagesInfo; - const imagesInfoFormatter = this._imageHandler; - - if (testResult.status !== UPDATED) { - _.set(testResult, 'imagesInfo', newImagesInfo); - return; + private _extendTestWithImagePaths(formattedResult: ReporterTestResult): ReporterTestResult { + if (formattedResult.status !== UPDATED) { + return formattedResult; } - const failResultId = copyAndUpdate(formattedResult, {attempt: formattedResult.attempt - 1}, {imagesInfoFormatter}).id; - const failImagesInfo = this._testsTree.getImagesInfo(failResultId); + const failResultId = copyAndUpdate(formattedResult, {attempt: formattedResult.attempt - 1}).id; + const failImagesInfo = _.clone(this._testsTree.getImagesInfo(failResultId)) as ImageInfoFull[]; if (failImagesInfo.length) { - testResult.imagesInfo = _.clone(failImagesInfo); - - newImagesInfo?.forEach((imageInfo) => { + formattedResult.imagesInfo?.forEach((imageInfo) => { const {stateName} = imageInfo as ImageInfoWithState; - let index = _.findIndex(testResult.imagesInfo, {stateName}); - index = index >= 0 ? index : _.findLastIndex(testResult.imagesInfo); - testResult.imagesInfo[index] = imageInfo; + let index = _.findIndex(failImagesInfo, {stateName}); + index = index >= 0 ? index : _.findLastIndex(failImagesInfo); + failImagesInfo[index] = imageInfo; }); } + + return copyAndUpdate(formattedResult, {imagesInfo: failImagesInfo}); } } diff --git a/lib/report-builder/static.ts b/lib/report-builder/static.ts index ee7bb8743..38843eceb 100644 --- a/lib/report-builder/static.ts +++ b/lib/report-builder/static.ts @@ -7,30 +7,24 @@ import { IDLE, RUNNING, SKIPPED, - FAIL, - ERROR, - SUCCESS, - TestStatus, LOCAL_DATABASE_NAME, PluginEvents, UNKNOWN_ATTEMPT, UPDATED } from '../constants'; -import type {PreparedTestResult, SqliteClient} from '../sqlite-client'; +import type {SqliteClient} from '../sqlite-client'; import {ReporterTestResult} from '../test-adapter'; -import {hasImage, saveErrorDetails, saveStaticFilesToReportDir, writeDatabaseUrlsFile} from '../server-utils'; +import {saveErrorDetails, saveStaticFilesToReportDir, writeDatabaseUrlsFile} from '../server-utils'; import {ReporterConfig} from '../types'; import {HtmlReporter} from '../plugin-api'; import {ImageHandler} from '../image-handler'; import {SqliteImageStore} from '../image-store'; -import {getUrlWithBase, getError, getRelativeUrl, hasDiff, hasNoRefImageErrors} from '../common-utils'; import {getTestFromDb} from '../db-utils/server'; -import {ImageDiffError} from '../errors'; import {TestAttemptManager} from '../test-attempt-manager'; import {copyAndUpdate} from '../test-adapter/utils'; import {RegisterWorkers} from '../workers/create-workers'; const ignoredStatuses = [RUNNING, IDLE]; -interface StaticReportBuilderOptions { +export interface StaticReportBuilderOptions { dbClient: SqliteClient; } @@ -84,33 +78,6 @@ export class StaticReportBuilder { ]); } - async addSkipped(result: ReporterTestResult): Promise { - return this._addTestResult(result, { - status: SKIPPED, - skipReason: result.skipReason - }); - } - - async addSuccess(result: ReporterTestResult): Promise { - return this._addTestResult(result, {status: SUCCESS}); - } - - async addFail(result: ReporterTestResult): Promise { - return this._addFailResult(result); - } - - async addError(result: ReporterTestResult): Promise { - return this._addErrorResult(result); - } - - async addRetry(result: ReporterTestResult): Promise { - if (hasDiff(result.assertViewResults as ImageDiffError[])) { - return this._addFailResult(result); - } else { - return this._addErrorResult(result); - } - } - registerWorkers(workers: RegisterWorkers<['saveDiffTo']>): void { this._workers = workers; } @@ -129,9 +96,8 @@ export class StaticReportBuilder { let formattedResult = testResultOriginal; if (testResultOriginal.attempt === UNKNOWN_ATTEMPT) { - const imagesInfoFormatter = this._imageHandler; const attempt = this._testAttemptManager.registerAttempt(testResultOriginal, testResultOriginal.status); - formattedResult = copyAndUpdate(testResultOriginal, {attempt}, {imagesInfoFormatter}); + formattedResult = copyAndUpdate(testResultOriginal, {attempt}); } return formattedResult; @@ -155,69 +121,22 @@ export class StaticReportBuilder { await Promise.all(actions); } - protected async _addFailResult(formattedResult: ReporterTestResult): Promise { - return this._addTestResult(formattedResult, {status: FAIL}); - } - - protected async _addErrorResult(formattedResult: ReporterTestResult): Promise { - return this._addTestResult(formattedResult, {status: ERROR}); - } - - protected async _addTestResult(formattedResultOriginal: ReporterTestResult, props: {status: TestStatus} & Partial): Promise { + async addTestResult(formattedResultOriginal: ReporterTestResult): Promise { const formattedResult = this._provideAttempt(formattedResultOriginal); // Test result data has to be saved before writing to db, because user may save data to custom location await this._saveTestResultData(formattedResult); - formattedResult.image = hasImage(formattedResult); - - const testResult = this._createTestResult(formattedResult, _.extend(props, { - timestamp: formattedResult.timestamp ?? 0 - })); - - if (hasNoRefImageErrors(formattedResult as {assertViewResults: ImageDiffError[]})) { - testResult.status = FAIL; - } - // To prevent skips duplication on reporter startup - const isPreviouslySkippedTest = testResult.status === SKIPPED && getTestFromDb(this._dbClient, formattedResult); + const isPreviouslySkippedTest = formattedResult.status === SKIPPED && getTestFromDb(this._dbClient, formattedResult); - if (!ignoredStatuses.includes(testResult.status) && !isPreviouslySkippedTest) { + if (!ignoredStatuses.includes(formattedResult.status) && !isPreviouslySkippedTest) { this._dbClient.write(formattedResult); } return formattedResult; } - protected _createTestResult(result: ReporterTestResult, props: {attempt?: number | null, status: TestStatus, timestamp: number;} & Partial): PreparedTestResult { - const { - browserId, file, sessionId, description, history, - imagesInfo = [], screenshot, multipleTabs, errorDetails, testPath - } = result; - - const {baseHost, saveErrorDetails} = this._pluginConfig; - const suiteUrl: string = getUrlWithBase(result.url, baseHost); - const metaInfoFull = _.merge(_.cloneDeep(result.meta), {url: getRelativeUrl(suiteUrl) ?? '', file, sessionId}); - const metaInfo = _.omitBy(metaInfoFull, _.isEmpty); - - const testResult: PreparedTestResult = Object.assign({ - suiteUrl, name: browserId, metaInfo, description, history, - imagesInfo, screenshot: Boolean(screenshot), multipleTabs, - suitePath: testPath, suiteName: _.last(testPath) as string - }, props); - - const error = getError(result.error); - if (!_.isEmpty(error)) { - testResult.error = error; - } - - if (saveErrorDetails && errorDetails) { - testResult.errorDetails = _.pick(errorDetails, ['title', 'filePath']); - } - - return testResult; - } - protected _deleteTestResultFromDb(...args: Parameters): void { this._dbClient.delete(...args); } diff --git a/lib/server-utils.ts b/lib/server-utils.ts index 1db16430e..714e670ae 100644 --- a/lib/server-utils.ts +++ b/lib/server-utils.ts @@ -10,7 +10,7 @@ import type {ReporterTestResult} from './test-adapter'; import {CustomGuiItem, HermioneTestResult, ReporterConfig} from './types'; import type Hermione from 'hermione'; import crypto from 'crypto'; -import {ImageHandler, ImagesInfoFormatter} from './image-handler'; +import {ImagesInfoFormatter} from './image-handler'; import {HermioneTestAdapter} from './test-adapter'; import {Router} from 'express'; @@ -97,12 +97,6 @@ export function logError(e: Error): void { logger.error(`Html-reporter runtime error: ${e.stack}`); } -export function hasImage(formattedResult: ReporterTestResult): boolean { - return !!formattedResult.imagesInfo?.length || - !!ImageHandler.getCurrImg(formattedResult.assertViewResults)?.path || - !!formattedResult.screenshot; -} - export function prepareCommonJSData(data: unknown): string { const stringifiedData = JSON.stringify(data, (_key, val) => { return typeof val === 'function' ? val.toString() : val; @@ -118,10 +112,6 @@ export function shouldUpdateAttempt(status: TestStatus): boolean { return ![SKIPPED, UPDATED, RUNNING, IDLE].includes(status); } -export function getDetailsFileName(testId: string, browserId: string, attempt: number): string { - return `${testId}-${browserId}_${Number(attempt) + 1}_${Date.now()}.json`; -} - export async function saveStaticFilesToReportDir(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, destPath: string): Promise { const staticFolder = path.resolve(__dirname, './static'); await fs.ensureDir(destPath); diff --git a/lib/sqlite-client.ts b/lib/sqlite-client.ts index 4575056f2..106b11057 100644 --- a/lib/sqlite-client.ts +++ b/lib/sqlite-client.ts @@ -7,7 +7,7 @@ import NestedError from 'nested-error-stacks'; import {getShortMD5} from './common-utils'; import {TestStatus, DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS, LOCAL_DATABASE_NAME, DATABASE_URLS_JSON_NAME} from './constants'; import {createTablesQuery} from './db-utils/common'; -import type {ErrorDetails, ImageInfoFull, TestError} from './types'; +import type {ImageInfoFull, TestError} from './types'; import {HtmlReporter} from './plugin-api'; import {ReporterTestResult} from './test-adapter'; import {DbTestResultTransformer} from './test-adapter/transformers/db'; @@ -30,24 +30,6 @@ interface DeleteParams { limit?: number; } -export interface PreparedTestResult { - name: string; - suiteUrl: string; - metaInfo: Record; - history: string[]; - description?: string | null; - error?: {message?: string; stack?: string; stateName?: string}; - skipReason?: string; - imagesInfo: ImageInfoFull[]; - screenshot: boolean; - multipleTabs: boolean; - status: TestStatus; - timestamp: number; - errorDetails?: ErrorDetails; - suitePath: string[]; - suiteName: string; -} - export interface DbTestResult { description?: string | null; error?: TestError; diff --git a/lib/test-adapter/hermione.ts b/lib/test-adapter/hermione.ts index f307998d9..d3a59181b 100644 --- a/lib/test-adapter/hermione.ts +++ b/lib/test-adapter/hermione.ts @@ -56,8 +56,6 @@ export class HermioneTestAdapter implements ReporterTestResult { this._attempt = attempt; } - image?: boolean; - get fullName(): string { return this._testResult.fullTitle(); } diff --git a/lib/test-adapter/index.ts b/lib/test-adapter/index.ts index ffc2b3456..4458c2a77 100644 --- a/lib/test-adapter/index.ts +++ b/lib/test-adapter/index.ts @@ -8,16 +8,15 @@ export interface ReporterTestResult { readonly attempt: number; readonly browserId: string; readonly description: string | undefined; - error: undefined | TestError; + readonly error: undefined | TestError; readonly errorDetails: ErrorDetails | null; readonly file: string; readonly fullName: string; readonly history: string[]; readonly id: string; - image?: boolean; readonly imageDir: string; readonly imagesInfo: ImageInfoFull[] | undefined; - readonly meta: Record; + readonly meta: {browserVersion?: string} & Record; readonly multipleTabs: boolean; readonly screenshot: ImageBase64 | ImageData | null | undefined; readonly sessionId: string; diff --git a/lib/test-adapter/reporter.ts b/lib/test-adapter/reporter.ts index 15f402d4f..22a36bb22 100644 --- a/lib/test-adapter/reporter.ts +++ b/lib/test-adapter/reporter.ts @@ -1,27 +1,20 @@ import {TestStatus} from '../constants'; import {AssertViewResult, TestError, ErrorDetails, ImageInfoFull, ImageBase64, ImageData} from '../types'; import {ReporterTestResult} from './index'; -import {ImagesInfoFormatter} from '../image-handler'; import _ from 'lodash'; -import {extractErrorDetails} from './utils'; +import {extractErrorDetails, getTestHash} from './utils'; import {getShortMD5} from '../common-utils'; -interface ReporterTestAdapterOptions { - imagesInfoFormatter: ImagesInfoFormatter; -} - // This class is primarily useful when cloning ReporterTestResult. // It allows to override some properties while keeping computable // properties valid, e.g. id export class ReporterTestAdapter implements ReporterTestResult { private _testResult: ReporterTestResult; - private _imagesInfoFormatter: ImagesInfoFormatter; private _errorDetails: ErrorDetails | null; - constructor(testResult: ReporterTestResult, {imagesInfoFormatter}: ReporterTestAdapterOptions) { + constructor(testResult: ReporterTestResult) { this._testResult = testResult; - this._imagesInfoFormatter = imagesInfoFormatter; this._errorDetails = null; } @@ -68,7 +61,7 @@ export class ReporterTestAdapter implements ReporterTestResult { } get id(): string { - return this.testPath.concat(this.browserId, this.attempt.toString()).join(' '); + return getTestHash(this); } get imageDir(): string { @@ -76,7 +69,7 @@ export class ReporterTestAdapter implements ReporterTestResult { } get imagesInfo(): ImageInfoFull[] | undefined { - return this._imagesInfoFormatter.getImagesInfo(this); + return this._testResult.imagesInfo; } get meta(): Record { diff --git a/lib/test-adapter/sqlite.ts b/lib/test-adapter/sqlite.ts new file mode 100644 index 000000000..5093d5ccd --- /dev/null +++ b/lib/test-adapter/sqlite.ts @@ -0,0 +1,162 @@ +import _ from 'lodash'; +import {DB_COLUMN_INDEXES, TestStatus} from '../constants'; +import { + AssertViewResult, + TestError, + ErrorDetails, + ImageInfoFull, + ImageBase64, + ImageData, + RawSuitesRow +} from '../types'; +import {ReporterTestResult} from './index'; +import {Writable} from 'type-fest'; +import {getTestHash} from './utils'; + +const tryParseJson = (json: string): unknown | undefined => { + try { + return JSON.parse(json); + } catch { + return undefined; + } +}; + +interface SqliteTestAdapterOptions { + titleDelimiter: string; +} + +export class SqliteTestAdapter implements ReporterTestResult { + private _testResult: RawSuitesRow; + private _parsedTestResult: Writable>; + private _titleDelimiter: string; + + constructor(testResult: RawSuitesRow, attempt: number, options: SqliteTestAdapterOptions) { + this._testResult = testResult; + + this._parsedTestResult = {attempt}; + + this._titleDelimiter = options.titleDelimiter; + } + + get assertViewResults(): AssertViewResult[] { + // TODO: try to restore assertViewResults from imagesInfo + return []; + } + + get attempt(): number { + return this._parsedTestResult.attempt as number; + } + + get browserId(): string { + return this._testResult[DB_COLUMN_INDEXES.name]; + } + + get description(): string | undefined { + return this._testResult[DB_COLUMN_INDEXES.description] ?? undefined; + } + + get error(): TestError | undefined { + if (!_.has(this._parsedTestResult, 'error')) { + this._parsedTestResult.error = tryParseJson(this._testResult[DB_COLUMN_INDEXES.error]) as TestError | undefined; + } + + return this._parsedTestResult.error; + } + + get errorDetails(): ErrorDetails | null { + // TODO: implement returning error details + return null; + } + + get file(): string { + if (!_.has(this._parsedTestResult, 'meta')) { + this._parsedTestResult.meta = tryParseJson(this._testResult[DB_COLUMN_INDEXES.metaInfo]) as Record; + } + + return this._parsedTestResult.meta?.file as string; + } + + get fullName(): string { + if (!_.has(this._parsedTestResult, 'fullName')) { + this._parsedTestResult.fullName = this.testPath.join(this._titleDelimiter); + } + + return this._parsedTestResult.fullName as string; + } + + get history(): string[] { + if (!_.has(this._parsedTestResult, 'history')) { + this._parsedTestResult.history = tryParseJson(this._testResult[DB_COLUMN_INDEXES.history]) as string[]; + } + + return this._parsedTestResult.history as string[]; + } + + get id(): string { + return this.testPath.concat(this.browserId, this.attempt.toString()).join(' '); + } + + get imageDir(): string { + return getTestHash(this); + } + + get imagesInfo(): ImageInfoFull[] | undefined { + if (!_.has(this._parsedTestResult, 'imagesInfo')) { + this._parsedTestResult.imagesInfo = tryParseJson(this._testResult[DB_COLUMN_INDEXES.imagesInfo]) as ImageInfoFull[]; + } + + return this._parsedTestResult.imagesInfo as ImageInfoFull[]; + } + + get meta(): Record { + if (!_.has(this._parsedTestResult, 'meta')) { + this._parsedTestResult.meta = tryParseJson(this._testResult[DB_COLUMN_INDEXES.metaInfo]) as Record; + } + + return this._parsedTestResult.meta as Record; + } + + get multipleTabs(): boolean { + return Boolean(this._testResult[DB_COLUMN_INDEXES.multipleTabs]); + } + + get screenshot(): ImageBase64 | ImageData | null | undefined { + return this.error?.screenshot; + } + + get sessionId(): string { + if (!_.has(this._parsedTestResult, 'meta')) { + this._parsedTestResult.meta = tryParseJson(this._testResult[DB_COLUMN_INDEXES.metaInfo]) as Record; + } + + return this._parsedTestResult.meta?.sessionId as string; + } + + get skipReason(): string | undefined { + return this._testResult[DB_COLUMN_INDEXES.skipReason]; + } + + get state(): { name: string; } { + return {name: this.testPath.at(-1) as string}; + } + + get status(): TestStatus { + return this._testResult[DB_COLUMN_INDEXES.status] as TestStatus; + } + + get testPath(): string[] { + if (!_.has(this._parsedTestResult, 'testPath')) { + this._parsedTestResult.testPath = tryParseJson(this._testResult[DB_COLUMN_INDEXES.suitePath]) as string[]; + } + + return this._parsedTestResult.testPath as string[]; + } + + get timestamp(): number | undefined { + return Number(this._testResult[DB_COLUMN_INDEXES.timestamp]); + } + + get url(): string | undefined { + return this._testResult[DB_COLUMN_INDEXES.suiteUrl]; + } +} diff --git a/lib/test-adapter/transformers/tree.ts b/lib/test-adapter/transformers/tree.ts new file mode 100644 index 000000000..f407243d9 --- /dev/null +++ b/lib/test-adapter/transformers/tree.ts @@ -0,0 +1,27 @@ +import {ReporterTestResult} from '../index'; +import _ from 'lodash'; +import {BaseTreeTestResult} from '../../tests-tree-builder/base'; +import {DbTestResultTransformer} from './db'; +import {extractErrorDetails} from '../utils'; + +interface Options { + baseHost?: string; +} + +export class TreeTestResultTransformer { + private _transformer: DbTestResultTransformer; + + constructor(options: Options) { + this._transformer = new DbTestResultTransformer(options); + } + + transform(testResult: ReporterTestResult): BaseTreeTestResult { + const result = this._transformer.transform(testResult); + + return { + ..._.omit(result, 'imagesInfo'), + attempt: testResult.attempt, + errorDetails: extractErrorDetails(testResult) + }; + } +} diff --git a/lib/test-adapter/utils/index.ts b/lib/test-adapter/utils/index.ts index 48e5c7d29..09a1e1d79 100644 --- a/lib/test-adapter/utils/index.ts +++ b/lib/test-adapter/utils/index.ts @@ -3,14 +3,12 @@ import {ReporterTestResult} from '../index'; import {TupleToUnion} from 'type-fest'; import {ErrorDetails} from '../../types'; import {ERROR_DETAILS_PATH} from '../../constants'; -import * as utils from '../../server-utils'; import {ReporterTestAdapter} from '../reporter'; -import {ImagesInfoFormatter} from '../../image-handler'; +import {getDetailsFileName} from '../../common-utils'; export const copyAndUpdate = ( original: ReporterTestResult, - updates: Partial, - {imagesInfoFormatter}: {imagesInfoFormatter: ImagesInfoFormatter} + updates: Partial ): ReporterTestResult => { const keys = [ 'assertViewResults', @@ -23,7 +21,6 @@ export const copyAndUpdate = ( 'fullName', 'history', 'id', - 'image', 'imageDir', 'imagesInfo', 'meta', @@ -48,7 +45,7 @@ export const copyAndUpdate = ( const updatedTestResult = _.assign({}, _.pick(original, keysTypeChecked) as ReporterTestResult, updates); - return new ReporterTestAdapter(updatedTestResult, {imagesInfoFormatter}); + return new ReporterTestAdapter(updatedTestResult); }; export const extractErrorDetails = (testResult: ReporterTestResult): ErrorDetails | null => { @@ -58,7 +55,7 @@ export const extractErrorDetails = (testResult: ReporterTestResult): ErrorDetail return { title: details.title || 'error details', data: details.data, - filePath: `${ERROR_DETAILS_PATH}/${utils.getDetailsFileName( + filePath: `${ERROR_DETAILS_PATH}/${getDetailsFileName( testResult.imageDir, testResult.browserId, testResult.attempt )}` }; @@ -66,3 +63,7 @@ export const extractErrorDetails = (testResult: ReporterTestResult): ErrorDetail return null; }; + +export const getTestHash = (testResult: ReporterTestResult): string => { + return testResult.testPath.concat(testResult.browserId, testResult.attempt.toString()).join(' '); +}; diff --git a/lib/tests-tree-builder/base.ts b/lib/tests-tree-builder/base.ts index ee71e270f..995303b5b 100644 --- a/lib/tests-tree-builder/base.ts +++ b/lib/tests-tree-builder/base.ts @@ -2,15 +2,21 @@ import _ from 'lodash'; import {determineFinalStatus} from '../common-utils'; import {BrowserVersions, PWT_TITLE_DELIMITER, TestStatus, ToolName} from '../constants'; import {ReporterTestResult} from '../test-adapter'; -import {ImageInfoFull, ParsedSuitesRow} from '../types'; +import {ErrorDetails, ImageInfoFull} from '../types'; +import {TreeTestResultTransformer} from '../test-adapter/transformers/tree'; +import {DbTestResult} from '../sqlite-client'; -export type TreeResult = { - attempt: number; +export type BaseTreeTestResult = Omit & { + attempt?: number; + errorDetails?: ErrorDetails | null; +} + +export interface TreeTestResult extends BaseTreeTestResult { id: string; parentId: string; - status: TestStatus; imageIds: string[]; -} & Omit; + attempt: number; +} interface TreeBrowser { id: string; @@ -47,7 +53,7 @@ export interface Tree { allIds: string[] }, results: { - byId: Record, + byId: Record, allIds: string[] }, images: { @@ -59,7 +65,7 @@ export interface Tree { interface ResultPayload { id: string; parentId: string; - result: ParsedSuitesRow; + result: ReporterTestResult; } interface BrowserPayload { @@ -76,11 +82,13 @@ interface ImagesPayload { export interface BaseTestsTreeBuilderOptions { toolName: ToolName; + baseHost: string; } export class BaseTestsTreeBuilder { protected _tree: Tree; protected _toolName: ToolName; + protected _transformer: TreeTestResultTransformer; static create( this: new (options: BaseTestsTreeBuilderOptions) => T, @@ -89,9 +97,11 @@ export class BaseTestsTreeBuilder { return new this(options); } - constructor({toolName}: BaseTestsTreeBuilderOptions) { + constructor({toolName, baseHost}: BaseTestsTreeBuilderOptions) { this._toolName = toolName; + this._transformer = new TreeTestResultTransformer({baseHost}); + this._tree = { suites: {byId: {}, allIds: [], allRootIds: []}, browsers: {byId: {}, allIds: []}, @@ -120,10 +130,9 @@ export class BaseTestsTreeBuilder { this._tree.suites.allRootIds.sort().forEach(sortChildSuites); } - addTestResult(testResult: ParsedSuitesRow, formattedResult: Pick): void { - const {testPath, browserId: browserName, attempt} = formattedResult; - const {imagesInfo} = testResult; - const {browserVersion = BrowserVersions.UNKNOWN} = testResult.metaInfo as {browserVersion: string}; + addTestResult(formattedResult: ReporterTestResult): void { + const {testPath, browserId: browserName, attempt, imagesInfo = []} = formattedResult; + const {browserVersion = BrowserVersions.UNKNOWN} = formattedResult.meta as {browserVersion: string}; const suiteId = this._buildId(testPath); const browserId = this._buildId(suiteId, browserName); @@ -134,7 +143,7 @@ export class BaseTestsTreeBuilder { this._addSuites(testPath, browserId); this._addBrowser({id: browserId, parentId: suiteId, name: browserName, version: browserVersion}, testResultId, attempt); - this._addResult({id: testResultId, parentId: browserId, result: testResult}, imageIds); + this._addResult({id: testResultId, parentId: browserId, result: formattedResult}, imageIds); this._addImages(imageIds, {imagesInfo, parentId: testResultId}); this._setStatusForBranch(testPath); @@ -217,13 +226,13 @@ export class BaseTestsTreeBuilder { } protected _addResult({id, parentId, result}: ResultPayload, imageIds: string[]): void { - const resultWithoutImagesInfo = _.omit(result, 'imagesInfo'); + const treeResult = this._transformer.transform(result); if (!this._tree.results.byId[id]) { this._tree.results.allIds.push(id); } - this._tree.results.byId[id] = {attempt: 0, id, parentId, ...resultWithoutImagesInfo, imageIds}; + this._tree.results.byId[id] = {attempt: 0, id, parentId, ...treeResult, imageIds}; } protected _addImages(imageIds: string[], {imagesInfo, parentId}: ImagesPayload): void { diff --git a/lib/tests-tree-builder/gui.ts b/lib/tests-tree-builder/gui.ts index 83c74ec59..3336a8fbd 100644 --- a/lib/tests-tree-builder/gui.ts +++ b/lib/tests-tree-builder/gui.ts @@ -1,5 +1,5 @@ import _ from 'lodash'; -import {BaseTestsTreeBuilder, Tree, TreeImage, TreeResult, TreeSuite} from './base'; +import {BaseTestsTreeBuilder, Tree, TreeImage, TreeTestResult, TreeSuite} from './base'; import {TestStatus, UPDATED} from '../constants'; import {isUpdatedStatus} from '../common-utils'; import {ImageInfoFail, ImageInfoWithState} from '../types'; @@ -10,17 +10,17 @@ interface SuiteBranch { } export interface TestBranch { - result: TreeResult; + result: TreeTestResult; images: TreeImage[]; suites: SuiteBranch[]; } export interface TestRefUpdateData { browserId: string; - error?: TreeResult['error']; + error?: TreeTestResult['error']; suite: {path: string[]}; state: {name: string}; - metaInfo: TreeResult['metaInfo']; + metaInfo: TreeTestResult['metaInfo']; imagesInfo: { stateName: ImageInfoWithState['stateName']; actualImg: ImageInfoWithState['actualImg']; diff --git a/lib/tests-tree-builder/static.ts b/lib/tests-tree-builder/static.ts index 2c1ffdb5e..a45707829 100644 --- a/lib/tests-tree-builder/static.ts +++ b/lib/tests-tree-builder/static.ts @@ -1,7 +1,10 @@ import _ from 'lodash'; import {BaseTestsTreeBuilder, BaseTestsTreeBuilderOptions, Tree} from './base'; import {BrowserVersions, DB_COLUMN_INDEXES, TestStatus} from '../constants'; -import {Attempt, ParsedSuitesRow, RawSuitesRow} from '../types'; +import {ReporterTestResult} from '../test-adapter'; +import {SqliteTestAdapter} from '../test-adapter/sqlite'; +import {getTitleDelimiter} from '../common-utils'; +import {RawSuitesRow} from '../types'; interface Stats { total: number; @@ -67,13 +70,12 @@ export class StaticTestsTreeBuilder extends BaseTestsTreeBuilder { attemptsMap.set(browserId, attemptsMap.has(browserId) ? attemptsMap.get(browserId) as number + 1 : 0); const attempt = attemptsMap.get(browserId) as number; - const testResult = mkTestResult(row, {attempt}); - const formattedResult = {browserId: browserName, testPath, attempt}; + const formattedResult = new SqliteTestAdapter(row, attempt, {titleDelimiter: getTitleDelimiter(this._toolName)}); - addBrowserVersion(browsers, testResult); + addBrowserVersion(browsers, formattedResult); - this.addTestResult(testResult, formattedResult); - this._calcStats(testResult, {testId, browserName}); + this.addTestResult(formattedResult); + this._calcStats(formattedResult, {testId, browserName}); } this.sortTree(); @@ -90,10 +92,10 @@ export class StaticTestsTreeBuilder extends BaseTestsTreeBuilder { this._tree.browsers.byId[browserId].resultIds.push(testResultId); } - protected _calcStats(testResult: ParsedSuitesRow, {testId, browserName}: { testId: string; browserName: string }): void { + protected _calcStats(testResult: ReporterTestResult, {testId, browserName}: { testId: string; browserName: string }): void { const testIdWithBrowser = this._buildId(testId, browserName); const {status} = testResult; - const {browserVersion} = testResult.metaInfo; + const {browserVersion} = testResult.meta; const version = browserVersion || BrowserVersions.UNKNOWN; if (!this._stats.perBrowser[browserName]) { @@ -191,33 +193,13 @@ function initStats(): Stats { }; } -function mkTestResult(row: RawSuitesRow, data: {attempt: number}): ParsedSuitesRow & Attempt { - return { - description: row[DB_COLUMN_INDEXES.description] as string | null, - imagesInfo: JSON.parse(row[DB_COLUMN_INDEXES.imagesInfo] as string), - metaInfo: JSON.parse(row[DB_COLUMN_INDEXES.metaInfo] as string), - history: JSON.parse(row[DB_COLUMN_INDEXES.history] as string), - multipleTabs: Boolean(row[DB_COLUMN_INDEXES.multipleTabs]), - name: row[DB_COLUMN_INDEXES.name] as string, - screenshot: Boolean(row[DB_COLUMN_INDEXES.screenshot]), - status: row[DB_COLUMN_INDEXES.status] as TestStatus, - suiteName: row[DB_COLUMN_INDEXES.suiteName] as string, - suitePath: JSON.parse(row[DB_COLUMN_INDEXES.suitePath] as string), - suiteUrl: row[DB_COLUMN_INDEXES.suiteUrl] as string, - skipReason: row[DB_COLUMN_INDEXES.skipReason] as string, - error: JSON.parse(row[DB_COLUMN_INDEXES.error] as string), - timestamp: Number(row[DB_COLUMN_INDEXES.timestamp]), - ...data - }; -} - -function addBrowserVersion(browsers: Record>, testResult: ParsedSuitesRow): void { - const browserId = testResult.name; +function addBrowserVersion(browsers: Record>, testResult: ReporterTestResult): void { + const {browserId} = testResult; if (!browsers[browserId]) { browsers[browserId] = new Set(); } - const {browserVersion = BrowserVersions.UNKNOWN} = testResult.metaInfo; + const {browserVersion = BrowserVersions.UNKNOWN} = testResult.meta; browsers[browserId].add(browserVersion); } diff --git a/lib/types.ts b/lib/types.ts index 8c1d5db43..d129f0ec9 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -1,8 +1,9 @@ import type {LooksSameOptions, CoordBounds} from 'looks-same'; import type {default as Hermione, TestResult as HermioneTestResultOriginal} from 'hermione'; -import {DiffModeId, SaveFormat, TestStatus, ViewMode} from './constants'; +import {DB_TYPES, DiffModeId, SaveFormat, SUITES_TABLE_COLUMNS, TestStatus, ViewMode} from './constants'; import type {HtmlReporter} from './plugin-api'; import {ImageDiffError, NoRefImageError} from './errors'; +import {EmptyObject, ValueOf} from 'type-fest'; declare module 'tmp' { export const tmpdir: string; @@ -108,40 +109,6 @@ export interface TestError { screenshot?: ImageBase64 | ImageData } -export interface LabeledSuitesRow { - imagesInfo: string; - timestamp: number; -} - -export type RawSuitesRow = LabeledSuitesRow[keyof LabeledSuitesRow][]; - -export interface ParsedSuitesRow { - description?: string | null; - error?: { - message?: string; - stack?: string; - }; - history: unknown; - imagesInfo: ImageInfoFull[]; - metaInfo: { - browserVersion?: string; - [key: string]: unknown; - }; - multipleTabs: boolean; - name: string; - screenshot: boolean; - skipReason?: string; - status: TestStatus; - suiteName: string; - suitePath: string[]; - suiteUrl: string; - timestamp: number; -} - -export interface Attempt { - attempt: number; -} - export interface HtmlReporterApi { htmlReporter: HtmlReporter; } @@ -191,3 +158,38 @@ export interface DbUrlsJsonData { dbUrls: string[]; jsonUrls: string[]; } + +type DbType = ValueOf; + +type Length = + T extends { length: infer L } ? L : never; + +// Writable from type-fest didn't work here, because it transforms array to object +export type Mutable = { + -readonly [K in keyof T]: T[K] +} + +type ExtractType = T extends typeof DB_TYPES.int ? number : + T extends typeof DB_TYPES.text ? string : never; + +// This type accepts an array of objects shaped {name: string} and returns map of shape {: } +// Useful to produce precise type of db columns order, e.g. {suitePath: 0, suiteName: 1, ...} +export type NameToIndexMap = T extends [infer Head, ...infer Tail] ? + Tail extends {name: string}[] ? + Head extends {name: string} ? NameToIndexMap}, [Head, ...Processed]> : never + : never + : Result; + +// This type accepts an array of objects shaped {type: DbType} and returns map of shape {: } +// Useful to produce precise type of table row in db, similar to a tuple, e.g. [string, string, number, ...] +type IndexToTypeMap = T extends [infer Head, ...infer Tail] ? + Tail extends {type: DbType}[] ? + Head extends {type: DbType} ? IndexToTypeMap]: ExtractType}, [Head, ...Processed]> : never + : never + : Result; + +export type RawSuitesRow = IndexToTypeMap>; + +export type LabeledSuitesRow = { + [K in (typeof SUITES_TABLE_COLUMNS)[number]['name']]: string; +}; diff --git a/playwright.ts b/playwright.ts index 03464bd9e..c0056a248 100644 --- a/playwright.ts +++ b/playwright.ts @@ -10,9 +10,9 @@ import {StaticReportBuilder} from './lib/report-builder/static'; import {HtmlReporter} from './lib/plugin-api'; import {ReporterConfig} from './lib/types'; import {parseConfig} from './lib/config'; -import {PluginEvents, TestStatus, ToolName} from './lib/constants'; +import {PluginEvents, ToolName} from './lib/constants'; import {RegisterWorkers} from './lib/workers/create-workers'; -import {PlaywrightTestAdapter, getStatus} from './lib/test-adapter/playwright'; +import {PlaywrightTestAdapter} from './lib/test-adapter/playwright'; import {SqliteClient} from './lib/sqlite-client'; export {ReporterConfig} from './lib/types'; @@ -55,20 +55,9 @@ class MyReporter implements Reporter { const staticReportBuilder = this._staticReportBuilder as StaticReportBuilder; - const status = getStatus(result); const formattedResult = new PlaywrightTestAdapter(test, result, {imagesInfoFormatter: staticReportBuilder.imageHandler}); - if (status === TestStatus.FAIL) { - if (formattedResult.status === TestStatus.FAIL) { - await staticReportBuilder.addFail(formattedResult); - } else { - await staticReportBuilder.addError(formattedResult); - } - } else if (status === TestStatus.SUCCESS) { - await staticReportBuilder.addSuccess(formattedResult); - } else if (status === TestStatus.SKIPPED) { - await staticReportBuilder.addSkipped(formattedResult); - } + await staticReportBuilder.addTestResult(formattedResult); }); } From 1d4b873ed39c793bf9f245e6dc4a8ce1f79e25bb Mon Sep 17 00:00:00 2001 From: shadowusr Date: Thu, 21 Dec 2023 02:18:31 +0300 Subject: [PATCH 2/7] refactor: get rid of asssertViewResults in favor of imagesInfo --- lib/test-adapter/hermione.ts | 80 ++++++++++++--- lib/test-adapter/index.ts | 7 +- test/unit/lib/test-adapter/hermione.ts | 134 +++++++++++++++++++++---- 3 files changed, 185 insertions(+), 36 deletions(-) diff --git a/lib/test-adapter/hermione.ts b/lib/test-adapter/hermione.ts index d3a59181b..1edfc689f 100644 --- a/lib/test-adapter/hermione.ts +++ b/lib/test-adapter/hermione.ts @@ -2,16 +2,21 @@ import _ from 'lodash'; import path from 'path'; import {getCommandsHistory} from '../history-utils'; -import {TestStatus} from '../constants'; -import {wrapLinkByTag} from '../common-utils'; +import {ERROR, FAIL, SUCCESS, TestStatus, UPDATED} from '../constants'; +import {getError, isImageDiffError, isNoRefImageError, wrapLinkByTag} from '../common-utils'; import { - AssertViewResult, ErrorDetails, ImageBase64, ImageInfoFull, - HermioneTestResult, HermioneSuite, TestError + HermioneTestResult, + HermioneSuite, + TestError, + ImageInfoDiff, + ImageInfoNoRef, + ImageInfoSuccess, + ImageFile, + ImageInfoPageError, ImageInfoPageSuccess, ImageInfoUpdated } from '../types'; -import {ImagesInfoFormatter} from '../image-handler'; import {ReporterTestResult} from './index'; import {getSuitePath} from '../plugin-utils'; import {extractErrorDetails} from './utils'; @@ -27,11 +32,9 @@ const wrapSkipComment = (skipComment: string | null | undefined): string => { export interface HermioneTestAdapterOptions { attempt: number; status: TestStatus; - imagesInfoFormatter: ImagesInfoFormatter; } export class HermioneTestAdapter implements ReporterTestResult { - private _imagesInfoFormatter: ImagesInfoFormatter; private _testResult: HermioneTestResult; private _errorDetails: ErrorDetails | null; private _timestamp: number | undefined; @@ -42,8 +45,7 @@ export class HermioneTestAdapter implements ReporterTestResult { return new this(testResult, options); } - constructor(testResult: HermioneTestResult, {attempt, status, imagesInfoFormatter}: HermioneTestAdapterOptions) { - this._imagesInfoFormatter = imagesInfoFormatter; + constructor(testResult: HermioneTestResult, {attempt, status}: HermioneTestAdapterOptions) { this._testResult = testResult; this._errorDetails = null; this._timestamp = this._testResult.timestamp ?? this._testResult.startTime ?? Date.now(); @@ -76,18 +78,66 @@ export class HermioneTestAdapter implements ReporterTestResult { return this._testResult.browserId; } - get imagesInfo(): ImageInfoFull[] | undefined { - return this._imagesInfoFormatter.getImagesInfo(this); + get imagesInfo(): ImageInfoFull[] { + const {assertViewResults = []} = this._testResult; + + const imagesInfo: ImageInfoFull[] = assertViewResults.map((assertResult): ImageInfoFull => { + if (isImageDiffError(assertResult)) { + const diffBufferImg = assertResult.diffBuffer ? {buffer: assertResult.diffBuffer as Buffer} : undefined; + const diffImg = assertResult.diffImg ?? diffBufferImg; + + return { + status: FAIL, + stateName: assertResult.stateName, + refImg: assertResult.refImg, + actualImg: assertResult.currImg, + ...(diffImg ? {diffImg} : {}), + expectedImg: assertResult.refImg, + diffClusters: assertResult.diffClusters, + diffOptions: assertResult.diffOpts + } satisfies ImageInfoDiff; + } else if (isNoRefImageError(assertResult)) { + return { + status: ERROR, + stateName: assertResult.stateName, + error: _.pick(assertResult, ['message', 'name', 'stack']), + refImg: assertResult.refImg, + actualImg: assertResult.currImg + } satisfies ImageInfoNoRef; + } else if ((assertResult as {isUpdated?: boolean}).isUpdated) { + return { + status: UPDATED, + stateName: assertResult.stateName, + refImg: assertResult.refImg, + expectedImg: assertResult.refImg, + actualImg: (assertResult as {currImg: ImageFile}).currImg + } satisfies ImageInfoUpdated; + } else { + const {currImg} = assertResult as {currImg?: ImageFile}; + return { + status: SUCCESS, + stateName: assertResult.stateName, + refImg: assertResult.refImg, + expectedImg: assertResult.refImg, + ...(currImg ? {actualImg: currImg} : {}) + } satisfies ImageInfoSuccess; + } + }); + + if (this.screenshot) { + imagesInfo.push({ + status: _.isEmpty(getError(this.error)) ? SUCCESS : ERROR, + actualImg: this.screenshot + } satisfies ImageInfoPageSuccess | ImageInfoPageError as ImageInfoPageSuccess | ImageInfoPageError); + } + + return imagesInfo; } get attempt(): number { return this._attempt; } - get assertViewResults(): AssertViewResult[] { - return this._testResult.assertViewResults || []; - } - get history(): string[] { return getCommandsHistory(this._testResult.history) as string[]; } diff --git a/lib/test-adapter/index.ts b/lib/test-adapter/index.ts index 4458c2a77..0f360d6f3 100644 --- a/lib/test-adapter/index.ts +++ b/lib/test-adapter/index.ts @@ -1,10 +1,9 @@ import {TestStatus} from '../constants'; -import {AssertViewResult, ErrorDetails, ImageBase64, ImageData, ImageInfoFull, TestError} from '../types'; +import {ErrorDetails, ImageBase64, ImageFile, ImageInfoFull, TestError} from '../types'; export * from './hermione'; export interface ReporterTestResult { - readonly assertViewResults: AssertViewResult[]; readonly attempt: number; readonly browserId: string; readonly description: string | undefined; @@ -15,10 +14,10 @@ export interface ReporterTestResult { readonly history: string[]; readonly id: string; readonly imageDir: string; - readonly imagesInfo: ImageInfoFull[] | undefined; + readonly imagesInfo: ImageInfoFull[]; readonly meta: {browserVersion?: string} & Record; readonly multipleTabs: boolean; - readonly screenshot: ImageBase64 | ImageData | null | undefined; + readonly screenshot: ImageBase64 | ImageFile | null | undefined; readonly sessionId: string; readonly skipReason?: string; readonly state: { name: string }; diff --git a/test/unit/lib/test-adapter/hermione.ts b/test/unit/lib/test-adapter/hermione.ts index fad52c465..cf31927d3 100644 --- a/test/unit/lib/test-adapter/hermione.ts +++ b/test/unit/lib/test-adapter/hermione.ts @@ -8,8 +8,8 @@ import {TestStatus} from 'lib/constants/test-statuses'; import {ERROR_DETAILS_PATH} from 'lib/constants/paths'; import {HermioneTestAdapter, HermioneTestAdapterOptions, ReporterTestResult} from 'lib/test-adapter'; import {HermioneTestResult} from 'lib/types'; -import {ImagesInfoFormatter} from 'lib/image-handler'; import * as originalUtils from 'lib/server-utils'; +import * as originalCommonUtils from 'lib/common-utils'; import * as originalTestAdapterUtils from 'lib/test-adapter/utils'; describe('HermioneTestAdapter', () => { @@ -19,20 +19,17 @@ describe('HermioneTestAdapter', () => { let getCommandsHistory: sinon.SinonStub; let getSuitePath: sinon.SinonStub; let utils: sinon.SinonStubbedInstance; + let commonUtils: sinon.SinonStubbedInstance; let fs: sinon.SinonStubbedInstance; let tmp: typeof tmpOriginal; let hermioneCache: typeof import('lib/test-adapter/cache/hermione'); let testAdapterUtils: sinon.SinonStubbedInstance; - const mkImagesInfoFormatter = (): sinon.SinonStubbedInstance => { - return {} as sinon.SinonStubbedInstance; - }; - const mkHermioneTestResultAdapter = ( testResult: HermioneTestResult, - {status = TestStatus.SUCCESS, imagesInfoFormatter = mkImagesInfoFormatter()}: {status?: TestStatus, imagesInfoFormatter?: ImagesInfoFormatter} = {} + {status = TestStatus.SUCCESS}: {status?: TestStatus} = {} ): HermioneTestAdapter => { - return new HermioneTestAdapter(testResult, {status, imagesInfoFormatter, attempt: 0}) as HermioneTestAdapter; + return new HermioneTestAdapter(testResult, {status, attempt: 0}) as HermioneTestAdapter; }; const mkTestResult_ = (result: Partial): HermioneTestResult => _.defaults(result, { @@ -52,8 +49,12 @@ describe('HermioneTestAdapter', () => { }); utils = _.clone(originalUtils); + const originalCommonUtils = proxyquire('lib/common-utils', {}); + commonUtils = _.clone(originalCommonUtils); + const originalTestAdapterUtils = proxyquire('lib/test-adapter/utils', { - '../../server-utils': utils + '../../server-utils': utils, + '../../common-utils': commonUtils }); testAdapterUtils = _.clone(originalTestAdapterUtils); @@ -126,19 +127,11 @@ describe('HermioneTestAdapter', () => { assert.deepEqual(hermioneTestAdapter.state, {name: 'some-test'}); }); - it('should return assert view results', () => { - const testResult = mkTestResult_({assertViewResults: [1 as any]}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.assertViewResults, [1 as any]); - }); - describe('error details', () => { let getDetailsFileName: sinon.SinonStub; beforeEach(() => { - getDetailsFileName = sandbox.stub(utils, 'getDetailsFileName').returns(''); + getDetailsFileName = sandbox.stub(commonUtils, 'getDetailsFileName').returns(''); }); it('should be returned for test if they are available', () => { @@ -233,4 +226,111 @@ describe('HermioneTestAdapter', () => { assert.strictEqual(hermioneTestAdapter.timestamp, 100500); }); }); + + describe('imagesInfo', () => { + it('should correctly format diff assert view result', () => { + const testResult = mkTestResult_({ + timestamp: 100500, + assertViewResults: [{ + currImg: {path: 'curr-path', size: {width: 20, height: 10}}, + diffClusters: [], + diffOpts: {diffColor: '#000'} as any, + message: 'diff message', + name: 'ImageDiffError', + refImg: {path: 'ref-path', size: {width: 25, height: 15}}, + stack: 'some-stack', + stateName: 'some-state' + }] + }); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.imagesInfo, [ + { + status: TestStatus.FAIL, + stateName: 'some-state', + actualImg: {path: 'curr-path', size: {height: 10, width: 20}}, + expectedImg: {path: 'ref-path', size: {height: 15, width: 25}}, + refImg: {path: 'ref-path', size: {height: 15, width: 25}}, + diffClusters: [], + diffOptions: {diffColor: '#000'} as any + } + ]); + }); + + it('should correctly format no ref assert view result', () => { + const testResult = mkTestResult_({ + timestamp: 100500, + assertViewResults: [{ + currImg: {path: 'curr-path', size: {height: 10, width: 20}}, + message: 'no ref message', + name: 'NoRefImageError', + refImg: {path: 'ref-path', size: {height: 15, width: 25}}, + stack: 'some-stack', + stateName: 'some-state' + }] + }); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.imagesInfo, [ + { + status: TestStatus.ERROR, + stateName: 'some-state', + error: { + name: 'NoRefImageError', + message: 'no ref message', + stack: 'some-stack' + }, + actualImg: {path: 'curr-path', size: {height: 10, width: 20}}, + refImg: {path: 'ref-path', size: {height: 15, width: 25}} + } + ]); + }); + + it('should correctly format updated assert view result', () => { + const testResult = mkTestResult_({ + timestamp: 100500, + assertViewResults: [{ + isUpdated: true, + stateName: 'some-state', + currImg: {path: 'curr-path', size: {height: 10, width: 20}}, + refImg: {path: 'ref-path', size: {height: 15, width: 25}} + } as HermioneTestResult['assertViewResults'][number]] + }); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.imagesInfo, [ + { + status: TestStatus.UPDATED, + stateName: 'some-state', + actualImg: {path: 'curr-path', size: {height: 10, width: 20}}, + expectedImg: {path: 'ref-path', size: {height: 15, width: 25}}, + refImg: {path: 'ref-path', size: {height: 15, width: 25}} + } + ]); + }); + + it('should correctly format success assert view result', () => { + const testResult = mkTestResult_({ + timestamp: 100500, + assertViewResults: [{ + stateName: 'some-state', + refImg: {path: 'ref-path', size: {height: 15, width: 25}} + }] + }); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.imagesInfo, [ + { + status: TestStatus.SUCCESS, + stateName: 'some-state', + expectedImg: {path: 'ref-path', size: {height: 15, width: 25}}, + refImg: {path: 'ref-path', size: {height: 15, width: 25}} + } + ]); + }); + }); }); From bbe24283434c107c97b481cfb2c55e7e8ebfae7f Mon Sep 17 00:00:00 2001 From: shadowusr Date: Thu, 21 Dec 2023 02:25:23 +0300 Subject: [PATCH 3/7] refactor: replace image-handler with images-info-saver --- hermione.ts | 27 +- lib/cache.ts | 33 ++ lib/common-utils.ts | 26 +- lib/constants/errors.js | 9 - lib/constants/errors.ts | 3 + lib/constants/index.ts | 1 + lib/errors/index.ts | 12 +- lib/gui/tool-runner/index.ts | 76 +-- lib/gui/tool-runner/report-subscriber.ts | 10 +- lib/gui/tool-runner/utils.ts | 4 +- lib/image-handler.ts | 366 --------------- lib/image-store.ts | 10 +- lib/images-info-saver.ts | 231 +++++++++ ...ges-saver.ts => local-image-file-saver.ts} | 4 +- lib/plugin-api.ts | 12 +- lib/report-builder/gui.ts | 23 +- lib/report-builder/static.ts | 66 ++- lib/reporter-helpers.ts | 70 ++- lib/server-utils.ts | 47 +- .../section/body/page-screenshot.tsx | 4 +- lib/test-adapter/playwright.ts | 122 ++--- lib/test-adapter/reporter.ts | 14 +- lib/test-adapter/sqlite.ts | 8 +- lib/test-adapter/utils/index.ts | 16 +- lib/tests-tree-builder/gui.ts | 14 +- lib/types.ts | 82 ++-- lib/workers/worker.ts | 6 +- playwright.ts | 19 +- test/unit/hermione.js | 7 +- test/unit/lib/image-handler.ts | 441 ------------------ test/unit/lib/images-info-saver.ts | 265 +++++++++++ test/unit/lib/report-builder/gui.js | 6 +- test/unit/lib/report-builder/static.js | 7 +- test/unit/workers/worker.js | 8 +- 34 files changed, 944 insertions(+), 1105 deletions(-) create mode 100644 lib/cache.ts delete mode 100644 lib/constants/errors.js create mode 100644 lib/constants/errors.ts delete mode 100644 lib/image-handler.ts create mode 100644 lib/images-info-saver.ts rename lib/{local-images-saver.ts => local-image-file-saver.ts} (67%) delete mode 100644 test/unit/lib/image-handler.ts create mode 100644 test/unit/lib/images-info-saver.ts diff --git a/hermione.ts b/hermione.ts index f3788405e..be061abe8 100644 --- a/hermione.ts +++ b/hermione.ts @@ -11,10 +11,13 @@ import {parseConfig} from './lib/config'; import {SKIPPED, SUCCESS, TestStatus, ToolName, UNKNOWN_ATTEMPT} from './lib/constants'; import {HtmlReporter} from './lib/plugin-api'; import {StaticReportBuilder} from './lib/report-builder/static'; -import {formatTestResult, logPathToHtmlReport, logError} from './lib/server-utils'; +import {formatTestResult, logPathToHtmlReport, logError, getExpectedCacheKey} from './lib/server-utils'; import {SqliteClient} from './lib/sqlite-client'; -import {HtmlReporterApi, ImageInfoFull, ReporterOptions} from './lib/types'; +import {HtmlReporterApi, ImageInfoFull, ReporterOptions, TestSpecByPath} from './lib/types'; import {createWorkers, CreateWorkersRunner} from './lib/workers/create-workers'; +import {SqliteImageStore} from './lib/image-store'; +import {Cache} from './lib/cache'; +import {ImagesInfoSaver} from './lib/images-info-saver'; export = (hermione: Hermione, opts: Partial): void => { if (hermione.isWorker() || !opts.enabled) { @@ -56,7 +59,17 @@ export = (hermione: Hermione, opts: Partial): void => { hermione.on(hermione.events.INIT, withMiddleware(async () => { const dbClient = await SqliteClient.create({htmlReporter, reportPath: config.path}); - staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient}); + const imageStore = new SqliteImageStore(dbClient); + const expectedPathsCache = new Cache<[TestSpecByPath, string | undefined], string>(getExpectedCacheKey); + + const imagesInfoSaver = new ImagesInfoSaver({ + imageFileSaver: htmlReporter.imagesSaver, + expectedPathsCache, + imageStore, + reportPath: htmlReporter.config.path + }); + + staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient, imagesInfoSaver}); handlingTestResults = Promise.all([ staticReportBuilder.saveStaticFiles(), @@ -92,7 +105,7 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport hermione.on(hermione.events.TEST_PASS, testResult => { promises.push(queue.add(async () => { - const formattedResult = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResult = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT); await reportBuilder.addTestResult(formattedResult); }).catch(reject)); @@ -102,7 +115,7 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport promises.push(queue.add(async () => { const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); await reportBuilder.addTestResult(formattedResult); }).catch((e) => { @@ -114,7 +127,7 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport promises.push(queue.add(async () => { const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); await reportBuilder.addTestResult(formattedResult); }).catch((e) => { @@ -124,7 +137,7 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport hermione.on(hermione.events.TEST_PENDING, testResult => { promises.push(queue.add(async () => { - const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT); await reportBuilder.addTestResult(formattedResult); }).catch(reject)); diff --git a/lib/cache.ts b/lib/cache.ts new file mode 100644 index 000000000..5a8fc4227 --- /dev/null +++ b/lib/cache.ts @@ -0,0 +1,33 @@ +export class Cache { + private _getKeyHash: (key: Key) => string; + private _cache: Map; + + constructor(hashFn: (key: Key) => string) { + this._getKeyHash = hashFn; + this._cache = new Map(); + } + + has(key: Key): boolean { + const keyHash = this._getKeyHash(key); + + return this._cache.has(keyHash); + } + + get(key: Key): Value | undefined { + const keyHash = this._getKeyHash(key); + + return this._cache.get(keyHash); + } + + set(key: Key, value: Value): this { + const keyHash = this._getKeyHash(key); + + if (value !== undefined) { + this._cache.set(keyHash, value); + } else { + this._cache.delete(keyHash); + } + + return this; + } +} diff --git a/lib/common-utils.ts b/lib/common-utils.ts index 1988b43e4..25bab40e4 100644 --- a/lib/common-utils.ts +++ b/lib/common-utils.ts @@ -17,7 +17,15 @@ import { } from './constants'; import {CHECKED, INDETERMINATE, UNCHECKED} from './constants/checked-statuses'; -import {ImageBase64, ImageData, ImageInfoFail, ImageInfoFull, TestError} from './types'; +import { + ImageBase64, + ImageBuffer, + ImageFile, + ImageInfoDiff, + ImageInfoFull, + ImageInfoWithState, + TestError +} from './types'; import {ErrorName, ImageDiffError, NoRefImageError} from './errors'; import {ReporterTestResult} from './test-adapter'; @@ -119,7 +127,7 @@ export const hasNoRefImageErrors = ({assertViewResults = []}: {assertViewResults export const hasFailedImages = (imagesInfo: ImageInfoFull[] = []): boolean => { return imagesInfo.some((imageInfo: ImageInfoFull) => { - return (imageInfo as ImageInfoFail).stateName && + return (imageInfo as ImageInfoDiff).stateName && (isErrorStatus(imageInfo.status) || isFailStatus(imageInfo.status) || isNoRefImageError(imageInfo) || isImageDiffError(imageInfo)); }); }; @@ -164,7 +172,7 @@ export const determineStatus = (testResult: Pick { +export const isBase64Image = (image: ImageFile | ImageBuffer | ImageBase64 | null | undefined): image is ImageBase64 => { return Boolean((image as ImageBase64 | undefined)?.base64); }; @@ -244,3 +252,15 @@ export const getTitleDelimiter = (toolName: ToolName): string => { export function getDetailsFileName(testId: string, browserId: string, attempt: number): string { return `${testId}-${browserId}_${Number(attempt) + 1}_${Date.now()}.json`; } + +export const getTestHash = (testResult: ReporterTestResult): string => { + return testResult.testPath.concat(testResult.browserId, testResult.attempt.toString()).join(' '); +}; + +export const isImageBufferData = (imageData: ImageBuffer | ImageFile | ImageBase64 | undefined): imageData is ImageBuffer => { + return Boolean((imageData as ImageBuffer).buffer); +}; + +export const isImageInfoWithState = (imageInfo: ImageInfoFull): imageInfo is ImageInfoWithState => { + return Boolean((imageInfo as ImageInfoWithState).stateName); +}; diff --git a/lib/constants/errors.js b/lib/constants/errors.js deleted file mode 100644 index 1d1c07c6e..000000000 --- a/lib/constants/errors.js +++ /dev/null @@ -1,9 +0,0 @@ -'use strict'; - -module.exports = { - getCommonErrors: () => ({ - NO_REF_IMAGE_ERROR: 'NoRefImageError', - ASSERT_VIEW_ERROR: 'AssertViewError' - }), - ERROR_TITLE_TEXT_LENGTH: 200 -}; diff --git a/lib/constants/errors.ts b/lib/constants/errors.ts new file mode 100644 index 000000000..f90376362 --- /dev/null +++ b/lib/constants/errors.ts @@ -0,0 +1,3 @@ +export const ERROR_TITLE_TEXT_LENGTH = 200; + +export const NEW_ISSUE_LINK = 'https://github.com/gemini-testing/html-reporter/issues/new'; diff --git a/lib/constants/index.ts b/lib/constants/index.ts index ea288b471..da621fc00 100644 --- a/lib/constants/index.ts +++ b/lib/constants/index.ts @@ -2,6 +2,7 @@ export * from './browser'; export * from './database'; export * from './defaults'; export * from './diff-modes'; +export * from './errors'; export * from './group-tests'; export * from './paths'; export * from './tests'; diff --git a/lib/errors/index.ts b/lib/errors/index.ts index 0e6b1d449..f68dcab3c 100644 --- a/lib/errors/index.ts +++ b/lib/errors/index.ts @@ -1,5 +1,5 @@ import {CoordBounds} from 'looks-same'; -import {DiffOptions, ImageData} from '../types'; +import {DiffOptions, ImageFile} from '../types'; import {ValueOf} from 'type-fest'; export const ErrorName = { @@ -17,11 +17,11 @@ export interface ImageDiffError { stack: string; stateName: string; diffOpts: DiffOptions; - currImg: ImageData; - refImg: ImageData; + currImg: ImageFile; + refImg: ImageFile; diffClusters: CoordBounds[]; diffBuffer?: ArrayBuffer; - diffImg?: ImageData; + diffImg?: ImageFile; } export interface NoRefImageError { @@ -29,6 +29,6 @@ export interface NoRefImageError { stateName: string; message: string; stack?: string; - currImg: ImageData; - refImg?: ImageData; + currImg: ImageFile; + refImg?: ImageFile; } diff --git a/lib/gui/tool-runner/index.ts b/lib/gui/tool-runner/index.ts index 5eed1cd92..c8f3e1a6e 100644 --- a/lib/gui/tool-runner/index.ts +++ b/lib/gui/tool-runner/index.ts @@ -26,14 +26,14 @@ import { } from '../../constants'; import {formatId, mkFullTitle, mergeDatabasesForReuse, filterByEqualDiffSizes} from './utils'; import {getTestsTreeFromDatabase} from '../../db-utils/server'; -import {formatTestResult} from '../../server-utils'; +import {formatTestResult, getExpectedCacheKey} from '../../server-utils'; import { AssertViewResult, HermioneTestResult, HtmlReporterApi, - ImageData, - ImageInfoFail, - ReporterConfig + ImageFile, + ImageInfoDiff, ImageInfoUpdated, + ReporterConfig, TestSpecByPath } from '../../types'; import {GuiCliOptions, GuiConfigs} from '../index'; import {Tree, TreeImage} from '../../tests-tree-builder/base'; @@ -41,19 +41,21 @@ import {TestSpec} from './runner/runner'; import {Response} from 'express'; import {TestBranch, TestEqualDiffsData, TestRefUpdateData} from '../../tests-tree-builder/gui'; import {ReporterTestResult} from '../../test-adapter'; -import {ImagesInfoFormatter} from '../../image-handler'; import {SqliteClient} from '../../sqlite-client'; import PQueue from 'p-queue'; import os from 'os'; +import {Cache} from '../../cache'; +import {ImagesInfoSaver} from '../../images-info-saver'; +import {SqliteImageStore} from '../../image-store'; type ToolRunnerArgs = [paths: string[], hermione: Hermione & HtmlReporterApi, configs: GuiConfigs]; export type ToolRunnerTree = GuiReportBuilderResult & Pick; interface HermioneTestExtended extends HermioneTest { - assertViewResults: {stateName: string, refImg: ImageData, currImg: ImageData}; + assertViewResults: {stateName: string, refImg: ImageFile, currImg: ImageFile}; attempt: number; - imagesInfo: Pick[]; + imagesInfo: Pick[]; } type HermioneTestPlain = Pick; @@ -67,10 +69,9 @@ export interface UndoAcceptImagesResult { const formatTestResultUnsafe = ( test: HermioneTest | HermioneTestExtended | HermioneTestPlain, status: TestStatus, - attempt: number, - {imageHandler}: {imageHandler: ImagesInfoFormatter} + attempt: number ): ReporterTestResult => { - return formatTestResult(test as HermioneTestResult, status, attempt, {imageHandler}); + return formatTestResult(test as HermioneTestResult, status, attempt); }; export class ToolRunner { @@ -85,6 +86,7 @@ export class ToolRunner { private _eventSource: EventSource; protected _reportBuilder: GuiReportBuilder | null; private _tests: Record; + private _expectedImagesCache: Cache<[TestSpecByPath, string | undefined], string>; static create(this: new (...args: ToolRunnerArgs) => T, ...args: ToolRunnerArgs): T { return new this(...args); @@ -105,6 +107,8 @@ export class ToolRunner { this._reportBuilder = null; this._tests = {}; + + this._expectedImagesCache = new Cache(getExpectedCacheKey); } get config(): HermioneConfig { @@ -119,8 +123,16 @@ export class ToolRunner { await mergeDatabasesForReuse(this._reportPath); const dbClient = await SqliteClient.create({htmlReporter: this._hermione.htmlReporter, reportPath: this._reportPath, reuse: true}); + const imageStore = new SqliteImageStore(dbClient); + + const imagesInfoSaver = new ImagesInfoSaver({ + imageFileSaver: this._hermione.htmlReporter.imagesSaver, + expectedPathsCache: this._expectedImagesCache, + imageStore, + reportPath: this._hermione.htmlReporter.config.path + }); - this._reportBuilder = GuiReportBuilder.create(this._hermione.htmlReporter, this._pluginConfig, {dbClient}); + this._reportBuilder = GuiReportBuilder.create(this._hermione.htmlReporter, this._pluginConfig, {dbClient, imagesInfoSaver}); this._subscribeOnEvents(); this._collection = await this._readTests(); @@ -174,7 +186,7 @@ export class ToolRunner { const [selectedImage, ...comparedImages] = this._ensureReportBuilder().getImageDataToFindEqualDiffs(imageIds); const imagesWithEqualBrowserName = comparedImages.filter((image) => image.browserName === selectedImage.browserName); - const imagesWithEqualDiffSizes = filterByEqualDiffSizes(imagesWithEqualBrowserName, (selectedImage as ImageInfoFail).diffClusters); + const imagesWithEqualDiffSizes = filterByEqualDiffSizes(imagesWithEqualBrowserName, (selectedImage as ImageInfoDiff).diffClusters); return _.isEmpty(imagesWithEqualDiffSizes) ? [] : [selectedImage].concat(imagesWithEqualDiffSizes); } @@ -185,22 +197,14 @@ export class ToolRunner { return Promise.all(tests.map(async (test): Promise => { const updateResult = this._prepareTestResult(test); - const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT, reportBuilder); - - const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); - - updateResult.attempt = formattedResult.attempt; + const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT); + const formattedResult = reportBuilder.provideAttempt(formattedResultWithoutAttempt); - await Promise.all(updateResult.imagesInfo.map(async (imageInfo) => { - const {stateName} = imageInfo; + const formattedResultUpdated = await reporterHelper.updateReferenceImages(formattedResult, this._reportPath, this._handleReferenceUpdate.bind(this)); - await reporterHelper.updateReferenceImage(formattedResult, this._reportPath, stateName); + await reportBuilder.addTestResult(formattedResultUpdated); - const result = _.extend(updateResult, {refImg: imageInfo.expectedImg}); - this._emitUpdateReference(result, stateName); - })); - - return reportBuilder.getTestBranch(formattedResult.id); + return reportBuilder.getTestBranch(formattedResultUpdated.id); })); } @@ -210,7 +214,7 @@ export class ToolRunner { await Promise.all(tests.map(async (test) => { const updateResult = this._prepareTestResult(test); - const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT); await Promise.all(updateResult.imagesInfo.map(async (imageInfo) => { const {stateName} = imageInfo; @@ -241,10 +245,10 @@ export class ToolRunner { } if (previousExpectedPath && (updateResult as HermioneTest).fullTitle) { - reportBuilder.imageHandler.updateCacheExpectedPath({ - fullName: (updateResult as HermioneTest).fullTitle(), + this._expectedImagesCache.set([{ + testPath: [(updateResult as HermioneTest).fullTitle()], browserId: (updateResult as HermioneTest).browserId - }, stateName, previousExpectedPath); + }, stateName], previousExpectedPath); } })); })); @@ -253,7 +257,7 @@ export class ToolRunner { } async findEqualDiffs(images: TestEqualDiffsData[]): Promise { - const [selectedImage, ...comparedImages] = images as (ImageInfoFail & {diffClusters: CoordBounds[]})[]; + const [selectedImage, ...comparedImages] = images as (ImageInfoDiff & {diffClusters: CoordBounds[]})[]; const {tolerance, antialiasingTolerance} = this.config; const compareOpts = {tolerance, antialiasingTolerance, stopOnFirstFail: true, shouldCluster: false}; @@ -316,9 +320,9 @@ export class ToolRunner { this._tests[testId] = _.extend(test, {browserId}); if (test.pending) { - queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder))); + queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, SKIPPED, UNKNOWN_ATTEMPT))); } else { - queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, IDLE, UNKNOWN_ATTEMPT, reportBuilder))); + queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, IDLE, UNKNOWN_ATTEMPT))); } }); @@ -345,7 +349,7 @@ export class ToolRunner { const imagesInfo = test.imagesInfo .filter(({stateName, actualImg}) => Boolean(stateName) && Boolean(actualImg)) .map((imageInfo) => { - const {stateName, actualImg} = imageInfo as {stateName: string, actualImg: ImageData}; + const {stateName, actualImg} = imageInfo as {stateName: string, actualImg: ImageFile}; const path = this._hermione.config.browsers[browserId].getScreenshotPath(rawTest, stateName); const refImg = {path, size: actualImg.size}; @@ -370,10 +374,12 @@ export class ToolRunner { : res; } - protected _emitUpdateReference({refImg}: {refImg: ImageData}, state: string): void { + protected _handleReferenceUpdate(testResult: ReporterTestResult, imageInfo: ImageInfoUpdated, state: string): void { + this._expectedImagesCache.set([testResult, imageInfo.stateName], imageInfo.expectedImg.path); + this._hermione.emit( this._hermione.events.UPDATE_REFERENCE, - {refImg, state} + {refImg: imageInfo.refImg, state} ); } diff --git a/lib/gui/tool-runner/report-subscriber.ts b/lib/gui/tool-runner/report-subscriber.ts index 5edd00145..aa3791694 100644 --- a/lib/gui/tool-runner/report-subscriber.ts +++ b/lib/gui/tool-runner/report-subscriber.ts @@ -31,7 +31,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo hermione.on(hermione.events.TEST_BEGIN, (data) => { queue.add(async () => { - const formattedResultWithoutAttempt = formatTestResult(data as HermioneTestResult, RUNNING, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(data as HermioneTestResult, RUNNING, UNKNOWN_ATTEMPT); const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); @@ -42,7 +42,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo hermione.on(hermione.events.TEST_PASS, (testResult) => { queue.add(async () => { - const formattedResultWithoutAttempt = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT); const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); @@ -55,7 +55,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo queue.add(async () => { const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); @@ -68,7 +68,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo queue.add(async () => { const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); @@ -79,7 +79,7 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo hermione.on(hermione.events.TEST_PENDING, async (testResult) => { queue.add(async () => { - const formattedResultWithoutAttempt = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT); const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); diff --git a/lib/gui/tool-runner/utils.ts b/lib/gui/tool-runner/utils.ts index c0732e838..f69af9d3d 100644 --- a/lib/gui/tool-runner/utils.ts +++ b/lib/gui/tool-runner/utils.ts @@ -9,7 +9,7 @@ import {logger} from '../../common-utils'; import {DATABASE_URLS_JSON_NAME, LOCAL_DATABASE_NAME} from '../../constants'; import {mergeTables} from '../../db-utils/server'; import {TestEqualDiffsData, TestRefUpdateData} from '../../tests-tree-builder/gui'; -import {ImageInfoFail, ImageSize} from '../../types'; +import {ImageInfoDiff, ImageSize} from '../../types'; export const formatId = (hash: string, browserId: string): string => `${hash}/${browserId}`; @@ -58,7 +58,7 @@ export const filterByEqualDiffSizes = (imagesInfo: TestEqualDiffsData[], refDiff const refDiffSizes = refDiffClusters.map(getDiffClusterSizes); return _.filter(imagesInfo, (imageInfo) => { - const imageInfoFail = imageInfo as ImageInfoFail; + const imageInfoFail = imageInfo as ImageInfoDiff; const imageDiffSizes = imageInfoFail.diffClusters?.map(getDiffClusterSizes) ?? []; const equal = compareDiffSizes(imageDiffSizes, refDiffSizes); diff --git a/lib/image-handler.ts b/lib/image-handler.ts deleted file mode 100644 index c53badaf9..000000000 --- a/lib/image-handler.ts +++ /dev/null @@ -1,366 +0,0 @@ -import path from 'path'; -import EventEmitter2 from 'eventemitter2'; -import fs from 'fs-extra'; -import _ from 'lodash'; -import tmp from 'tmp'; - -import type {ImageStore} from './image-store'; -import {RegisterWorkers} from './workers/create-workers'; -import * as utils from './server-utils'; -import { - AssertViewResult, - ImageBase64, - ImageData, - ImageInfo, ImageInfoError, - ImageInfoFail, - ImageInfoFull, - ImagesSaver, - ImageInfoPageSuccess -} from './types'; -import {ERROR, FAIL, PluginEvents, SUCCESS, TestStatus, UNKNOWN_ATTEMPT, UPDATED} from './constants'; -import { - getError, - getShortMD5, - isBase64Image, - isImageDiffError, - isNoRefImageError, - logger, - mkTestId -} from './common-utils'; -import {ImageDiffError} from './errors'; -import {cacheExpectedPaths, cacheAllImages, cacheDiffImages} from './image-cache'; -import {ReporterTestResult} from './test-adapter'; - -// A type to prevent accidental infinite recursion on a type level -export type ReporterTestResultPlain = Omit; - -export interface ImagesInfoFormatter { - getImagesInfo(testResult: ReporterTestResultPlain): ImageInfoFull[]; -} - -export interface ImageHandlerOptions { - reportPath: string; -} - -interface TestSpec { - fullName: string; - browserId: string; -} - -export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter { - private _imageStore: ImageStore; - private _imagesSaver: ImagesSaver; - private _options: ImageHandlerOptions; - - constructor(imageStore: ImageStore, imagesSaver: ImagesSaver, options: ImageHandlerOptions) { - super(); - this._imageStore = imageStore; - this._imagesSaver = imagesSaver; - this._options = options; - } - - static getCurrImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined { - return _.get(_.find(assertViewResults, {stateName}), 'currImg'); - } - - static getDiffImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined { - return _.get(_.find(assertViewResults, {stateName}), 'diffImg'); - } - - static getRefImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined { - return _.get(_.find(assertViewResults, {stateName}), 'refImg'); - } - - static getScreenshot(testResult: ReporterTestResultPlain): ImageBase64 | ImageData | null | undefined { - return testResult.screenshot; - } - - getImagesFor(testResult: ReporterTestResultPlain, assertViewStatus: TestStatus, stateName?: string): ImageInfo | undefined { - const refImg = ImageHandler.getRefImg(testResult.assertViewResults, stateName); - const currImg = ImageHandler.getCurrImg(testResult.assertViewResults, stateName); - - const pageImg = ImageHandler.getScreenshot(testResult); - - const {path: refPath} = this._getExpectedPath(testResult, stateName); - const currPath = utils.getCurrentPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName}); - const diffPath = utils.getDiffPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName}); - - // Handling whole page common screenshots - if (!stateName && pageImg) { - return { - actualImg: { - path: this._getImgFromStorage(currPath), - size: pageImg.size - } - }; - } - - if ((assertViewStatus === SUCCESS || assertViewStatus === UPDATED) && refImg) { - const result: ImageInfo = { - expectedImg: {path: this._getImgFromStorage(refPath), size: refImg.size} - }; - if (currImg) { - result.actualImg = {path: this._getImgFromStorage(currPath), size: currImg.size}; - } - - return result; - } - - if (assertViewStatus === FAIL && refImg && currImg) { - return { - expectedImg: { - path: this._getImgFromStorage(refPath), - size: refImg.size - }, - actualImg: { - path: this._getImgFromStorage(currPath), - size: currImg.size - }, - diffImg: { - path: this._getImgFromStorage(diffPath), - size: { - width: _.max([_.get(refImg, 'size.width'), _.get(currImg, 'size.width')]) as number, - height: _.max([_.get(refImg, 'size.height'), _.get(currImg, 'size.height')]) as number - } - } - }; - } - - if (assertViewStatus === ERROR && currImg) { - return { - actualImg: { - path: this._getImgFromStorage(currPath), - size: currImg.size - } - }; - } - - return; - } - - getImagesInfo(testResult: ReporterTestResultPlain): ImageInfoFull[] { - const imagesInfo: ImageInfoFull[] = testResult.assertViewResults?.map((assertResult): ImageInfoFull => { - let status: TestStatus, error: {message: string; stack?: string;} | undefined; - - if (assertResult.isUpdated === true) { - status = UPDATED; - } else if (isImageDiffError(assertResult)) { - status = FAIL; - } else if (isNoRefImageError(assertResult)) { - status = ERROR; - error = _.pick(assertResult, ['message', 'name', 'stack']); - } else { - status = SUCCESS; - } - - const {stateName, refImg} = assertResult; - const diffClusters = (assertResult as ImageDiffError).diffClusters; - - return _.extend( - {stateName, refImg, status: status, error, diffClusters}, - this.getImagesFor(testResult, status, stateName) - ) as ImageInfoFull; - }) ?? []; - - // Common page screenshot - if (ImageHandler.getScreenshot(testResult)) { - const error = getError(testResult.error); - - if (!_.isEmpty(error)) { - imagesInfo.push(_.extend( - {status: ERROR, error}, - this.getImagesFor(testResult, ERROR) - ) as ImageInfoError); - } else { - imagesInfo.push(_.extend( - {status: SUCCESS}, - this.getImagesFor(testResult, SUCCESS) - ) as ImageInfoPageSuccess); - } - } - - return imagesInfo; - } - - async saveTestImages(testResult: ReporterTestResultPlain, worker: RegisterWorkers<['saveDiffTo']>): Promise { - const {assertViewResults = []} = testResult; - - const result = await Promise.all(assertViewResults.map(async (assertResult) => { - const {stateName} = assertResult; - const {path: destRefPath, reused: reusedReference} = this._getExpectedPath(testResult, stateName); - const srcRefPath = ImageHandler.getRefImg(testResult.assertViewResults, stateName)?.path; - - const destCurrPath = utils.getCurrentPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName}); - const srcCurrPath = ImageHandler.getCurrImg(testResult.assertViewResults, stateName)?.path; - - const destDiffPath = utils.getDiffPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName}); - const srcDiffPath = ImageHandler.getDiffImg(assertViewResults, stateName)?.path ?? path.resolve(tmp.tmpdir, destDiffPath); - const actions: unknown[] = []; - - if (!(assertResult instanceof Error)) { - actions.push(this._saveImg(srcRefPath, destRefPath)); - } - - if (isImageDiffError(assertResult)) { - if (!assertResult.diffImg) { - await this._saveDiffInWorker(assertResult, srcDiffPath, worker); - } - - actions.push( - this._saveImg(srcCurrPath, destCurrPath), - this._saveImg(srcDiffPath, destDiffPath) - ); - - if (!reusedReference) { - actions.push(this._saveImg(srcRefPath, destRefPath)); - } - } - - if (isNoRefImageError(assertResult)) { - actions.push(this._saveImg(srcCurrPath, destCurrPath)); - } - - return Promise.all(actions); - })); - - if (ImageHandler.getScreenshot(testResult)) { - await this._savePageScreenshot(testResult); - } - - await this.emitAsync(PluginEvents.TEST_SCREENSHOTS_SAVED, { - testId: mkTestId(testResult.fullName, testResult.browserId), - attempt: testResult.attempt, - imagesInfo: this.getImagesInfo(testResult) - }); - - return result; - } - - setImagesSaver(newImagesSaver: ImagesSaver): void { - this._imagesSaver = newImagesSaver; - } - - updateCacheExpectedPath(testResult: TestSpec, stateName: string, expectedPath: string): void { - const key = this._getExpectedKey(testResult, stateName); - - if (expectedPath) { - cacheExpectedPaths.set(key, expectedPath); - } else { - cacheExpectedPaths.delete(key); - } - } - - private _getExpectedKey(testResult: TestSpec, stateName?: string): string { - const shortTestId = getShortMD5(mkTestId(testResult.fullName, testResult.browserId)); - - return shortTestId + '#' + stateName; - } - - private _getExpectedPath(testResult: ReporterTestResultPlain, stateName?: string): {path: string, reused: boolean} { - const key = this._getExpectedKey(testResult, stateName); - let result: {path: string; reused: boolean}; - - if (testResult.status === UPDATED) { - const expectedPath = utils.getReferencePath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName}); - - result = {path: expectedPath, reused: false}; - } else if (cacheExpectedPaths.has(key)) { - result = {path: cacheExpectedPaths.get(key) as string, reused: true}; - } else { - const imageInfo = this._imageStore.getLastImageInfoFromDb(testResult, stateName); - - if (imageInfo && (imageInfo as ImageInfoFail).expectedImg) { - const expectedPath = (imageInfo as ImageInfoFail).expectedImg.path; - - result = {path: expectedPath, reused: true}; - } else { - const expectedPath = utils.getReferencePath({ - attempt: testResult.attempt, - browserId: testResult.browserId, - imageDir: testResult.imageDir, - stateName - }); - - result = {path: expectedPath, reused: false}; - } - } - - if (testResult.attempt !== UNKNOWN_ATTEMPT) { - cacheExpectedPaths.set(key, result.path); - } - - return result; - } - - private _getImgFromStorage(imgPath: string): string { - // fallback for updating image in gui mode - return cacheAllImages.get(imgPath) || imgPath; - } - - private async _saveDiffInWorker(imageDiffError: ImageDiffError, destPath: string, worker: RegisterWorkers<['saveDiffTo']>): Promise { - await utils.makeDirFor(destPath); - - // new versions of hermione provide `diffBuffer` - if (imageDiffError.diffBuffer) { - const pngBuffer = Buffer.from(imageDiffError.diffBuffer); - - await fs.writeFile(destPath, pngBuffer); - - return; - } - - const currPath = imageDiffError.currImg.path; - const refPath = imageDiffError.refImg.path; - - const [currBuffer, refBuffer] = await Promise.all([ - fs.readFile(currPath), - fs.readFile(refPath) - ]); - - const hash = utils.createHash(currBuffer) + utils.createHash(refBuffer); - - if (cacheDiffImages.has(hash)) { - const cachedDiffPath = cacheDiffImages.get(hash) as string; - - await fs.copy(cachedDiffPath, destPath); - return; - } - - await worker.saveDiffTo(imageDiffError, destPath); - - cacheDiffImages.set(hash, destPath); - } - - private async _savePageScreenshot(testResult: ReporterTestResultPlain): Promise { - const screenshot = ImageHandler.getScreenshot(testResult); - if (!(screenshot as ImageBase64)?.base64 && !(screenshot as ImageData)?.path) { - logger.warn('Cannot save screenshot on reject'); - - return Promise.resolve(); - } - - const currPath = utils.getCurrentPath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir}); - let localPath: string; - - if (isBase64Image(screenshot)) { - localPath = path.resolve(tmp.tmpdir, currPath); - await utils.makeDirFor(localPath); - await fs.writeFile(localPath, new Buffer(screenshot.base64, 'base64'), 'base64'); - } else { - localPath = screenshot?.path as string; - } - - await this._saveImg(localPath, currPath); - } - - private async _saveImg(localPath: string | undefined, destPath: string): Promise { - if (!localPath) { - return Promise.resolve(undefined); - } - - const res = await this._imagesSaver.saveImg(localPath, {destPath, reportDir: this._options.reportPath}); - - cacheAllImages.set(destPath, res || destPath); - return res; - } -} diff --git a/lib/image-store.ts b/lib/image-store.ts index 5970f0ea3..44805afa8 100644 --- a/lib/image-store.ts +++ b/lib/image-store.ts @@ -1,10 +1,9 @@ import {DB_COLUMNS} from './constants'; import {SqliteClient} from './sqlite-client'; -import {ImageInfo, ImageInfoFull, LabeledSuitesRow} from './types'; -import {ReporterTestResultPlain} from './image-handler'; +import {ImageInfo, ImageInfoFull, LabeledSuitesRow, TestSpecByPath} from './types'; export interface ImageStore { - getLastImageInfoFromDb(testResult: ReporterTestResultPlain, stateName?: string): ImageInfo | undefined ; + getLastImageInfoFromDb(testResult: TestSpecByPath, stateName?: string): ImageInfo | undefined ; } export class SqliteImageStore implements ImageStore { @@ -14,7 +13,7 @@ export class SqliteImageStore implements ImageStore { this._dbClient = dbClient; } - getLastImageInfoFromDb(testResult: ReporterTestResultPlain, stateName?: string): ImageInfo | undefined { + getLastImageInfoFromDb(testResult: TestSpecByPath, stateName?: string): ImageInfo | undefined { const browserName = testResult.browserId; const suitePath = testResult.testPath; const suitePathString = JSON.stringify(suitePath); @@ -23,7 +22,8 @@ export class SqliteImageStore implements ImageStore { select: DB_COLUMNS.IMAGES_INFO, where: `${DB_COLUMNS.SUITE_PATH} = ? AND ${DB_COLUMNS.NAME} = ?`, orderBy: DB_COLUMNS.TIMESTAMP, - orderDescending: true + orderDescending: true, + noCache: true }, suitePathString, browserName); const imagesInfo: ImageInfoFull[] = imagesInfoResult && JSON.parse(imagesInfoResult[DB_COLUMNS.IMAGES_INFO as keyof Pick]) || []; diff --git a/lib/images-info-saver.ts b/lib/images-info-saver.ts new file mode 100644 index 000000000..f8c5c4581 --- /dev/null +++ b/lib/images-info-saver.ts @@ -0,0 +1,231 @@ +import util from 'util'; +import makeDebug from 'debug'; +import EventEmitter2 from 'eventemitter2'; +import fs from 'fs-extra'; +import sizeOf from 'image-size'; +import _ from 'lodash'; +import PQueue from 'p-queue'; + +import {RegisterWorkers} from './workers/create-workers'; +import {ReporterTestResult} from './test-adapter'; +import { + DiffOptions, ImageBase64, ImageBuffer, + ImageFile, + ImageFileSaver, + ImageInfoDiff, + ImageInfoFull, + ImageSize, TestSpecByPath +} from './types'; +import {copyAndUpdate, removeBufferFromImagesInfo} from './test-adapter/utils'; +import {cacheDiffImages} from './image-cache'; +import {NEW_ISSUE_LINK, PluginEvents, TestStatus, UPDATED} from './constants'; +import {createHash, getCurrentPath, getDiffPath, getReferencePath, getTempPath, makeDirFor} from './server-utils'; +import {isBase64Image, mkTestId, isImageBufferData} from './common-utils'; +import {ImageStore} from './image-store'; +import {Cache} from './cache'; + +const debug = makeDebug('html-reporter:images-info-saver'); + +interface ImagesInfoSaverOptions { + imageFileSaver: ImageFileSaver; + reportPath: string; + imageStore: ImageStore; + expectedPathsCache: Cache<[TestSpecByPath, string | undefined], string>; +} + +export class ImagesInfoSaver extends EventEmitter2 { + private _imageFileSaver: ImageFileSaver; + private _reportPath: string; + private _imageStore: ImageStore; + private _expectedPathsCache: Cache<[TestSpecByPath, string | undefined], string>; + + constructor(options: ImagesInfoSaverOptions) { + super(); + + this._imageFileSaver = options.imageFileSaver; + this._reportPath = options.reportPath; + this._imageStore = options.imageStore; + this._expectedPathsCache = options.expectedPathsCache; + } + + async save(testResult: ReporterTestResult, workers?: RegisterWorkers<['saveDiffTo']>): Promise { + const testDebug = debug.extend(testResult.imageDir); + testDebug(`Saving images of ${testResult.id}`); + + const newImagesInfos: ImageInfoFull[] = []; + + await Promise.all(testResult.imagesInfo.map(async (imagesInfo, index) => { + const imageDebug = testDebug.extend(index.toString()); + imageDebug.enabled && imageDebug('Handling %j', removeBufferFromImagesInfo(imagesInfo)); + + const newImagesInfo = _.clone(imagesInfo); + const {stateName} = imagesInfo as ImageInfoDiff; + const actions = new PQueue(); + + actions.add(async () => { + (newImagesInfo as {actualImg?: ImageFile}).actualImg + = await this._saveActualImageIfNeeded(testResult, imagesInfo, stateName, {logger: imageDebug}); + }); + + actions.add(async () => { + (newImagesInfo as {diffImg?: ImageFile}).diffImg = + await this._saveDiffImageIfNeeded(testResult, imagesInfo, stateName, {workers, logger: imageDebug}); + }); + + actions.add(async () => { + (newImagesInfo as {expectedImg?: ImageFile}).expectedImg = + await this._saveExpectedImageIfNeeded(testResult, imagesInfo, stateName, {logger: imageDebug}); + }); + + await actions.onIdle(); + + newImagesInfos.push(_.omitBy(newImagesInfo, _.isNil) as ImageInfoFull); + })); + + await this.emitAsync(PluginEvents.TEST_SCREENSHOTS_SAVED, { + testId: mkTestId(testResult.fullName, testResult.browserId), + attempt: testResult.attempt, + imagesInfo: newImagesInfos + }); + + return copyAndUpdate(testResult, {imagesInfo: newImagesInfos}); + } + + setImageFileSaver(imageFileSaver: ImageFileSaver): void { + this._imageFileSaver = imageFileSaver; + } + + private async _createDiffInFile(imagesInfo: ImageInfoDiff, filePath: string, workers: RegisterWorkers<['saveDiffTo']>): Promise { + await makeDirFor(filePath); + + const actualPath = imagesInfo.actualImg.path; + const expectedPath = imagesInfo.expectedImg.path; + + const [currBuffer, refBuffer] = await Promise.all([ + fs.readFile(actualPath), + fs.readFile(expectedPath) + ]); + + const hash = createHash(currBuffer) + createHash(refBuffer); + + if (cacheDiffImages.has(hash)) { + const cachedDiffPath = cacheDiffImages.get(hash) as string; + + await fs.copy(cachedDiffPath, filePath); + } else { + await workers.saveDiffTo({ + ...imagesInfo.diffOptions, + reference: expectedPath, + current: actualPath + } satisfies DiffOptions, filePath); + + cacheDiffImages.set(hash, filePath); + } + + return {path: filePath, size: _.pick(sizeOf(filePath), ['height', 'width']) as ImageSize}; + } + + private _getReusedExpectedPath(testResult: TestSpecByPath, imagesInfo: ImageInfoFull): string | null { + if (imagesInfo.status === UPDATED) { + return null; + } + + const {stateName} = imagesInfo as ImageInfoDiff; + + if (this._expectedPathsCache.has([testResult, stateName])) { + return this._expectedPathsCache.get([testResult, stateName]) as string; + } + + const lastImageInfo = this._imageStore.getLastImageInfoFromDb(testResult, stateName) as ImageInfoDiff; + + if (lastImageInfo && lastImageInfo.expectedImg) { + this._expectedPathsCache.set([testResult, stateName], (lastImageInfo.expectedImg as ImageFile).path); + return (lastImageInfo.expectedImg as ImageFile).path; + } + + return null; + } + + private async _saveImage(imageData: ImageFile | ImageBuffer | ImageBase64, destPath: string): Promise { + const sourceFilePath = isImageBufferData(imageData) || isBase64Image(imageData) ? getTempPath(destPath) : imageData.path; + if (isImageBufferData(imageData)) { + await fs.writeFile(sourceFilePath, Buffer.from(imageData.buffer)); + } else if (isBase64Image(imageData)) { + await makeDirFor(sourceFilePath); + await fs.writeFile(sourceFilePath, Buffer.from(imageData.base64, 'base64'), 'base64'); + } + + const savedFilePath = await this._imageFileSaver.saveImg(sourceFilePath, { + destPath, + reportDir: this._reportPath + }); + + return savedFilePath || destPath; + } + + private async _saveActualImageIfNeeded(testResult: ReporterTestResult, imagesInfo: ImageInfoFull, stateName: string | undefined, {logger}: {logger: debug.Debugger}): Promise { + const actualImg = imagesInfo.actualImg; + if (!actualImg) { + return actualImg; + } + + const reportActualPath = getCurrentPath(testResult, stateName); + + const newActualPath = await this._saveImage(actualImg, reportActualPath); + logger(`Saved actual image from ${(actualImg as ImageFile).path ?? ''} to ${newActualPath}`); + + return {path: newActualPath, size: actualImg.size}; + } + + private async _saveDiffImageIfNeeded( + testResult: ReporterTestResult, + imagesInfo: ImageInfoFull, + stateName: string | undefined, + {workers, logger}: {workers?: RegisterWorkers<['saveDiffTo']>, logger: debug.Debugger} + ): Promise { + const shouldSaveDiff = imagesInfo.status === TestStatus.FAIL && + (imagesInfo.diffImg || (imagesInfo.actualImg && imagesInfo.expectedImg)); + if (!shouldSaveDiff) { + return; + } + let {diffImg} = imagesInfo; + const reportDiffPath = getDiffPath(testResult, stateName); + + if (!diffImg) { + if (!workers) { + throw new Error('Couldn\'t generate diff image, because workers were not passed.\n' + + util.format('Test result: %o\n', testResult) + + `Please report this error to html-reporter team: ${NEW_ISSUE_LINK}.`); + } + diffImg = await this._createDiffInFile(imagesInfo, reportDiffPath, workers); + logger(`Created new diff in file ${reportDiffPath}`); + } + + const newDiffPath = await this._saveImage(diffImg, reportDiffPath); + logger(`Saved diff image from ${(diffImg as ImageFile).path ?? ''} to ${newDiffPath}`); + + const size = _.pick(sizeOf(isImageBufferData(diffImg) ? Buffer.from(diffImg.buffer) : diffImg.path), ['height', 'width']) as ImageSize; + + return {path: newDiffPath, size}; + } + + private async _saveExpectedImageIfNeeded(testResult: ReporterTestResult, imagesInfo: ImageInfoFull, stateName: string | undefined, {logger}: {logger: debug.Debugger}): Promise { + if (!(imagesInfo as ImageInfoDiff).expectedImg) { + return; + } + const {expectedImg} = imagesInfo as ImageInfoDiff; + const reusedExpectedPath = this._getReusedExpectedPath(testResult, imagesInfo); + const reportDiffPath = reusedExpectedPath ?? getReferencePath(testResult, stateName); + + let newExpectedPath = reportDiffPath; + + if (!reusedExpectedPath) { + newExpectedPath = await this._saveImage(expectedImg, reportDiffPath); + logger(`Saved expected image from ${(expectedImg as ImageFile).path ?? ''} to ${newExpectedPath}`); + } else { + logger(`Reused expected image from ${reusedExpectedPath}`); + } + + return {path: newExpectedPath, size: expectedImg.size}; + } +} diff --git a/lib/local-images-saver.ts b/lib/local-image-file-saver.ts similarity index 67% rename from lib/local-images-saver.ts rename to lib/local-image-file-saver.ts index dea4dc890..205ede6f3 100644 --- a/lib/local-images-saver.ts +++ b/lib/local-image-file-saver.ts @@ -1,7 +1,7 @@ import {copyFileAsync} from './server-utils'; -import type {ImagesSaver} from './types'; +import type {ImageFileSaver} from './types'; -export const LocalImagesSaver: ImagesSaver = { +export const LocalImageFileSaver: ImageFileSaver = { saveImg: async (srcCurrPath, {destPath, reportDir}) => { await copyFileAsync(srcCurrPath, destPath, {reportDir}); diff --git a/lib/plugin-api.ts b/lib/plugin-api.ts index e23f56544..5176d8d7d 100644 --- a/lib/plugin-api.ts +++ b/lib/plugin-api.ts @@ -1,15 +1,15 @@ import EventsEmitter2 from 'eventemitter2'; import {PluginEvents, ToolName} from './constants'; import {downloadDatabases, getTestsTreeFromDatabase, mergeDatabases} from './db-utils/server'; -import {LocalImagesSaver} from './local-images-saver'; +import {LocalImageFileSaver} from './local-image-file-saver'; import {version} from '../package.json'; -import {ImagesSaver, ReporterConfig, ReportsSaver} from './types'; +import {ImageFileSaver, ReporterConfig, ReportsSaver} from './types'; export interface HtmlReporterValues { toolName: ToolName; extraItems: Record; metaInfoExtenders: Record; - imagesSaver: ImagesSaver; + imagesSaver: ImageFileSaver; reportsSaver: ReportsSaver | null; } @@ -41,7 +41,7 @@ export class HtmlReporter extends EventsEmitter2 { toolName: toolName ?? ToolName.Hermione, extraItems: {}, metaInfoExtenders: {}, - imagesSaver: LocalImagesSaver, + imagesSaver: LocalImageFileSaver, reportsSaver: null }; this._version = version; @@ -75,12 +75,12 @@ export class HtmlReporter extends EventsEmitter2 { return this._values.metaInfoExtenders; } - set imagesSaver(imagesSaver: ImagesSaver) { + set imagesSaver(imagesSaver: ImageFileSaver) { this.emit(PluginEvents.IMAGES_SAVER_UPDATED, imagesSaver); this._values.imagesSaver = imagesSaver; } - get imagesSaver(): ImagesSaver { + get imagesSaver(): ImageFileSaver { return this._values.imagesSaver; } diff --git a/lib/report-builder/gui.ts b/lib/report-builder/gui.ts index a491be61f..386c7936e 100644 --- a/lib/report-builder/gui.ts +++ b/lib/report-builder/gui.ts @@ -147,30 +147,31 @@ export class GuiReportBuilder extends StaticReportBuilder { this._skips.push({suite, browser, comment}); } - const formattedResultWithImagePaths = this._extendTestWithImagePaths(formattedResult); + const formattedResultWithImages = this._loadImagesFromPreviousAttempt(formattedResult); - this._testsTree.addTestResult(formattedResultWithImagePaths); + this._testsTree.addTestResult(formattedResultWithImages); - return formattedResultWithImagePaths; + return formattedResultWithImages; } - private _extendTestWithImagePaths(formattedResult: ReporterTestResult): ReporterTestResult { + private _loadImagesFromPreviousAttempt(formattedResult: ReporterTestResult): ReporterTestResult { if (formattedResult.status !== UPDATED) { return formattedResult; } - const failResultId = copyAndUpdate(formattedResult, {attempt: formattedResult.attempt - 1}).id; - const failImagesInfo = _.clone(this._testsTree.getImagesInfo(failResultId)) as ImageInfoFull[]; + const previousResultId = copyAndUpdate(formattedResult, {attempt: formattedResult.attempt - 1}).id; + const newImagesInfo = _.clone(this._testsTree.getImagesInfo(previousResultId)) as ImageInfoFull[]; - if (failImagesInfo.length) { + if (newImagesInfo.length) { formattedResult.imagesInfo?.forEach((imageInfo) => { const {stateName} = imageInfo as ImageInfoWithState; - let index = _.findIndex(failImagesInfo, {stateName}); - index = index >= 0 ? index : _.findLastIndex(failImagesInfo); - failImagesInfo[index] = imageInfo; + let index = _.findIndex(newImagesInfo, {stateName}); + index = index >= 0 ? index : _.findLastIndex(newImagesInfo); + + newImagesInfo[index] = imageInfo; }); } - return copyAndUpdate(formattedResult, {imagesInfo: failImagesInfo}); + return copyAndUpdate(formattedResult, {imagesInfo: newImagesInfo}); } } diff --git a/lib/report-builder/static.ts b/lib/report-builder/static.ts index 38843eceb..ea8d6b1e0 100644 --- a/lib/report-builder/static.ts +++ b/lib/report-builder/static.ts @@ -2,6 +2,7 @@ import path from 'path'; import {GeneralEventEmitter} from 'eventemitter2'; import _ from 'lodash'; import fs from 'fs-extra'; +import PQueue from 'p-queue'; import { IDLE, @@ -15,26 +16,26 @@ import {ReporterTestResult} from '../test-adapter'; import {saveErrorDetails, saveStaticFilesToReportDir, writeDatabaseUrlsFile} from '../server-utils'; import {ReporterConfig} from '../types'; import {HtmlReporter} from '../plugin-api'; -import {ImageHandler} from '../image-handler'; -import {SqliteImageStore} from '../image-store'; import {getTestFromDb} from '../db-utils/server'; import {TestAttemptManager} from '../test-attempt-manager'; import {copyAndUpdate} from '../test-adapter/utils'; import {RegisterWorkers} from '../workers/create-workers'; +import {ImagesInfoSaver} from '../images-info-saver'; const ignoredStatuses = [RUNNING, IDLE]; export interface StaticReportBuilderOptions { dbClient: SqliteClient; + imagesInfoSaver: ImagesInfoSaver; } export class StaticReportBuilder { protected _htmlReporter: HtmlReporter; protected _pluginConfig: ReporterConfig; protected _dbClient: SqliteClient; - protected _imageHandler: ImageHandler; + protected _imagesInfoSaver: ImagesInfoSaver; protected _testAttemptManager: TestAttemptManager; - private _workers: RegisterWorkers<['saveDiffTo']> | null; + private _workers?: RegisterWorkers<['saveDiffTo']>; static create( this: new (htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, options: StaticReportBuilderOptions) => T, @@ -45,7 +46,7 @@ export class StaticReportBuilder { return new this(htmlReporter, pluginConfig, options); } - constructor(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, {dbClient}: StaticReportBuilderOptions) { + constructor(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, {dbClient, imagesInfoSaver}: StaticReportBuilderOptions) { this._htmlReporter = htmlReporter; this._pluginConfig = pluginConfig; @@ -53,20 +54,13 @@ export class StaticReportBuilder { this._testAttemptManager = new TestAttemptManager(); - const imageStore = new SqliteImageStore(this._dbClient); - this._imageHandler = new ImageHandler(imageStore, htmlReporter.imagesSaver, {reportPath: pluginConfig.path}); - - this._workers = null; + this._imagesInfoSaver = imagesInfoSaver; this._htmlReporter.on(PluginEvents.IMAGES_SAVER_UPDATED, (newImagesSaver) => { - this._imageHandler.setImagesSaver(newImagesSaver); + this._imagesInfoSaver.setImageFileSaver(newImagesSaver); }); - this._htmlReporter.listenTo(this._imageHandler as unknown as GeneralEventEmitter, [PluginEvents.TEST_SCREENSHOTS_SAVED]); - } - - get imageHandler(): ImageHandler { - return this._imageHandler; + this._htmlReporter.listenTo(this._imagesInfoSaver as unknown as GeneralEventEmitter, [PluginEvents.TEST_SCREENSHOTS_SAVED]); } async saveStaticFiles(): Promise { @@ -82,17 +76,8 @@ export class StaticReportBuilder { this._workers = workers; } - private _ensureWorkers(): RegisterWorkers<['saveDiffTo']> { - if (!this._workers) { - throw new Error('You must register workers before using report builder.' + - 'Make sure registerWorkers() was called before adding any test results.'); - } - - return this._workers; - } - /** If passed test result doesn't have attempt, this method registers new attempt and sets attempt number */ - private _provideAttempt(testResultOriginal: ReporterTestResult): ReporterTestResult { + provideAttempt(testResultOriginal: ReporterTestResult): ReporterTestResult { let formattedResult = testResultOriginal; if (testResultOriginal.attempt === UNKNOWN_ATTEMPT) { @@ -103,38 +88,41 @@ export class StaticReportBuilder { return formattedResult; } - private async _saveTestResultData(testResult: ReporterTestResult): Promise { + private async _saveTestResultData(testResult: ReporterTestResult): Promise { if ([IDLE, RUNNING, UPDATED].includes(testResult.status)) { - return; + return testResult; } - const actions: Promise[] = []; + const actions = new PQueue(); + let testResultWithImagePaths: ReporterTestResult = testResult; - if (!_.isEmpty(testResult.assertViewResults)) { - actions.push(this._imageHandler.saveTestImages(testResult, this._ensureWorkers())); - } + actions.add(async () => { + testResultWithImagePaths = await this._imagesInfoSaver.save(testResult, this._workers); + }); if (this._pluginConfig.saveErrorDetails && testResult.errorDetails) { - actions.push(saveErrorDetails(testResult, this._pluginConfig.path)); + actions.add(async () => saveErrorDetails(testResult, this._pluginConfig.path)); } - await Promise.all(actions); + await actions.onIdle(); + + return testResultWithImagePaths; } async addTestResult(formattedResultOriginal: ReporterTestResult): Promise { - const formattedResult = this._provideAttempt(formattedResultOriginal); + const formattedResult = this.provideAttempt(formattedResultOriginal); // Test result data has to be saved before writing to db, because user may save data to custom location - await this._saveTestResultData(formattedResult); + const testResultWithImagePaths = await this._saveTestResultData(formattedResult); // To prevent skips duplication on reporter startup - const isPreviouslySkippedTest = formattedResult.status === SKIPPED && getTestFromDb(this._dbClient, formattedResult); + const isPreviouslySkippedTest = testResultWithImagePaths.status === SKIPPED && getTestFromDb(this._dbClient, formattedResult); - if (!ignoredStatuses.includes(formattedResult.status) && !isPreviouslySkippedTest) { - this._dbClient.write(formattedResult); + if (!ignoredStatuses.includes(testResultWithImagePaths.status) && !isPreviouslySkippedTest) { + this._dbClient.write(testResultWithImagePaths); } - return formattedResult; + return testResultWithImagePaths; } protected _deleteTestResultFromDb(...args: Parameters): void { diff --git a/lib/reporter-helpers.ts b/lib/reporter-helpers.ts index f5d4ec2a2..4499e9491 100644 --- a/lib/reporter-helpers.ts +++ b/lib/reporter-helpers.ts @@ -1,39 +1,65 @@ -import * as path from 'path'; -import * as tmp from 'tmp'; -import {getShortMD5} from './common-utils'; +import path from 'path'; +import tmp from 'tmp'; +import _ from 'lodash'; +import {getShortMD5, isImageInfoWithState} from './common-utils'; import * as utils from './server-utils'; -import {ImageHandler} from './image-handler'; import {ReporterTestResult} from './test-adapter'; +import {getImagesInfoByStateName} from './server-utils'; +import {copyAndUpdate} from './test-adapter/utils'; +import {ImageInfoFull, ImageInfoUpdated} from './types'; +import {UPDATED} from './constants'; const mkReferenceHash = (testId: string, stateName: string): string => getShortMD5(`${testId}#${stateName}`); -export const updateReferenceImage = async (testResult: ReporterTestResult, reportPath: string, stateName: string): Promise => { - const currImg = ImageHandler.getCurrImg(testResult.assertViewResults, stateName); +type OnReferenceUpdateCb = (testResult: ReporterTestResult, images: ImageInfoUpdated, state: string) => void; - const src = currImg?.path - ? path.resolve(reportPath, currImg.path) - : utils.getCurrentAbsolutePath(testResult, reportPath, stateName); +export const updateReferenceImages = async (testResult: ReporterTestResult, reportPath: string, onReferenceUpdateCb: OnReferenceUpdateCb): Promise => { + const newImagesInfo: ImageInfoFull[] = await Promise.all(testResult.imagesInfo.map(async (imageInfo) => { + const newImageInfo = _.clone(imageInfo); - // TODO: get rid of type assertion - const referencePath = ImageHandler.getRefImg(testResult.assertViewResults, stateName)?.path as string; + if (!isImageInfoWithState(newImageInfo) || newImageInfo.status !== UPDATED) { + return newImageInfo; + } - if (utils.fileExists(referencePath)) { - const referenceId = mkReferenceHash(testResult.id, stateName); - const oldReferencePath = path.resolve(tmp.tmpdir, referenceId); - await utils.copyFileAsync(referencePath, oldReferencePath); - } + const {stateName} = newImageInfo; + + const {actualImg} = newImageInfo; + const src = actualImg?.path + ? path.resolve(reportPath, actualImg.path) + : utils.getCurrentAbsolutePath(testResult, reportPath, stateName); + + // TODO: get rid of type assertion + const referencePath = newImageInfo?.refImg?.path as string; + + if (utils.fileExists(referencePath)) { + const referenceId = mkReferenceHash(testResult.id, stateName); + const oldReferencePath = path.resolve(tmp.tmpdir, referenceId); + await utils.copyFileAsync(referencePath, oldReferencePath); + } + + const reportReferencePath = utils.getReferencePath(testResult, stateName); + + await Promise.all([ + utils.copyFileAsync(src, referencePath), + utils.copyFileAsync(src, path.resolve(reportPath, reportReferencePath)) + ]); + + const {expectedImg} = newImageInfo; + expectedImg.path = reportReferencePath; + + onReferenceUpdateCb(testResult, newImageInfo, stateName); + + return newImageInfo; + })); - return Promise.all([ - utils.copyFileAsync(src, referencePath), - utils.copyFileAsync(src, utils.getReferenceAbsolutePath(testResult, reportPath, stateName)) - ]); + return copyAndUpdate(testResult, {imagesInfo: newImagesInfo}); }; export const revertReferenceImage = async (removedResult: ReporterTestResult, newResult: ReporterTestResult, stateName: string): Promise => { const referenceId = removedResult.id; const referenceHash = mkReferenceHash(referenceId, stateName); const oldReferencePath = path.resolve(tmp.tmpdir, referenceHash); - const referencePath = ImageHandler.getRefImg(newResult.assertViewResults, stateName)?.path; + const referencePath = getImagesInfoByStateName(newResult.imagesInfo, stateName)?.refImg?.path; if (!referencePath) { return; @@ -43,7 +69,7 @@ export const revertReferenceImage = async (removedResult: ReporterTestResult, ne }; export const removeReferenceImage = async (testResult: ReporterTestResult, stateName: string): Promise => { - const imagePath = ImageHandler.getRefImg(testResult.assertViewResults, stateName)?.path; + const imagePath = getImagesInfoByStateName(testResult.imagesInfo, stateName)?.refImg?.path; if (!imagePath) { return; diff --git a/lib/server-utils.ts b/lib/server-utils.ts index 714e670ae..d2930090f 100644 --- a/lib/server-utils.ts +++ b/lib/server-utils.ts @@ -3,49 +3,58 @@ import url from 'url'; import chalk from 'chalk'; import _ from 'lodash'; import fs from 'fs-extra'; -import {logger} from './common-utils'; +import {getShortMD5, logger, mkTestId} from './common-utils'; import {UPDATED, RUNNING, IDLE, SKIPPED, IMAGES_PATH, TestStatus} from './constants'; import type {HtmlReporter} from './plugin-api'; import type {ReporterTestResult} from './test-adapter'; -import {CustomGuiItem, HermioneTestResult, ReporterConfig} from './types'; +import { + CustomGuiItem, + HermioneTestResult, + ImageInfoWithState, + ReporterConfig, + TestSpecByPath +} from './types'; import type Hermione from 'hermione'; import crypto from 'crypto'; -import {ImagesInfoFormatter} from './image-handler'; import {HermioneTestAdapter} from './test-adapter'; import {Router} from 'express'; +import tmp from 'tmp'; const DATA_FILE_NAME = 'data.js'; interface GetPathOptions { - stateName?: string; imageDir: string; attempt: number; browserId: string; } -export const getReferencePath = (options: GetPathOptions): string => createPath({kind: 'ref', ...options}); -export const getCurrentPath = (options: GetPathOptions): string => createPath({kind: 'current', ...options}); -export const getDiffPath = (options: GetPathOptions): string => createPath({kind: 'diff', ...options}); +export const getReferencePath = (options: GetPathOptions, stateName?: string): string => + createPath({kind: 'ref', stateName, ..._.pick(options, ['attempt', 'browserId', 'imageDir'])}); +export const getCurrentPath = (options: GetPathOptions, stateName?: string): string => + createPath({kind: 'current', stateName, ..._.pick(options, ['attempt', 'browserId', 'imageDir'])}); +export const getDiffPath = (options: GetPathOptions, stateName?: string): string => + createPath({kind: 'diff', stateName, ..._.pick(options, ['attempt', 'browserId', 'imageDir'])}); export const getReferenceAbsolutePath = (testResult: ReporterTestResult, reportDir: string, stateName: string): string => { - const referenceImagePath = getReferencePath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId, stateName}); + const referenceImagePath = getReferencePath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId}, stateName); return path.resolve(reportDir, referenceImagePath); }; export const getCurrentAbsolutePath = (testResult: ReporterTestResult, reportDir: string, stateName: string): string => { - const currentImagePath = getCurrentPath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId, stateName}); + const currentImagePath = getCurrentPath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId}, stateName); return path.resolve(reportDir, currentImagePath); }; export const getDiffAbsolutePath = (testResult: ReporterTestResult, reportDir: string, stateName: string): string => { - const diffImagePath = getDiffPath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId, stateName}); + const diffImagePath = getDiffPath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId}, stateName); return path.resolve(reportDir, diffImagePath); }; interface CreatePathOptions extends GetPathOptions { + stateName?: string; kind: string; } @@ -57,6 +66,8 @@ export function createPath({attempt: attemptInput, imageDir: imageDirInput, brow return path.join(...components); } +export const getTempPath = (destPath: string): string => path.resolve(tmp.tmpdir, destPath); + export function createHash(buffer: Buffer): string { return crypto .createHash('sha1') @@ -304,10 +315,9 @@ export function mapPlugins(plugins: ReporterConfig['plugins'], callback: (nam export const formatTestResult = ( rawResult: HermioneTestResult, status: TestStatus, - attempt: number, - {imageHandler}: {imageHandler: ImagesInfoFormatter} + attempt: number ): ReporterTestResult => { - return new HermioneTestAdapter(rawResult, {attempt, status, imagesInfoFormatter: imageHandler}); + return new HermioneTestAdapter(rawResult, {attempt, status}); }; export const saveErrorDetails = async (testResult: ReporterTestResult, reportPath: string): Promise => { @@ -323,3 +333,14 @@ export const saveErrorDetails = async (testResult: ReporterTestResult, reportPat await makeDirFor(detailsFilePath); await fs.writeFile(detailsFilePath, detailsData); }; + +export const getExpectedCacheKey = ([testResult, stateName]: [TestSpecByPath, string | undefined]): string => { + const shortTestId = getShortMD5(mkTestId(testResult.testPath.join(' '), testResult.browserId)); + + return shortTestId + '#' + stateName; +}; + +export const getImagesInfoByStateName = (imagesInfo: ReporterTestResult['imagesInfo'], stateName: string): ImageInfoWithState | undefined => { + return imagesInfo.find( + imagesInfo => (imagesInfo as ImageInfoWithState).stateName === stateName) as ImageInfoWithState | undefined; +}; diff --git a/lib/static/components/section/body/page-screenshot.tsx b/lib/static/components/section/body/page-screenshot.tsx index f2e87424b..fafc15537 100644 --- a/lib/static/components/section/body/page-screenshot.tsx +++ b/lib/static/components/section/body/page-screenshot.tsx @@ -1,10 +1,10 @@ import React, {Component} from 'react'; import Details from '../../details'; import ResizedScreenshot from '../../state/screenshot/resized'; -import {ImageData} from '../../../../types'; +import {ImageFile} from '../../../../types'; interface PageScreenshotProps { - image: ImageData; + image: ImageFile; } export class PageScreenshot extends Component { diff --git a/lib/test-adapter/playwright.ts b/lib/test-adapter/playwright.ts index be38db2f6..7074e8cd6 100644 --- a/lib/test-adapter/playwright.ts +++ b/lib/test-adapter/playwright.ts @@ -6,11 +6,18 @@ import stripAnsi from 'strip-ansi'; import {ReporterTestResult} from './index'; import {testsAttempts} from './cache/playwright'; -import {getShortMD5, isImageDiffError, isNoRefImageError, mkTestId} from '../common-utils'; -import {FAIL, PWT_TITLE_DELIMITER, TestStatus} from '../constants'; +import {getError, getShortMD5, isImageDiffError, isNoRefImageError, mkTestId} from '../common-utils'; +import {ERROR, FAIL, PWT_TITLE_DELIMITER, SUCCESS, TestStatus} from '../constants'; import {ErrorName} from '../errors'; -import {ImagesInfoFormatter} from '../image-handler'; -import {AssertViewResult, ErrorDetails, ImageData, ImageInfoFull, ImageSize, TestError} from '../types'; +import { + DiffOptions, + ErrorDetails, + ImageFile, + ImageInfoDiff, + ImageInfoFull, ImageInfoNoRef, ImageInfoPageError, ImageInfoPageSuccess, ImageInfoSuccess, + ImageSize, + TestError +} from '../types'; import * as utils from '../server-utils'; import type {CoordBounds} from 'looks-same'; @@ -39,6 +46,10 @@ export enum ImageTitleEnding { const ANY_IMAGE_ENDING_REGEXP = new RegExp(Object.values(ImageTitleEnding).map(ending => `${ending}$`).join('|')); +const DEFAULT_DIFF_OPTIONS = { + diffColor: '#ff00ff' +} satisfies Partial; + export const getStatus = (result: PlaywrightTestResult): TestStatus => { if (result.status === PwtTestStatus.PASSED) { return TestStatus.SUCCESS; @@ -108,7 +119,7 @@ const extractImageError = (result: PlaywrightTestResult, {state, expectedAttachm } : null; }; -const getImageData = (attachment: PlaywrightAttachment | undefined): ImageData | null => { +const getImageData = (attachment: PlaywrightAttachment | undefined): ImageFile | null => { if (!attachment) { return null; } @@ -119,20 +130,14 @@ const getImageData = (attachment: PlaywrightAttachment | undefined): ImageData | }; }; -export interface PlaywrightTestAdapterOptions { - imagesInfoFormatter: ImagesInfoFormatter; -} - export class PlaywrightTestAdapter implements ReporterTestResult { private readonly _testCase: PlaywrightTestCase; private readonly _testResult: PlaywrightTestResult; private _attempt: number; - private _imagesInfoFormatter: ImagesInfoFormatter; - constructor(testCase: PlaywrightTestCase, testResult: PlaywrightTestResult, {imagesInfoFormatter}: PlaywrightTestAdapterOptions) { + constructor(testCase: PlaywrightTestCase, testResult: PlaywrightTestResult) { this._testCase = testCase; this._testResult = testResult; - this._imagesInfoFormatter = imagesInfoFormatter; const testId = mkTestId(this.fullName, this.browserId); if (utils.shouldUpdateAttempt(this.status)) { @@ -142,44 +147,6 @@ export class PlaywrightTestAdapter implements ReporterTestResult { this._attempt = testsAttempts.get(testId) || 0; } - get assertViewResults(): AssertViewResult[] { - return Object.entries(this._attachmentsByState).map(([state, attachments]): AssertViewResult | null => { - const expectedAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Expected)); - const diffAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Diff)); - const actualAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Actual)); - - const [refImg, diffImg, currImg] = [expectedAttachment, diffAttachment, actualAttachment].map(getImageData); - - const error = extractImageError(this._testResult, {state, expectedAttachment, diffAttachment, actualAttachment}) || this.error; - - if (error?.name === ErrorName.IMAGE_DIFF && refImg && diffImg && currImg) { - return { - name: ErrorName.IMAGE_DIFF, - stateName: state, - refImg, - diffImg, - currImg, - diffClusters: _.get(error, 'diffClusters', []) - }; - } else if (error?.name === ErrorName.NO_REF_IMAGE && currImg) { - return { - name: ErrorName.NO_REF_IMAGE, - message: error.message, - stack: error.stack, - stateName: state, - currImg - }; - } else if (!error && refImg) { - return { - stateName: state, - refImg - }; - } - - return null; - }).filter(Boolean) as AssertViewResult[]; - } - get attempt(): number { return this._attempt; } @@ -237,8 +204,57 @@ export class PlaywrightTestAdapter implements ReporterTestResult { return getShortMD5(this.fullName); } - get imagesInfo(): ImageInfoFull[] | undefined { - return this._imagesInfoFormatter.getImagesInfo(this); + get imagesInfo(): ImageInfoFull[] { + const imagesInfo = Object.entries(this._attachmentsByState).map(([state, attachments]): ImageInfoFull | null => { + const expectedAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Expected)); + const diffAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Diff)); + const actualAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Actual)); + + const [refImg, diffImg, actualImg] = [expectedAttachment, diffAttachment, actualAttachment].map(getImageData); + + const error = extractImageError(this._testResult, {state, expectedAttachment, diffAttachment, actualAttachment}) || this.error; + + if (error?.name === ErrorName.IMAGE_DIFF && refImg && diffImg && actualImg) { + return { + status: FAIL, + stateName: state, + refImg, + diffImg, + actualImg, + expectedImg: refImg, + diffClusters: _.get(error, 'diffClusters', []), + // TODO: extract diffOptions from config + diffOptions: {current: actualImg.path, reference: refImg.path, ...DEFAULT_DIFF_OPTIONS} + } satisfies ImageInfoDiff; + } else if (error?.name === ErrorName.NO_REF_IMAGE && refImg && actualImg) { + return { + status: ERROR, + stateName: state, + error: _.pick(error, ['message', 'name', 'stack']), + refImg, + actualImg + } satisfies ImageInfoNoRef; + } else if (!error && refImg) { + return { + status: SUCCESS, + stateName: state, + refImg, + expectedImg: refImg, + ...(actualImg ? {actualImg} : {}) + } satisfies ImageInfoSuccess; + } + + return null; + }).filter((value): value is ImageInfoFull => value !== null); + + if (this.screenshot) { + imagesInfo.push({ + status: _.isEmpty(getError(this.error)) ? SUCCESS : ERROR, + actualImg: this.screenshot + } satisfies ImageInfoPageSuccess | ImageInfoPageError as ImageInfoPageSuccess | ImageInfoPageError); + } + + return imagesInfo; } get meta(): Record { @@ -249,7 +265,7 @@ export class PlaywrightTestAdapter implements ReporterTestResult { return true; } - get screenshot(): ImageData | null { + get screenshot(): ImageFile | null { const pageScreenshot = this._testResult.attachments.find(a => a.contentType === 'image/png' && a.name === 'screenshot'); return getImageData(pageScreenshot); diff --git a/lib/test-adapter/reporter.ts b/lib/test-adapter/reporter.ts index 22a36bb22..bcd0ac593 100644 --- a/lib/test-adapter/reporter.ts +++ b/lib/test-adapter/reporter.ts @@ -1,9 +1,9 @@ import {TestStatus} from '../constants'; -import {AssertViewResult, TestError, ErrorDetails, ImageInfoFull, ImageBase64, ImageData} from '../types'; +import {TestError, ErrorDetails, ImageInfoFull, ImageBase64, ImageFile} from '../types'; import {ReporterTestResult} from './index'; import _ from 'lodash'; -import {extractErrorDetails, getTestHash} from './utils'; -import {getShortMD5} from '../common-utils'; +import {extractErrorDetails} from './utils'; +import {getShortMD5, getTestHash} from '../common-utils'; // This class is primarily useful when cloning ReporterTestResult. // It allows to override some properties while keeping computable @@ -18,10 +18,6 @@ export class ReporterTestAdapter implements ReporterTestResult { this._errorDetails = null; } - get assertViewResults(): AssertViewResult[] { - return this._testResult.assertViewResults; - } - get attempt(): number { return this._testResult.attempt; } @@ -68,7 +64,7 @@ export class ReporterTestAdapter implements ReporterTestResult { return getShortMD5(this.fullName); } - get imagesInfo(): ImageInfoFull[] | undefined { + get imagesInfo(): ImageInfoFull[] { return this._testResult.imagesInfo; } @@ -80,7 +76,7 @@ export class ReporterTestAdapter implements ReporterTestResult { return this._testResult.multipleTabs; } - get screenshot(): ImageBase64 | ImageData | null | undefined { + get screenshot(): ImageBase64 | ImageFile | null | undefined { return this.error?.screenshot; } diff --git a/lib/test-adapter/sqlite.ts b/lib/test-adapter/sqlite.ts index 5093d5ccd..9c156d642 100644 --- a/lib/test-adapter/sqlite.ts +++ b/lib/test-adapter/sqlite.ts @@ -6,12 +6,12 @@ import { ErrorDetails, ImageInfoFull, ImageBase64, - ImageData, + ImageFile, RawSuitesRow } from '../types'; import {ReporterTestResult} from './index'; import {Writable} from 'type-fest'; -import {getTestHash} from './utils'; +import {getTestHash} from '../common-utils'; const tryParseJson = (json: string): unknown | undefined => { try { @@ -100,7 +100,7 @@ export class SqliteTestAdapter implements ReporterTestResult { return getTestHash(this); } - get imagesInfo(): ImageInfoFull[] | undefined { + get imagesInfo(): ImageInfoFull[] { if (!_.has(this._parsedTestResult, 'imagesInfo')) { this._parsedTestResult.imagesInfo = tryParseJson(this._testResult[DB_COLUMN_INDEXES.imagesInfo]) as ImageInfoFull[]; } @@ -120,7 +120,7 @@ export class SqliteTestAdapter implements ReporterTestResult { return Boolean(this._testResult[DB_COLUMN_INDEXES.multipleTabs]); } - get screenshot(): ImageBase64 | ImageData | null | undefined { + get screenshot(): ImageBase64 | ImageFile | null | undefined { return this.error?.screenshot; } diff --git a/lib/test-adapter/utils/index.ts b/lib/test-adapter/utils/index.ts index 09a1e1d79..2ccd1476d 100644 --- a/lib/test-adapter/utils/index.ts +++ b/lib/test-adapter/utils/index.ts @@ -1,17 +1,16 @@ import _ from 'lodash'; import {ReporterTestResult} from '../index'; import {TupleToUnion} from 'type-fest'; -import {ErrorDetails} from '../../types'; +import {ErrorDetails, ImageInfoDiff, ImageInfoFull} from '../../types'; import {ERROR_DETAILS_PATH} from '../../constants'; import {ReporterTestAdapter} from '../reporter'; -import {getDetailsFileName} from '../../common-utils'; +import {getDetailsFileName, isImageBufferData} from '../../common-utils'; export const copyAndUpdate = ( original: ReporterTestResult, updates: Partial ): ReporterTestResult => { const keys = [ - 'assertViewResults', 'attempt', 'browserId', 'description', @@ -64,6 +63,13 @@ export const extractErrorDetails = (testResult: ReporterTestResult): ErrorDetail return null; }; -export const getTestHash = (testResult: ReporterTestResult): string => { - return testResult.testPath.concat(testResult.browserId, testResult.attempt.toString()).join(' '); +export const removeBufferFromImagesInfo = (imagesInfo: ImageInfoFull): ImageInfoFull => { + const {diffImg} = imagesInfo as ImageInfoDiff; + const newImagesInfo = _.clone(imagesInfo); + + if (isImageBufferData(diffImg)) { + (newImagesInfo as ImageInfoDiff).diffImg = {...diffImg, buffer: Buffer.from('')}; + } + + return newImagesInfo; }; diff --git a/lib/tests-tree-builder/gui.ts b/lib/tests-tree-builder/gui.ts index 3336a8fbd..b94f7f300 100644 --- a/lib/tests-tree-builder/gui.ts +++ b/lib/tests-tree-builder/gui.ts @@ -2,7 +2,7 @@ import _ from 'lodash'; import {BaseTestsTreeBuilder, Tree, TreeImage, TreeTestResult, TreeSuite} from './base'; import {TestStatus, UPDATED} from '../constants'; import {isUpdatedStatus} from '../common-utils'; -import {ImageInfoFail, ImageInfoWithState} from '../types'; +import {ImageFile, ImageInfoWithState} from '../types'; interface SuiteBranch { id: string; @@ -22,8 +22,8 @@ export interface TestRefUpdateData { state: {name: string}; metaInfo: TreeTestResult['metaInfo']; imagesInfo: { - stateName: ImageInfoWithState['stateName']; - actualImg: ImageInfoWithState['actualImg']; + stateName: string; + actualImg: ImageFile; status: TestStatus; }[]; attempt: number; @@ -81,10 +81,10 @@ export class GuiTestsTreeBuilder extends BaseTestsTreeBuilder { const suite = this._tree.suites.byId[browser.parentId]; const imagesInfo = imagesByResultId[resultId] - .filter(treeImage => (treeImage as ImageInfoFail).stateName) - .map((treeImage) => ({ - stateName: (treeImage as ImageInfoWithState).stateName, - actualImg: treeImage.actualImg, + .filter(treeImage => (treeImage as ImageInfoWithState).stateName) + .map((treeImage) => ({ + stateName: (treeImage as ImageInfoWithState).stateName as string, + actualImg: treeImage.actualImg as ImageFile, status: UPDATED })); diff --git a/lib/types.ts b/lib/types.ts index d129f0ec9..5097baca0 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -15,7 +15,7 @@ export interface HermioneTestResult extends HermioneTestResultOriginal { timestamp?: number; } -export interface ImagesSaver { +export interface ImageFileSaver { saveImg: (localFilePath: string, options: {destPath: string; reportDir: string}) => string | Promise; } @@ -34,11 +34,15 @@ export interface ImageSize { height: number; } -export interface ImageData { +export interface ImageFile { path: string; size: ImageSize; } +export interface ImageBuffer { + buffer: Buffer; +} + export interface ImageBase64 { base64: string; size: ImageSize @@ -50,63 +54,81 @@ export interface DiffOptions extends LooksSameOptions { diffColor: string; } -export interface ImageInfoFail { +export interface TestError { + name: string; + message: string; + stack?: string; + stateName?: string; + details?: ErrorDetails + screenshot?: ImageBase64 | ImageFile +} + +export interface ImageInfoDiff { status: TestStatus.FAIL; stateName: string; - refImg?: ImageData; + refImg: ImageFile; diffClusters?: CoordBounds[]; - expectedImg: ImageData; - actualImg: ImageData; - diffImg: ImageData; + expectedImg: ImageFile; + actualImg: ImageFile; + diffImg?: ImageFile | ImageBuffer; + diffOptions: DiffOptions; } interface AssertViewSuccess { stateName: string; - refImg: ImageData; + refImg: ImageFile; } export interface ImageInfoSuccess { - status: TestStatus.SUCCESS | TestStatus.UPDATED; + status: TestStatus.SUCCESS; stateName: string; - refImg?: ImageData; + refImg?: ImageFile; diffClusters?: CoordBounds[]; - expectedImg: ImageData; - actualImg?: ImageData; + expectedImg: ImageFile; + actualImg?: ImageFile; } export interface ImageInfoPageSuccess { status: TestStatus.SUCCESS; - actualImg: ImageData; + actualImg: ImageFile | ImageBase64; } -export interface ImageInfoError { +export interface ImageInfoPageError { status: TestStatus.ERROR; - error?: {message: string; stack: string;} - stateName?: string; - refImg?: ImageData; - diffClusters?: CoordBounds[]; - actualImg: ImageData; + actualImg: ImageFile | ImageBase64; } -export type ImageInfoWithState = ImageInfoFail | ImageInfoSuccess | ImageInfoError; +export interface ImageInfoNoRef { + status: TestStatus.ERROR; + error?: TestError; + stateName: string; + refImg: ImageFile; + actualImg: ImageFile; +} + +export interface ImageInfoUpdated { + status: TestStatus.UPDATED; + stateName: string; + refImg: ImageFile; + actualImg: ImageFile; + expectedImg: ImageFile; +} + +export type ImageInfoWithState = ImageInfoDiff | ImageInfoSuccess | ImageInfoNoRef | ImageInfoUpdated; -export type ImageInfoFull = ImageInfoFail | ImageInfoSuccess | ImageInfoError | ImageInfoPageSuccess; +export type ImageInfoFull = ImageInfoWithState | ImageInfoPageSuccess | ImageInfoPageError; export type ImageInfo = - | Omit + | Omit | Omit - | Omit + | Omit | Omit; export type AssertViewResult = (AssertViewSuccess | ImageDiffError | NoRefImageError) & {isUpdated?: boolean}; -export interface TestError { - name: string; - message: string; - stack?: string; - stateName?: string; - details?: ErrorDetails - screenshot?: ImageBase64 | ImageData +export interface TestSpecByPath { + testPath: string[]; + browserId: string; } export interface HtmlReporterApi { diff --git a/lib/workers/worker.ts b/lib/workers/worker.ts index 5476c4bb4..3a27c1ff2 100644 --- a/lib/workers/worker.ts +++ b/lib/workers/worker.ts @@ -1,8 +1,8 @@ import looksSame from 'looks-same'; -import type {ImageDiffError} from '../errors'; +import {DiffOptions} from '../types'; -export function saveDiffTo(imageDiffError: ImageDiffError, diffPath: string): Promise { - const {diffColor: highlightColor, ...otherOpts} = imageDiffError.diffOpts; +export function saveDiffTo(diffOpts: DiffOptions, diffPath: string): Promise { + const {diffColor: highlightColor, ...otherOpts} = diffOpts; return looksSame.createDiff({diff: diffPath, highlightColor, ...otherOpts}); } diff --git a/playwright.ts b/playwright.ts index c0056a248..93454caa7 100644 --- a/playwright.ts +++ b/playwright.ts @@ -8,12 +8,16 @@ import type {Reporter, TestCase, TestResult as PwtTestResult} from '@playwright/ import {StaticReportBuilder} from './lib/report-builder/static'; import {HtmlReporter} from './lib/plugin-api'; -import {ReporterConfig} from './lib/types'; +import {ReporterConfig, TestSpecByPath} from './lib/types'; import {parseConfig} from './lib/config'; import {PluginEvents, ToolName} from './lib/constants'; import {RegisterWorkers} from './lib/workers/create-workers'; import {PlaywrightTestAdapter} from './lib/test-adapter/playwright'; import {SqliteClient} from './lib/sqlite-client'; +import {SqliteImageStore} from './lib/image-store'; +import {ImagesInfoSaver} from './lib/images-info-saver'; +import {Cache} from './lib/cache'; +import {getExpectedCacheKey} from './lib/server-utils'; export {ReporterConfig} from './lib/types'; @@ -39,8 +43,17 @@ class MyReporter implements Reporter { this._initPromise = (async (htmlReporter: HtmlReporter, config: ReporterConfig): Promise => { const dbClient = await SqliteClient.create({htmlReporter, reportPath: config.path}); + const imageStore = new SqliteImageStore(dbClient); + const expectedPathsCache = new Cache<[TestSpecByPath, string | undefined], string>(getExpectedCacheKey); - this._staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient}); + const imagesInfoSaver = new ImagesInfoSaver({ + imageFileSaver: htmlReporter.imagesSaver, + expectedPathsCache, + imageStore, + reportPath: htmlReporter.config.path + }); + + this._staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient, imagesInfoSaver}); this._staticReportBuilder.registerWorkers(workers); await this._staticReportBuilder.saveStaticFiles(); @@ -55,7 +68,7 @@ class MyReporter implements Reporter { const staticReportBuilder = this._staticReportBuilder as StaticReportBuilder; - const formattedResult = new PlaywrightTestAdapter(test, result, {imagesInfoFormatter: staticReportBuilder.imageHandler}); + const formattedResult = new PlaywrightTestAdapter(test, result); await staticReportBuilder.addTestResult(formattedResult); }); diff --git a/test/unit/hermione.js b/test/unit/hermione.js index 6dd21fdbf..f418baf25 100644 --- a/test/unit/hermione.js +++ b/test/unit/hermione.js @@ -33,9 +33,8 @@ describe('lib/hermione', () => { 'better-sqlite3': sinon.stub().returns(mkSqliteDb()) }); - const {ImageHandler} = proxyquire('lib/image-handler', { + const {ImagesInfoSaver} = proxyquire('lib/images-info-saver', { 'fs-extra': fs, - './image-cache': {cacheExpectedPaths, cacheAllImages, cacheDiffImages}, './server-utils': utils }); @@ -48,11 +47,11 @@ describe('lib/hermione', () => { 'fs-extra': fs, '../server-utils': utils, '../test-adapter': {TestAdapter}, - '../image-handler': {ImageHandler} + '../images-info-saver': {ImagesInfoSaver} }); const HtmlReporter = proxyquire('lib/plugin-api', { - './local-images-saver': proxyquire('lib/local-images-saver', { + './local-image-file-saver': proxyquire('lib/local-image-file-saver', { './server-utils': utils }) }).HtmlReporter; diff --git a/test/unit/lib/image-handler.ts b/test/unit/lib/image-handler.ts deleted file mode 100644 index 20f45687b..000000000 --- a/test/unit/lib/image-handler.ts +++ /dev/null @@ -1,441 +0,0 @@ -import * as fsOriginal from 'fs-extra'; -import _ from 'lodash'; -import proxyquire from 'proxyquire'; -import sinon, {SinonStubbedInstance} from 'sinon'; -import type tmpOriginal from 'tmp'; - -import type * as originalUtils from 'lib/server-utils'; -import {logger} from 'lib/common-utils'; -import {ImageHandler as ImageHandlerOriginal} from 'lib/image-handler'; -import {RegisterWorkers} from 'lib/workers/create-workers'; -import {AssertViewResult, ImageInfoFail, ImageInfoFull, ImageInfoSuccess, ImagesSaver} from 'lib/types'; -import {ErrorName, ImageDiffError} from 'lib/errors'; -import {ImageStore} from 'lib/image-store'; -import {FAIL, PluginEvents, SUCCESS, UPDATED} from 'lib/constants'; -import {ReporterTestResult} from 'lib/test-adapter'; - -describe('image-handler', function() { - const sandbox = sinon.sandbox.create(); - let fs: sinon.SinonStubbedInstance; - let utils: sinon.SinonStubbedInstance; - let tmp: typeof tmpOriginal; - let err: AssertViewResult; - let ImageHandler: typeof ImageHandlerOriginal; - const cacheExpectedPaths = new Map(), - cacheAllImages = new Map(), - cacheDiffImages = new Map(); - - class ImageDiffErrorStub extends Error { - name = ErrorName.IMAGE_DIFF; - } - class NoRefImageErrorStub extends Error { - name = ErrorName.NO_REF_IMAGE; - } - - const mkImageStore = (): SinonStubbedInstance => ({getLastImageInfoFromDb: sinon.stub()} as SinonStubbedInstance); - - const mkImagesSaver = (): SinonStubbedInstance => ({saveImg: sinon.stub()} as SinonStubbedInstance); - - const mkTestResult = (result: Partial): ReporterTestResult => _.defaults(result, { - id: 'some-id', - attempt: 0, - fullName: 'default-title' - }) as ReporterTestResult; - - const mkErrStub = (ErrType: typeof ImageDiffErrorStub | typeof NoRefImageErrorStub = ImageDiffErrorStub, {stateName, currImg, refImg, diffBuffer}: Partial = {}): AssertViewResult => { - const err: AssertViewResult = new ErrType() as any; - - err.stateName = stateName || 'plain'; - (err as ImageDiffError).currImg = currImg || {path: 'curr/path'} as any; - err.refImg = refImg || {path: 'ref/path'} as any; - (err as ImageDiffError).diffBuffer = diffBuffer; - - return err; - }; - - const mkWorker = (): sinon.SinonStubbedInstance> => { - return {saveDiffTo: sandbox.stub()} as any; - }; - - beforeEach(() => { - fs = sinon.stub(_.clone(fsOriginal)); - err = mkErrStub(); - tmp = {tmpdir: 'default/dir'} as any; - - const originalUtils = proxyquire('lib/server-utils', { - 'fs-extra': fs - }); - utils = _.clone(originalUtils); - - ImageHandler = proxyquire('lib/image-handler', { - tmp, - 'fs-extra': fs, - './server-utils': utils, - './image-cache': {cacheExpectedPaths, cacheAllImages, cacheDiffImages} - }).ImageHandler; - - sandbox.stub(utils, 'getCurrentPath').returns(''); - sandbox.stub(utils, 'getDiffPath').returns(''); - sandbox.stub(utils, 'getReferencePath').returns(''); - - fs.readFile.resolves(Buffer.from('')); - fs.writeFile.resolves(); - fs.copy.resolves(); - }); - - afterEach(() => { - sandbox.restore(); - - cacheExpectedPaths.clear(); - cacheAllImages.clear(); - cacheDiffImages.clear(); - }); - - describe('saveTestImages', () => { - it('should build diff to tmp dir', async () => { - (tmp as any).tmpdir = 'tmp/dir'; - const testResult = mkTestResult({ - assertViewResults: [err] - }); - utils.getDiffPath.returns('diff/report/path'); - - const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: 'some-dir'}); - const worker = mkWorker(); - await imageHandler.saveTestImages(testResult, worker); - - assert.calledOnceWith(worker.saveDiffTo, err, sinon.match('tmp/dir/diff/report/path')); - }); - - it('should save diff in report from tmp dir using external storage', async () => { - (tmp as any).tmpdir = 'tmp/dir'; - const testResult = mkTestResult({ - assertViewResults: [err] - }); - utils.getDiffPath.returns('diff/report/path'); - const imagesSaver = mkImagesSaver(); - const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'html-report/path'}); - const worker = mkWorker(); - await imageHandler.saveTestImages(testResult, worker); - - assert.calledWith( - imagesSaver.saveImg, - sinon.match('tmp/dir/diff/report/path'), - {destPath: 'diff/report/path', reportDir: 'html-report/path'} - ); - }); - - it('should emit TEST_SCREENSHOTS_SAVED event', async () => { - (tmp as any).tmpdir = 'tmp/dir'; - const testResult = mkTestResult({ - browserId: 'chrome', - assertViewResults: [err] - }); - utils.getDiffPath.returns('diff/report/path'); - - const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); - sinon.stub(imageHandler, 'getImagesInfo').returns([{test: 123}]); - const worker = mkWorker(); - - const screenshotsSavedHandler = sinon.stub(); - imageHandler.on(PluginEvents.TEST_SCREENSHOTS_SAVED, screenshotsSavedHandler); - - await imageHandler.saveTestImages(testResult, worker); - - assert.calledOnceWith(screenshotsSavedHandler, { - attempt: 0, - testId: 'default-title.chrome', - imagesInfo: [{test: 123}] - }); - }); - - describe('saving error screenshot', () => { - beforeEach(() => { - sandbox.stub(logger, 'warn'); - sandbox.stub(utils, 'makeDirFor').resolves(); - sandbox.stub(utils, 'copyFileAsync'); - }); - - describe('if screenshot on reject does not exist', () => { - it('should not save screenshot', () => { - const testResult = mkTestResult({ - error: {screenshot: {base64: null}} as any, - assertViewResults: [] - }); - const hermioneTestAdapter = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); - - return hermioneTestAdapter.saveTestImages(testResult, mkWorker()) - .then(() => assert.notCalled(fs.writeFile)); - }); - - it('should warn about it', () => { - const testResult = mkTestResult({ - screenshot: {base64: null} as any, - assertViewResults: [] - }); - const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); - - return imageHandler.saveTestImages(testResult, mkWorker()) - .then(() => assert.calledWith(logger.warn as sinon.SinonStub, 'Cannot save screenshot on reject')); - }); - }); - - it('should create directory for screenshot', () => { - const testResult = mkTestResult({ - screenshot: {base64: 'base64-data'} as any, - assertViewResults: [] - }); - utils.getCurrentPath.returns('dest/path'); - const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); - - return imageHandler.saveTestImages(testResult, mkWorker()) - .then(() => assert.calledOnceWith(utils.makeDirFor, sinon.match('dest/path'))); - }); - - it('should save screenshot from base64 format', async () => { - const testResult = mkTestResult({ - screenshot: {base64: 'base64-data'} as any, - assertViewResults: [] - }); - utils.getCurrentPath.returns('dest/path'); - const bufData = new Buffer('base64-data', 'base64'); - const imagesSaver = mkImagesSaver(); - const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'report/path'}); - - await imageHandler.saveTestImages(testResult, mkWorker()); - - assert.calledOnceWith(fs.writeFile, sinon.match('dest/path'), bufData, 'base64'); - assert.calledWith(imagesSaver.saveImg, sinon.match('dest/path'), {destPath: 'dest/path', reportDir: 'report/path'}); - }); - }); - - describe('saving reference image', () => { - it('should save reference, if it is not reused', async () => { - (tmp as any).tmpdir = 'tmp/dir'; - const testResult = mkTestResult({assertViewResults: [err]}); - utils.getReferencePath.returns('ref/report/path'); - const imagesSaver = mkImagesSaver(); - const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'html-report/path'}); - - await imageHandler.saveTestImages(testResult, mkWorker()); - - assert.calledWith( - imagesSaver.saveImg, 'ref/path', - {destPath: 'ref/report/path', reportDir: 'html-report/path'} - ); - }); - - it('should not save reference, if it is reused', async () => { - (tmp as any).tmpdir = 'tmp/dir'; - const error = mkErrStub(ImageDiffErrorStub, {stateName: 'plain'}); - const testResult = mkTestResult({assertViewResults: [error], browserId: 'browser-id'}); - utils.getReferencePath.returns('ref/report/path'); - const imagesSaver = mkImagesSaver(); - cacheExpectedPaths.set('da89771#plain', 'ref/report/path'); - const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'html-report/path'}); - - await imageHandler.saveTestImages(testResult, mkWorker()); - - assert.neverCalledWith( - imagesSaver.saveImg, 'ref/path', - {destPath: 'ref/report/path', reportDir: 'html-report/path'} - ); - }); - - it('should save png buffer, if it is passed', async () => { - const error = mkErrStub(ImageDiffErrorStub, {stateName: 'plain', diffBuffer: 'foo' as any}); - const testResult = mkTestResult({assertViewResults: [error]}); - utils.getDiffPath.returns('diff/report/path'); - - const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); - const workers = {saveDiffTo: sandbox.stub()}; - await imageHandler.saveTestImages(testResult, mkWorker()); - - assert.calledOnceWith(fs.writeFile, sinon.match('diff/report/path'), Buffer.from('foo')); - assert.notCalled(workers.saveDiffTo); - }); - }); - }); - - ([ - {field: 'refImg', method: 'getRefImg'}, - {field: 'currImg', method: 'getCurrImg'} - ] as const).forEach(({field, method}) => { - describe(`${method}`, () => { - it(`should return ${field} from test result`, () => { - const testResult = mkTestResult({assertViewResults: [ - {[field]: 'some-value', stateName: 'plain'} as any]}); - - assert.equal((ImageHandler[method])(testResult.assertViewResults, 'plain'), 'some-value' as any); - }); - }); - }); - - describe('getScreenshot', () => { - it('should return error screenshot from test result', () => { - const testResult = mkTestResult({screenshot: 'some-value'} as any); - - assert.equal(ImageHandler.getScreenshot(testResult), 'some-value' as any); - }); - }); - - describe('getImagesInfo', () => { - beforeEach(() => { - sandbox.stub(utils, 'copyFileAsync'); - utils.getReferencePath.returns('some/ref.png'); - }); - - it('should return diffClusters', () => { - const testResult = mkTestResult({ - assertViewResults: [{diffClusters: [{left: 0, top: 0, right: 1, bottom: 1}]}] as any - }); - const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); - - const [{diffClusters}] = imageHandler.getImagesInfo(testResult) as ImageInfoFail[]; - - assert.deepEqual(diffClusters, [{left: 0, top: 0, right: 1, bottom: 1}]); - }); - - it('should return saved images', async () => { - const testResult = mkTestResult({ - assertViewResults: [mkErrStub()], - status: SUCCESS - }); - - const imagesSaver = mkImagesSaver(); - imagesSaver.saveImg.withArgs( - 'ref/path', - {destPath: 'some/ref.png', reportDir: 'some/rep'} - ).returns('saved/ref.png'); - const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'some/rep'}); - const workers = mkWorker(); - - await imageHandler.saveTestImages(testResult, workers); - - const {expectedImg} = imageHandler.getImagesFor(testResult, SUCCESS, 'plain') as ImageInfoSuccess; - assert.equal(expectedImg.path, 'saved/ref.png'); - }); - - it('should return dest image path by default', async () => { - const testResult = mkTestResult({ - assertViewResults: [mkErrStub()], - status: SUCCESS - }); - - const imagesSaver = mkImagesSaver(); - const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'some/rep'}); - const workers = mkWorker(); - - await imageHandler.saveTestImages(testResult, workers); - - const {expectedImg} = imageHandler.getImagesFor(testResult, SUCCESS, 'plain') as ImageInfoSuccess; - assert.equal(expectedImg.path, 'some/ref.png'); - }); - - it('should return ref image path after update image for NoRefImageError', async () => { - const testResult = mkTestResult({ - assertViewResults: [mkErrStub(NoRefImageErrorStub)], - status: UPDATED - }); - - const imagesSaver = mkImagesSaver(); - const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'some/rep'}); - const workers = mkWorker(); - - await imageHandler.saveTestImages(testResult, workers); - - const {expectedImg} = imageHandler.getImagesFor(testResult, UPDATED, 'plain') as ImageInfoSuccess; - assert.equal(expectedImg.path, 'some/ref.png'); - }); - - describe('expected path', () => { - const mkLastImageInfo_ = (opts = {}): ImageInfoFull => { - const {stateName, expectedImgPath} = _.defaults(opts, { - stateName: 'plain', - expectedImgPath: 'default/expected/img/path.png' - }); - - return { - stateName, - expectedImg: { - path: expectedImgPath - } - } as any; - }; - - it('should be pulled from the store if exists', async () => { - const testResult = mkTestResult({ - fullName: 'some-title', - assertViewResults: [mkErrStub()] - }); - const imageStore = mkImageStore(); - imageStore.getLastImageInfoFromDb.withArgs(testResult, 'plain').returns(mkLastImageInfo_()); - - const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); - - imageHandler.getImagesFor(testResult, FAIL, 'plain'); - - assert.notCalled(utils.getReferencePath); - }); - - it('should be generated if does not exist in store', async () => { - const testResult = mkTestResult({ - fullName: 'some-title', - assertViewResults: [mkErrStub()] - }); - const imageStore = mkImageStore(); - imageStore.getLastImageInfoFromDb.withArgs(testResult, 'plain').returns(undefined); - - const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); - - imageHandler.getImagesFor(testResult, FAIL, 'plain'); - - assert.calledOnce(utils.getReferencePath); - }); - - it('should be generated on update', async () => { - const testResult = mkTestResult({ - assertViewResults: [mkErrStub()], - fullName: 'some-title', - status: UPDATED - }); - const imageStore = mkImageStore(); - imageStore.getLastImageInfoFromDb.withArgs(testResult, 'plain').returns(mkLastImageInfo_()); - const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); - - imageHandler.getImagesFor(testResult, UPDATED, 'plain'); - - assert.calledOnce(utils.getReferencePath); - }); - - it('should be queried from the database for each browser', async () => { - const chromeTestResult = mkTestResult({browserId: 'chrome'}); - const firefoxTestResult = mkTestResult({browserId: 'firefox'}); - - const imageStore = mkImageStore(); - const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); - - imageHandler.getImagesFor(chromeTestResult, FAIL, 'plain'); - imageHandler.getImagesFor(firefoxTestResult, FAIL, 'plain'); - - assert.calledTwice(imageStore.getLastImageInfoFromDb); - assert.calledWith(imageStore.getLastImageInfoFromDb.firstCall, chromeTestResult, 'plain'); - assert.calledWith(imageStore.getLastImageInfoFromDb.secondCall, firefoxTestResult, 'plain'); - }); - - it('should be queried from the database once per state', async () => { - const testResult = mkTestResult({ - fullName: 'some-title', - assertViewResults: [mkErrStub()] - }); - const imageStore = mkImageStore(); - imageStore.getLastImageInfoFromDb.returns(mkLastImageInfo_()); - const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); - - imageHandler.getImagesFor(testResult, FAIL, 'plain'); - imageHandler.getImagesFor(testResult, FAIL, 'plain'); - - assert.calledOnce(imageStore.getLastImageInfoFromDb); - }); - }); - }); -}); diff --git a/test/unit/lib/images-info-saver.ts b/test/unit/lib/images-info-saver.ts new file mode 100644 index 000000000..5d4bb3001 --- /dev/null +++ b/test/unit/lib/images-info-saver.ts @@ -0,0 +1,265 @@ +import * as fsOriginal from 'fs-extra'; +import {ImagesInfoSaver as ImagesInfoSaverOriginal} from 'lib/images-info-saver'; +import {Writable} from 'type-fest'; +import {ReporterTestResult} from 'lib/test-adapter'; +import { + ImageBase64, + ImageBuffer, + ImageFile, + ImageInfoDiff, + ImageInfoNoRef, + ImageInfoSuccess, + TestSpecByPath +} from 'lib/types'; +import sinon from 'sinon'; +import {LocalImageFileSaver} from 'lib/local-image-file-saver'; +import {SqliteImageStore} from 'lib/image-store'; +import {Cache} from 'lib/cache'; +import {PluginEvents, TestStatus} from 'lib/constants'; +import proxyquire from 'proxyquire'; +import _ from 'lodash'; +import {RegisterWorkers} from 'lib/workers/create-workers'; + +describe('images-info-saver', () => { + const sandbox = sinon.sandbox.create(); + + describe('save', () => { + const fs = _.clone(fsOriginal); + + const originalUtils = proxyquire('lib/server-utils', { + 'fs-extra': fs + }); + const utils = _.clone(originalUtils); + + const {ImagesInfoSaver} = proxyquire('lib/images-info-saver', { + 'fs-extra': fs, + './server-utils': utils, + 'image-size': sinon.stub() + }); + + let imagesInfoSaver: ImagesInfoSaverOriginal; + let imageFileSaver: sinon.SinonStubbedInstance; + let imageStore: sinon.SinonStubbedInstance; + let expectedPathsCache: sinon.SinonStubbedInstance>; + let reportPath: string; + + beforeEach(() => { + sandbox.stub(fs, 'readFile'); + sandbox.stub(fs, 'copy'); + sandbox.stub(fs, 'writeFile'); + + sandbox.stub(utils, 'makeDirFor').resolves(); + sandbox.stub(utils, 'getCurrentPath').returns('report-current-path'); + sandbox.stub(utils, 'getDiffPath').returns('report-diff-path'); + sandbox.stub(utils, 'getReferencePath').returns('report-expected-path'); + sandbox.stub(utils, 'getTempPath').returns('temp-path'); + + reportPath = 'test-report-path'; + imageFileSaver = { + saveImg: sinon.stub() + }; + imageStore = sinon.createStubInstance(SqliteImageStore); + expectedPathsCache = sinon.createStubInstance(Cache); + + imagesInfoSaver = new ImagesInfoSaver({ + imageFileSaver, + reportPath, + imageStore, + expectedPathsCache: expectedPathsCache as any + }); + }); + + afterEach(() => { + sandbox.restore(); + }); + + describe('actual images', () => { + it('should save and update path', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const sourcePath = 'path/to/actual-image.png'; + const destPath = 'path/to/saved-actual-image.png'; + + const actualImg: ImageFile = {path: sourcePath, size: {width: 100, height: 100}}; + const imagesInfo = [{actualImg} as ImageInfoNoRef]; + + imageFileSaver.saveImg.resolves(destPath); + testResult.imagesInfo = imagesInfo; + + const updatedTestResult = await imagesInfoSaver.save(testResult); + + const savedActualImg = updatedTestResult.imagesInfo[0].actualImg as ImageFile; + assert.calledWith(imageFileSaver.saveImg, sourcePath, sinon.match({reportDir: reportPath, destPath: 'report-current-path'})); + assert.equal(savedActualImg.path, destPath); + }); + + it('should not fail if it\'s not available', async () => { + const testResult: Writable = {} as ReporterTestResult; + + testResult.imagesInfo = [{} as ImageInfoNoRef]; + + const updatedTestResult = await imagesInfoSaver.save(testResult); + + assert.notCalled(imageFileSaver.saveImg); + assert.isUndefined(updatedTestResult.imagesInfo[0].actualImg); + }); + + it('should save base64 page screenshots', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const actualImg: ImageBase64 = {base64: 'base64string', size: {width: 100, height: 100}}; + testResult.imagesInfo = [{status: TestStatus.SUCCESS, actualImg}]; + + imageFileSaver.saveImg.resolves('path/to/saved-base64-image.png'); + + const updatedTestResult = await imagesInfoSaver.save(testResult); + + const savedActualImg = updatedTestResult.imagesInfo[0].actualImg as ImageFile; + assert.calledWith(imageFileSaver.saveImg, 'temp-path'); + assert.equal(savedActualImg.path, 'path/to/saved-base64-image.png'); + }); + }); + + describe('diff images', () => { + it('should generate diff in worker if needed', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const actualImg = {path: 'actual-path'} as ImageFile; + const expectedImg = {path: 'expected-path'} as ImageFile; + const imagesInfo = {status: TestStatus.FAIL, actualImg, expectedImg} as ImageInfoDiff; + testResult.imagesInfo = [imagesInfo]; + + const saveDiffToStub = sinon.stub().resolves(); + const workers = {saveDiffTo: saveDiffToStub} as unknown as RegisterWorkers<['saveDiffTo']>; + + sandbox.stub(utils, 'createHash').returns('123'); + + const updatedTestResult = await imagesInfoSaver.save(testResult, workers); + + const savedDiffImg = (updatedTestResult.imagesInfo[0] as ImageInfoDiff).diffImg as ImageFile; + assert.calledWith(saveDiffToStub, sinon.match({ + reference: expectedImg.path, + current: actualImg.path + }), 'report-diff-path'); + assert.calledWith(imageFileSaver.saveImg, 'report-diff-path', {reportDir: reportPath, destPath: 'report-diff-path'}); + assert.equal(savedDiffImg.path, 'report-diff-path'); + }); + + it('should do nothing unless needed', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const imagesInfo = {status: TestStatus.SUCCESS} as ImageInfoSuccess; + testResult.imagesInfo = [imagesInfo]; + + const saveDiffToStub = sinon.stub().resolves(); + const workers = {saveDiffTo: saveDiffToStub} as unknown as RegisterWorkers<['saveDiffTo']>; + + await imagesInfoSaver.save(testResult, workers); + + assert.notCalled(saveDiffToStub); + assert.notCalled(imageFileSaver.saveImg); + }); + + it('should save and update path when diff image path is provided', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const diffImg = {path: 'diff-path'} as ImageFile; + const imagesInfo = {status: TestStatus.FAIL, diffImg} as ImageInfoDiff; + testResult.imagesInfo = [imagesInfo]; + + const saveDiffToStub = sinon.stub().resolves(); + const workers = {saveDiffTo: saveDiffToStub} as unknown as RegisterWorkers<['saveDiffTo']>; + + sandbox.stub(utils, 'createHash').returns('123'); + + const updatedTestResult = await imagesInfoSaver.save(testResult, workers); + + const savedDiffImg = (updatedTestResult.imagesInfo[0] as ImageInfoDiff).diffImg as ImageFile; + assert.notCalled(saveDiffToStub); + assert.calledWith(imageFileSaver.saveImg, 'diff-path', {reportDir: reportPath, destPath: 'report-diff-path'}); + assert.equal(savedDiffImg.path, 'report-diff-path'); + }); + + it('should work fine with buffer', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const diffImg = {buffer: Buffer.from('')} as ImageBuffer; + const imagesInfo = {status: TestStatus.FAIL, diffImg} as ImageInfoDiff; + testResult.imagesInfo = [imagesInfo]; + + const saveDiffToStub = sinon.stub().resolves(); + const workers = {saveDiffTo: saveDiffToStub} as unknown as RegisterWorkers<['saveDiffTo']>; + + sandbox.stub(utils, 'createHash').returns('123'); + + const updatedTestResult = await imagesInfoSaver.save(testResult, workers); + + const savedDiffImg = (updatedTestResult.imagesInfo[0] as ImageInfoDiff).diffImg as ImageFile; + assert.notCalled(saveDiffToStub); + assert.calledWith(imageFileSaver.saveImg, 'temp-path', {reportDir: reportPath, destPath: 'report-diff-path'}); + assert.equal(savedDiffImg.path, 'report-diff-path'); + }); + }); + + describe('expected images', () => { + it('should save and update path', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const expectedImg = {path: 'expected-path'} as ImageFile; + const imagesInfo = {status: TestStatus.FAIL, expectedImg} as ImageInfoDiff; + testResult.imagesInfo = [imagesInfo]; + + const updatedTestResult = await imagesInfoSaver.save(testResult); + + const savedExpectedImg = (updatedTestResult.imagesInfo[0] as ImageInfoDiff).expectedImg as ImageFile; + assert.calledWith(imageFileSaver.saveImg, 'expected-path', {reportDir: reportPath, destPath: 'report-expected-path'}); + assert.equal(savedExpectedImg.path, 'report-expected-path'); + }); + it('should reuse previous images from cache', async () => { + const testResult: Writable = {} as ReporterTestResult; + + const expectedImg = {path: 'expected-path'} as ImageFile; + const imagesInfo = {status: TestStatus.FAIL, expectedImg} as ImageInfoDiff; + testResult.imagesInfo = [imagesInfo]; + + expectedPathsCache.has.returns(true); + expectedPathsCache.get.returns('cached-expected-path'); + + const updatedTestResult = await imagesInfoSaver.save(testResult); + + const savedExpectedImg = (updatedTestResult.imagesInfo[0] as ImageInfoDiff).expectedImg as ImageFile; + assert.notCalled(imageFileSaver.saveImg); + assert.equal(savedExpectedImg.path, 'cached-expected-path'); + }); + }); + + it('should emit TEST_SCREENSHOTS_SAVED event', async () => { + const testResult: Writable = { + fullName: 'some-name', + browserId: 'some-browser', + attempt: 0 + } as ReporterTestResult; + + const sourcePath = 'path/to/actual-image.png'; + const destPath = 'path/to/saved-actual-image.png'; + + const actualImg: ImageFile = {path: sourcePath, size: {width: 100, height: 100}}; + const imagesInfo = [{actualImg} as ImageInfoNoRef]; + + imageFileSaver.saveImg.resolves(destPath); + testResult.imagesInfo = imagesInfo; + + const eventHandler = sinon.stub(); + imagesInfoSaver.on(PluginEvents.TEST_SCREENSHOTS_SAVED, eventHandler); + + const updatedTestResult = await imagesInfoSaver.save(testResult); + + assert.calledWith(imageFileSaver.saveImg, sourcePath, sinon.match({reportDir: reportPath, destPath: 'report-current-path'})); + assert.calledWith(eventHandler, sinon.match({ + testId: 'some-name.some-browser', + attempt: 0, + imagesInfo: updatedTestResult.imagesInfo + })); + }); + }); +}); diff --git a/test/unit/lib/report-builder/gui.js b/test/unit/lib/report-builder/gui.js index a269c6c71..0ac3487a2 100644 --- a/test/unit/lib/report-builder/gui.js +++ b/test/unit/lib/report-builder/gui.js @@ -12,7 +12,7 @@ const {SUCCESS, FAIL, ERROR, SKIPPED, IDLE, RUNNING, UPDATED} = require('lib/con const {LOCAL_DATABASE_NAME} = require('lib/constants/database'); const {TestAttemptManager} = require('lib/test-attempt-manager'); const {ImageDiffError} = require('../../utils'); -const {ImageHandler} = require('lib/image-handler'); +const {ImagesInfoSaver} = require('lib/images-info-saver'); const TEST_REPORT_PATH = 'test'; const TEST_DB_PATH = `${TEST_REPORT_PATH}/${LOCAL_DATABASE_NAME}`; @@ -85,7 +85,7 @@ describe('GuiReportBuilder', () => { copyAndUpdate = sandbox.stub().callsFake(_.identity); - const imageHandler = sandbox.createStubInstance(ImageHandler); + const imagesInfoSaver = sandbox.createStubInstance(ImagesInfoSaver); hasImage = sandbox.stub().returns(true); deleteFile = sandbox.stub().resolves(); @@ -94,7 +94,7 @@ describe('GuiReportBuilder', () => { StaticReportBuilder: proxyquire('lib/report-builder/static', { '../sqlite-client': {SqliteClient}, '../image-handler': {ImageHandler: function() { - return imageHandler; + return imagesInfoSaver; }} }).StaticReportBuilder }, diff --git a/test/unit/lib/report-builder/static.js b/test/unit/lib/report-builder/static.js index 081dd5c10..49960e3ee 100644 --- a/test/unit/lib/report-builder/static.js +++ b/test/unit/lib/report-builder/static.js @@ -27,13 +27,12 @@ describe('StaticReportBuilder', () => { }); const utils = _.clone(originalUtils); - const {ImageHandler} = proxyquire('lib/image-handler', { + const {ImagesInfoSaver} = proxyquire('lib/images-info-saver', { 'fs-extra': fs, - './image-cache': {cacheExpectedPaths, cacheAllImages, cacheDiffImages}, './server-utils': utils }); - const {LocalImagesSaver} = proxyquire('lib/local-images-saver', { + const {LocalImagesSaver} = proxyquire('lib/local-image-file-saver.ts', { './server-utils': utils }); @@ -73,7 +72,7 @@ describe('StaticReportBuilder', () => { StaticReportBuilder = proxyquire('lib/report-builder/static', { 'fs-extra': fs, '../server-utils': utils, - '../image-handler': {ImageHandler} + '../images-info-saver': {ImagesInfoSaver} }).StaticReportBuilder; }); diff --git a/test/unit/workers/worker.js b/test/unit/workers/worker.js index d47407821..7dc27dbae 100644 --- a/test/unit/workers/worker.js +++ b/test/unit/workers/worker.js @@ -20,17 +20,13 @@ describe('worker', () => { }); it('should pass diffColor option as a highlightColor option to looks-same', async () => { - await worker.saveDiffTo({ - diffOpts: {diffColor: '#foobar'} - }); + await worker.saveDiffTo({diffColor: '#foobar'}); assert.calledWith(looksSame.createDiff, sinon.match({highlightColor: '#foobar'})); }); it('should pass through other diff options to looks-same', async () => { - await worker.saveDiffTo({ - diffOpts: {foo: 'bar', baz: 'qux'} - }); + await worker.saveDiffTo({foo: 'bar', baz: 'qux'}); assert.calledWith(looksSame.createDiff, sinon.match({foo: 'bar', baz: 'qux'})); }); From d0b04cb06a5d85d3c154dcfdb3e1121b6aece503 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Thu, 21 Dec 2023 02:50:37 +0300 Subject: [PATCH 4/7] fix: revert unintended changes --- hermione.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hermione.ts b/hermione.ts index be061abe8..32c57639a 100644 --- a/hermione.ts +++ b/hermione.ts @@ -118,9 +118,7 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); await reportBuilder.addTestResult(formattedResult); - }).catch((e) => { - reject(e); - })); + }).catch(reject)); }); hermione.on(hermione.events.TEST_FAIL, testResult => { @@ -130,9 +128,7 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); await reportBuilder.addTestResult(formattedResult); - }).catch((e) => { - reject(e); - })); + }).catch(reject)); }); hermione.on(hermione.events.TEST_PENDING, testResult => { From 3577a4efb44e495d9adce2ebe16ec2807fb2d926 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Tue, 26 Dec 2023 02:48:59 +0300 Subject: [PATCH 5/7] test: unit tests fixes --- hermione.ts | 51 ++-- lib/constants/database.ts | 23 +- lib/constants/tests.ts | 2 + lib/errors/index.ts | 2 +- lib/gui/tool-runner/index.ts | 70 ++---- lib/gui/tool-runner/report-subscriber.ts | 79 ++---- lib/history-utils.js | 12 +- lib/report-builder/static.ts | 1 - lib/server-utils.ts | 20 +- lib/test-adapter/hermione.ts | 64 +++-- lib/test-adapter/index.ts | 2 - lib/test-adapter/playwright.ts | 4 +- lib/test-adapter/sqlite.ts | 4 +- lib/types.ts | 49 ++-- test/unit/hermione.js | 19 +- test/unit/lib/common-utils.ts | 17 +- test/unit/lib/gui/tool-runner/index.js | 93 ++++--- .../lib/gui/tool-runner/report-subsciber.js | 14 +- test/unit/lib/history-utils.js | 4 +- ...ges-saver.js => local-image-file-saver.js} | 4 +- test/unit/lib/merge-reports/index.js | 2 + test/unit/lib/report-builder/gui.js | 238 +++--------------- test/unit/lib/report-builder/static.js | 174 +++---------- test/unit/lib/server-utils.js | 61 ++++- test/unit/lib/test-adapter/hermione.ts | 3 +- test/unit/lib/test-adapter/playwright.ts | 147 ++--------- test/unit/lib/tests-tree-builder/base.js | 98 ++++---- test/unit/lib/tests-tree-builder/gui.js | 88 +++---- test/unit/lib/tests-tree-builder/static.js | 67 +++-- test/unit/utils.js | 3 + 30 files changed, 519 insertions(+), 896 deletions(-) rename test/unit/lib/{local-images-saver.js => local-image-file-saver.js} (90%) diff --git a/hermione.ts b/hermione.ts index 32c57639a..6fea5ff4c 100644 --- a/hermione.ts +++ b/hermione.ts @@ -6,18 +6,18 @@ import PQueue from 'p-queue'; import {CommanderStatic} from '@gemini-testing/commander'; import {cliCommands} from './lib/cli-commands'; -import {hasFailedImages} from './lib/common-utils'; import {parseConfig} from './lib/config'; -import {SKIPPED, SUCCESS, TestStatus, ToolName, UNKNOWN_ATTEMPT} from './lib/constants'; +import {ToolName} from './lib/constants'; import {HtmlReporter} from './lib/plugin-api'; import {StaticReportBuilder} from './lib/report-builder/static'; import {formatTestResult, logPathToHtmlReport, logError, getExpectedCacheKey} from './lib/server-utils'; import {SqliteClient} from './lib/sqlite-client'; -import {HtmlReporterApi, ImageInfoFull, ReporterOptions, TestSpecByPath} from './lib/types'; +import {HtmlReporterApi, ReporterOptions, TestSpecByPath} from './lib/types'; import {createWorkers, CreateWorkersRunner} from './lib/workers/create-workers'; import {SqliteImageStore} from './lib/image-store'; import {Cache} from './lib/cache'; import {ImagesInfoSaver} from './lib/images-info-saver'; +import {getStatus} from './lib/test-adapter/hermione'; export = (hermione: Hermione, opts: Partial): void => { if (hermione.isWorker() || !opts.enabled) { @@ -103,40 +103,21 @@ async function handleTestResults(hermione: Hermione, reportBuilder: StaticReport const queue = new PQueue({concurrency: os.cpus().length}); const promises: Promise[] = []; - hermione.on(hermione.events.TEST_PASS, testResult => { - promises.push(queue.add(async () => { - const formattedResult = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT); + [ + {eventName: hermione.events.TEST_PASS}, + {eventName: hermione.events.RETRY}, + {eventName: hermione.events.TEST_FAIL}, + {eventName: hermione.events.TEST_PENDING} + ].forEach(({eventName}) => { + type AnyHermioneTestEvent = typeof hermione.events.TEST_PASS; - await reportBuilder.addTestResult(formattedResult); - }).catch(reject)); - }); - - hermione.on(hermione.events.RETRY, testResult => { - promises.push(queue.add(async () => { - const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - - const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); - - await reportBuilder.addTestResult(formattedResult); - }).catch(reject)); - }); - - hermione.on(hermione.events.TEST_FAIL, testResult => { - promises.push(queue.add(async () => { - const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; + hermione.on(eventName as AnyHermioneTestEvent, (testResult: HermioneTestResult) => { + promises.push(queue.add(async () => { + const formattedResult = formatTestResult(testResult, getStatus(eventName, hermione.events, testResult)); - const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); - - await reportBuilder.addTestResult(formattedResult); - }).catch(reject)); - }); - - hermione.on(hermione.events.TEST_PENDING, testResult => { - promises.push(queue.add(async () => { - const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT); - - await reportBuilder.addTestResult(formattedResult); - }).catch(reject)); + await reportBuilder.addTestResult(formattedResult); + }).catch(reject)); + }); }); hermione.on(hermione.events.RUNNER_END, () => { diff --git a/lib/constants/database.ts b/lib/constants/database.ts index 3a71ea866..23edbc78f 100644 --- a/lib/constants/database.ts +++ b/lib/constants/database.ts @@ -1,5 +1,3 @@ -import {Mutable, NameToIndexMap} from '../types'; - // TODO: change to enums export const DB_TYPES = {int: 'INT', text: 'TEXT'} as const; export const DB_COLUMNS = { @@ -41,7 +39,26 @@ export const DB_SUITES_TABLE_NAME = 'suites'; export const LOCAL_DATABASE_NAME = 'sqlite.db'; export const DATABASE_URLS_JSON_NAME = 'databaseUrls.json'; +// Precise numbers instead of just "number" make this possible: row[DB_COLUMN_INDEXES.name] === string +interface DbColumnIndexes { + [DB_COLUMNS.SUITE_PATH]: 0, + [DB_COLUMNS.SUITE_NAME]: 1, + [DB_COLUMNS.NAME]: 2, + [DB_COLUMNS.SUITE_URL]: 3, + [DB_COLUMNS.META_INFO]: 4, + [DB_COLUMNS.HISTORY]: 5, + [DB_COLUMNS.DESCRIPTION]: 6, + [DB_COLUMNS.ERROR]: 7, + [DB_COLUMNS.SKIP_REASON]: 8, + [DB_COLUMNS.IMAGES_INFO]: 9, + [DB_COLUMNS.SCREENSHOT]: 10, + [DB_COLUMNS.MULTIPLE_TABS]: 11, + [DB_COLUMNS.STATUS]: 12, + [DB_COLUMNS.TIMESTAMP]: 13, + +} + export const DB_COLUMN_INDEXES = SUITES_TABLE_COLUMNS.reduce((acc: Record, {name}, index) => { acc[name] = index; return acc; -}, {}) as NameToIndexMap>; +}, {}) as unknown as DbColumnIndexes; diff --git a/lib/constants/tests.ts b/lib/constants/tests.ts index 808755def..ea654ee40 100644 --- a/lib/constants/tests.ts +++ b/lib/constants/tests.ts @@ -3,3 +3,5 @@ export const HERMIONE_TITLE_DELIMITER = ' '; export const PWT_TITLE_DELIMITER = ' › '; export const UNKNOWN_ATTEMPT = -1; + +export const UNKNOWN_SESSION_ID = 'unknown session id'; diff --git a/lib/errors/index.ts b/lib/errors/index.ts index f68dcab3c..0025a8cd3 100644 --- a/lib/errors/index.ts +++ b/lib/errors/index.ts @@ -30,5 +30,5 @@ export interface NoRefImageError { message: string; stack?: string; currImg: ImageFile; - refImg?: ImageFile; + refImg: ImageFile; } diff --git a/lib/gui/tool-runner/index.ts b/lib/gui/tool-runner/index.ts index c8f3e1a6e..b8cfc003d 100644 --- a/lib/gui/tool-runner/index.ts +++ b/lib/gui/tool-runner/index.ts @@ -18,11 +18,10 @@ import { UPDATED, SKIPPED, IDLE, - TestStatus, ToolName, DATABASE_URLS_JSON_NAME, LOCAL_DATABASE_NAME, - PluginEvents, UNKNOWN_ATTEMPT + PluginEvents } from '../../constants'; import {formatId, mkFullTitle, mergeDatabasesForReuse, filterByEqualDiffSizes} from './utils'; import {getTestsTreeFromDatabase} from '../../db-utils/server'; @@ -32,7 +31,7 @@ import { HermioneTestResult, HtmlReporterApi, ImageFile, - ImageInfoDiff, ImageInfoUpdated, + ImageInfoDiff, ImageInfoUpdated, ImageInfoWithState, ReporterConfig, TestSpecByPath } from '../../types'; import {GuiCliOptions, GuiConfigs} from '../index'; @@ -52,28 +51,11 @@ type ToolRunnerArgs = [paths: string[], hermione: Hermione & HtmlReporterApi, co export type ToolRunnerTree = GuiReportBuilderResult & Pick; -interface HermioneTestExtended extends HermioneTest { - assertViewResults: {stateName: string, refImg: ImageFile, currImg: ImageFile}; - attempt: number; - imagesInfo: Pick[]; -} - -type HermioneTestPlain = Pick; - export interface UndoAcceptImagesResult { updatedImages: TreeImage[]; removedResults: string[]; } -// TODO: get rid of this function. It allows to format raw test, but is type-unsafe. -const formatTestResultUnsafe = ( - test: HermioneTest | HermioneTestExtended | HermioneTestPlain, - status: TestStatus, - attempt: number -): ReporterTestResult => { - return formatTestResult(test as HermioneTestResult, status, attempt); -}; - export class ToolRunner { private _testFiles: string[]; private _hermione: Hermione & HtmlReporterApi; @@ -195,9 +177,9 @@ export class ToolRunner { const reportBuilder = this._ensureReportBuilder(); return Promise.all(tests.map(async (test): Promise => { - const updateResult = this._prepareTestResult(test); + const updateResult = this._createHermioneTestResult(test); - const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT); + const formattedResultWithoutAttempt = formatTestResult(updateResult, UPDATED); const formattedResult = reportBuilder.provideAttempt(formattedResultWithoutAttempt); const formattedResultUpdated = await reporterHelper.updateReferenceImages(formattedResult, this._reportPath, this._handleReferenceUpdate.bind(this)); @@ -213,11 +195,11 @@ export class ToolRunner { const reportBuilder = this._ensureReportBuilder(); await Promise.all(tests.map(async (test) => { - const updateResult = this._prepareTestResult(test); - const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT); + const updateResult = this._createHermioneTestResult(test); + const formattedResultWithoutAttempt = formatTestResult(updateResult, UPDATED); - await Promise.all(updateResult.imagesInfo.map(async (imageInfo) => { - const {stateName} = imageInfo; + await Promise.all(formattedResultWithoutAttempt.imagesInfo.map(async (imageInfo) => { + const {stateName} = imageInfo as ImageInfoWithState; const undoResultData = reportBuilder.undoAcceptImage(formattedResultWithoutAttempt, stateName); if (undoResultData === null) { @@ -320,9 +302,9 @@ export class ToolRunner { this._tests[testId] = _.extend(test, {browserId}); if (test.pending) { - queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, SKIPPED, UNKNOWN_ATTEMPT))); + queue.add(async () => reportBuilder.addTestResult(formatTestResult(test, SKIPPED))); } else { - queue.add(async () => reportBuilder.addTestResult(formatTestResultUnsafe(test, IDLE, UNKNOWN_ATTEMPT))); + queue.add(async () => reportBuilder.addTestResult(formatTestResult(test, IDLE))); } }); @@ -338,40 +320,36 @@ export class ToolRunner { subscribeOnToolEvents(this._hermione, this._ensureReportBuilder(), this._eventSource); } - protected _prepareTestResult(test: TestRefUpdateData): HermioneTestExtended | HermioneTestPlain { - const {browserId, attempt} = test; - const fullTitle = mkFullTitle(test); + protected _createHermioneTestResult(updateData: TestRefUpdateData): HermioneTestResult { + const {browserId} = updateData; + const fullTitle = mkFullTitle(updateData); const testId = formatId(getShortMD5(fullTitle), browserId); - const rawTest = this._tests[testId]; - const {sessionId, url} = test.metaInfo; + const hermioneTest = this._tests[testId]; + const {sessionId, url} = updateData.metaInfo as {sessionId?: string; url?: string}; const assertViewResults: AssertViewResult[] = []; - const imagesInfo = test.imagesInfo + updateData.imagesInfo .filter(({stateName, actualImg}) => Boolean(stateName) && Boolean(actualImg)) - .map((imageInfo) => { + .forEach((imageInfo) => { const {stateName, actualImg} = imageInfo as {stateName: string, actualImg: ImageFile}; - const path = this._hermione.config.browsers[browserId].getScreenshotPath(rawTest, stateName); + const path = this._hermione.config.browsers[browserId].getScreenshotPath(hermioneTest, stateName); const refImg = {path, size: actualImg.size}; assertViewResults.push({stateName, refImg, currImg: actualImg, isUpdated: isUpdatedStatus(imageInfo.status)}); - - return _.extend(imageInfo, {expectedImg: refImg}); }); - const res = _.merge({}, rawTest, { + const hermioneTestResult: HermioneTestResult = _.merge({}, hermioneTest, { assertViewResults, - err: test.error, - imagesInfo, + err: updateData.error as HermioneTestResult['err'], sessionId, - attempt, meta: {url} - }); + } satisfies Partial) as unknown as HermioneTestResult; // _.merge can't fully clone test object since hermione@7+ // TODO: use separate object to represent test results. Do not extend test object with test results - return rawTest && rawTest.clone - ? Object.assign(rawTest.clone(), res) - : res; + return hermioneTest && hermioneTest.clone + ? Object.assign(hermioneTest.clone(), hermioneTestResult) + : hermioneTestResult; } protected _handleReferenceUpdate(testResult: ReporterTestResult, imageInfo: ImageInfoUpdated, state: string): void { diff --git a/lib/gui/tool-runner/report-subscriber.ts b/lib/gui/tool-runner/report-subscriber.ts index aa3791694..8db2b62f6 100644 --- a/lib/gui/tool-runner/report-subscriber.ts +++ b/lib/gui/tool-runner/report-subscriber.ts @@ -1,15 +1,15 @@ import os from 'os'; import PQueue from 'p-queue'; -import Hermione from 'hermione'; +import Hermione, {Test as HermioneTest} from 'hermione'; import {ClientEvents} from '../constants'; import {getSuitePath} from '../../plugin-utils'; import {createWorkers, CreateWorkersRunner} from '../../workers/create-workers'; import {logError, formatTestResult} from '../../server-utils'; -import {hasFailedImages} from '../../common-utils'; -import {TestStatus, RUNNING, SUCCESS, SKIPPED, UNKNOWN_ATTEMPT} from '../../constants'; +import {TestStatus} from '../../constants'; import {GuiReportBuilder} from '../../report-builder/gui'; import {EventSource} from '../event-source'; -import {HermioneTestResult, ImageInfoFull} from '../../types'; +import {HermioneTestResult} from '../../types'; +import {getStatus} from '../../test-adapter/hermione'; export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiReportBuilder, client: EventSource): void => { const queue = new PQueue({concurrency: os.cpus().length}); @@ -29,63 +29,26 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo }); }); - hermione.on(hermione.events.TEST_BEGIN, (data) => { - queue.add(async () => { - const formattedResultWithoutAttempt = formatTestResult(data as HermioneTestResult, RUNNING, UNKNOWN_ATTEMPT); + [ + {eventName: hermione.events.TEST_BEGIN, clientEventName: ClientEvents.BEGIN_STATE}, + {eventName: hermione.events.TEST_PASS, clientEventName: ClientEvents.TEST_RESULT}, + {eventName: hermione.events.RETRY, clientEventName: ClientEvents.TEST_RESULT}, + {eventName: hermione.events.TEST_FAIL, clientEventName: ClientEvents.TEST_RESULT}, + {eventName: hermione.events.TEST_PENDING, clientEventName: ClientEvents.TEST_RESULT} + ].forEach(({eventName, clientEventName}) => { + type AnyHermioneTestEvent = typeof hermione.events.TEST_PASS; - const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); - const testBranch = reportBuilder.getTestBranch(formattedResult.id); + hermione.on(eventName as AnyHermioneTestEvent, (data: HermioneTest | HermioneTestResult) => { + queue.add(async () => { + const status = getStatus(eventName, hermione.events, data as HermioneTestResult); + const formattedResultWithoutAttempt = formatTestResult(data, status); - return client.emit(ClientEvents.BEGIN_STATE, testBranch); - }); - }); - - hermione.on(hermione.events.TEST_PASS, (testResult) => { - queue.add(async () => { - const formattedResultWithoutAttempt = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT); - - const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); - - const testBranch = reportBuilder.getTestBranch(formattedResult.id); - client.emit(ClientEvents.TEST_RESULT, testBranch); - }).catch(logError); - }); - - hermione.on(hermione.events.RETRY, (testResult) => { - queue.add(async () => { - const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - - const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); + const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); + const testBranch = reportBuilder.getTestBranch(formattedResult.id); - const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); - - const testBranch = reportBuilder.getTestBranch(formattedResult.id); - client.emit(ClientEvents.TEST_RESULT, testBranch); - }).catch(logError); - }); - - hermione.on(hermione.events.TEST_FAIL, (testResult) => { - queue.add(async () => { - const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - - const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT); - - const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); - - const testBranch = reportBuilder.getTestBranch(formattedResult.id); - client.emit(ClientEvents.TEST_RESULT, testBranch); - }).catch(logError); - }); - - hermione.on(hermione.events.TEST_PENDING, async (testResult) => { - queue.add(async () => { - const formattedResultWithoutAttempt = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT); - - const formattedResult = await reportBuilder.addTestResult(formattedResultWithoutAttempt); - - const testBranch = reportBuilder.getTestBranch(formattedResult.id); - client.emit(ClientEvents.TEST_RESULT, testBranch); - }).catch(logError); + return client.emit(clientEventName, testBranch); + }).catch(logError); + }); }); hermione.on(hermione.events.RUNNER_END, async () => { diff --git a/lib/history-utils.js b/lib/history-utils.js index b3569488b..cf597b5b4 100644 --- a/lib/history-utils.js +++ b/lib/history-utils.js @@ -23,11 +23,11 @@ const traverseNodes = (nodes, traverseCb, depth = 0) => { const getCommandsHistory = (history) => { if (isEmpty(history)) { - return; + return []; } try { - const formatedHistory = []; + const formattedHistory = []; const traverseCb = (node, depth) => { const offset = '\t'.repeat(depth); @@ -35,18 +35,18 @@ const getCommandsHistory = (history) => { const duration = node.d; const isFailed = Boolean(node.f); const title = isGroup ? node.n : getCommand(node); - const formatedDuration = formatDuration(duration); + const formattedDuration = formatDuration(duration); - formatedHistory.push(`${offset}${title} ${formatedDuration}\n`); + formattedHistory.push(`${offset}${title} ${formattedDuration}\n`); return isFailed; }; traverseNodes(history, traverseCb); - return formatedHistory; + return formattedHistory; } catch (e) { - return `failed to get command history: ${e.message}`; + return [`failed to get command history: ${e.message}`]; } }; diff --git a/lib/report-builder/static.ts b/lib/report-builder/static.ts index ea8d6b1e0..37f14f299 100644 --- a/lib/report-builder/static.ts +++ b/lib/report-builder/static.ts @@ -1,6 +1,5 @@ import path from 'path'; import {GeneralEventEmitter} from 'eventemitter2'; -import _ from 'lodash'; import fs from 'fs-extra'; import PQueue from 'p-queue'; diff --git a/lib/server-utils.ts b/lib/server-utils.ts index d2930090f..59f099879 100644 --- a/lib/server-utils.ts +++ b/lib/server-utils.ts @@ -1,10 +1,16 @@ +import crypto from 'crypto'; import path from 'path'; import url from 'url'; + import chalk from 'chalk'; -import _ from 'lodash'; +import {Router} from 'express'; import fs from 'fs-extra'; +import Hermione, {Test as HermioneTest} from 'hermione'; +import _ from 'lodash'; +import tmp from 'tmp'; + import {getShortMD5, logger, mkTestId} from './common-utils'; -import {UPDATED, RUNNING, IDLE, SKIPPED, IMAGES_PATH, TestStatus} from './constants'; +import {UPDATED, RUNNING, IDLE, SKIPPED, IMAGES_PATH, TestStatus, UNKNOWN_ATTEMPT} from './constants'; import type {HtmlReporter} from './plugin-api'; import type {ReporterTestResult} from './test-adapter'; import { @@ -14,11 +20,7 @@ import { ReporterConfig, TestSpecByPath } from './types'; -import type Hermione from 'hermione'; -import crypto from 'crypto'; -import {HermioneTestAdapter} from './test-adapter'; -import {Router} from 'express'; -import tmp from 'tmp'; +import {HermioneTestAdapter} from './test-adapter/hermione'; const DATA_FILE_NAME = 'data.js'; @@ -313,9 +315,9 @@ export function mapPlugins(plugins: ReporterConfig['plugins'], callback: (nam } export const formatTestResult = ( - rawResult: HermioneTestResult, + rawResult: HermioneTest | HermioneTestResult, status: TestStatus, - attempt: number + attempt: number = UNKNOWN_ATTEMPT ): ReporterTestResult => { return new HermioneTestAdapter(rawResult, {attempt, status}); }; diff --git a/lib/test-adapter/hermione.ts b/lib/test-adapter/hermione.ts index 1edfc689f..c82b6ddf3 100644 --- a/lib/test-adapter/hermione.ts +++ b/lib/test-adapter/hermione.ts @@ -1,26 +1,43 @@ -import _ from 'lodash'; import path from 'path'; +import _ from 'lodash'; +import Hermione, {Test as HermioneTest} from 'hermione'; +import {ValueOf} from 'type-fest'; import {getCommandsHistory} from '../history-utils'; -import {ERROR, FAIL, SUCCESS, TestStatus, UPDATED} from '../constants'; -import {getError, isImageDiffError, isNoRefImageError, wrapLinkByTag} from '../common-utils'; +import {ERROR, FAIL, SUCCESS, TestStatus, UNKNOWN_SESSION_ID, UPDATED} from '../constants'; +import {getError, hasFailedImages, isImageDiffError, isNoRefImageError, wrapLinkByTag} from '../common-utils'; import { ErrorDetails, - ImageBase64, - ImageInfoFull, - HermioneTestResult, HermioneSuite, - TestError, + HermioneTestResult, + ImageBase64, + ImageFile, ImageInfoDiff, + ImageInfoFull, ImageInfoNoRef, + ImageInfoPageError, + ImageInfoPageSuccess, ImageInfoSuccess, - ImageFile, - ImageInfoPageError, ImageInfoPageSuccess, ImageInfoUpdated + ImageInfoUpdated, + TestError } from '../types'; import {ReporterTestResult} from './index'; import {getSuitePath} from '../plugin-utils'; import {extractErrorDetails} from './utils'; +export const getStatus = (eventName: ValueOf, events: Hermione['events'], testResult: HermioneTestResult): TestStatus => { + if (eventName === events.TEST_PASS) { + return TestStatus.SUCCESS; + } else if (eventName === events.TEST_PENDING) { + return TestStatus.SKIPPED; + } else if (eventName === events.RETRY || eventName === events.TEST_FAIL) { + return hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; + } else if (eventName === events.TEST_BEGIN) { + return TestStatus.RUNNING; + } + return TestStatus.IDLE; +}; + const getSkipComment = (suite: HermioneTestResult | HermioneSuite): string | null | undefined => { return suite.skipReason || suite.parent && getSkipComment(suite.parent); }; @@ -35,7 +52,7 @@ export interface HermioneTestAdapterOptions { } export class HermioneTestAdapter implements ReporterTestResult { - private _testResult: HermioneTestResult; + private _testResult: HermioneTest | HermioneTestResult; private _errorDetails: ErrorDetails | null; private _timestamp: number | undefined; private _attempt: number; @@ -45,10 +62,11 @@ export class HermioneTestAdapter implements ReporterTestResult { return new this(testResult, options); } - constructor(testResult: HermioneTestResult, {attempt, status}: HermioneTestAdapterOptions) { + constructor(testResult: HermioneTest | HermioneTestResult, {attempt, status}: HermioneTestAdapterOptions) { this._testResult = testResult; this._errorDetails = null; - this._timestamp = this._testResult.timestamp ?? this._testResult.startTime ?? Date.now(); + this._timestamp = (this._testResult as HermioneTestResult).timestamp ?? + (this._testResult as HermioneTestResult).startTime ?? Date.now(); this._status = status; const browserVersion = _.get(this._testResult, 'meta.browserVersion', this._testResult.browserVersion); @@ -63,7 +81,7 @@ export class HermioneTestAdapter implements ReporterTestResult { } get skipReason(): string { - return wrapSkipComment(getSkipComment(this._testResult)); + return wrapSkipComment(getSkipComment(this._testResult as HermioneTestResult)); } get status(): TestStatus { @@ -71,7 +89,7 @@ export class HermioneTestAdapter implements ReporterTestResult { } get sessionId(): string { - return this._testResult.sessionId || 'unknown session id'; + return (this._testResult as HermioneTestResult).sessionId || UNKNOWN_SESSION_ID; } get browserId(): string { @@ -79,7 +97,7 @@ export class HermioneTestAdapter implements ReporterTestResult { } get imagesInfo(): ImageInfoFull[] { - const {assertViewResults = []} = this._testResult; + const {assertViewResults = []} = this._testResult as HermioneTestResult; const imagesInfo: ImageInfoFull[] = assertViewResults.map((assertResult): ImageInfoFull => { if (isImageDiffError(assertResult)) { @@ -92,7 +110,7 @@ export class HermioneTestAdapter implements ReporterTestResult { refImg: assertResult.refImg, actualImg: assertResult.currImg, ...(diffImg ? {diffImg} : {}), - expectedImg: assertResult.refImg, + expectedImg: _.clone(assertResult.refImg), diffClusters: assertResult.diffClusters, diffOptions: assertResult.diffOpts } satisfies ImageInfoDiff; @@ -109,7 +127,7 @@ export class HermioneTestAdapter implements ReporterTestResult { status: UPDATED, stateName: assertResult.stateName, refImg: assertResult.refImg, - expectedImg: assertResult.refImg, + expectedImg: _.clone(assertResult.refImg), actualImg: (assertResult as {currImg: ImageFile}).currImg } satisfies ImageInfoUpdated; } else { @@ -118,7 +136,7 @@ export class HermioneTestAdapter implements ReporterTestResult { status: SUCCESS, stateName: assertResult.stateName, refImg: assertResult.refImg, - expectedImg: assertResult.refImg, + expectedImg: _.clone(assertResult.refImg), ...(currImg ? {actualImg: currImg} : {}) } satisfies ImageInfoSuccess; } @@ -139,11 +157,11 @@ export class HermioneTestAdapter implements ReporterTestResult { } get history(): string[] { - return getCommandsHistory(this._testResult.history) as string[]; + return getCommandsHistory((this._testResult as HermioneTestResult).history) as string[]; } get error(): undefined | TestError { - return this._testResult.err; + return (this._testResult as HermioneTestResult).err; } get imageDir(): string { @@ -168,11 +186,11 @@ export class HermioneTestAdapter implements ReporterTestResult { } get description(): string | undefined { - return this._testResult.description; + return (this._testResult as HermioneTestResult).description; } get meta(): Record { - return this._testResult.meta; + return (this._testResult as HermioneTestResult).meta ?? {}; } get errorDetails(): ErrorDetails | null { @@ -190,7 +208,7 @@ export class HermioneTestAdapter implements ReporterTestResult { } get url(): string | undefined { - return this._testResult.meta.url as string | undefined; + return (this._testResult as HermioneTestResult).meta?.url as string | undefined; } get multipleTabs(): boolean { diff --git a/lib/test-adapter/index.ts b/lib/test-adapter/index.ts index 0f360d6f3..db772e3bf 100644 --- a/lib/test-adapter/index.ts +++ b/lib/test-adapter/index.ts @@ -1,8 +1,6 @@ import {TestStatus} from '../constants'; import {ErrorDetails, ImageBase64, ImageFile, ImageInfoFull, TestError} from '../types'; -export * from './hermione'; - export interface ReporterTestResult { readonly attempt: number; readonly browserId: string; diff --git a/lib/test-adapter/playwright.ts b/lib/test-adapter/playwright.ts index 7074e8cd6..9eede117d 100644 --- a/lib/test-adapter/playwright.ts +++ b/lib/test-adapter/playwright.ts @@ -221,7 +221,7 @@ export class PlaywrightTestAdapter implements ReporterTestResult { refImg, diffImg, actualImg, - expectedImg: refImg, + expectedImg: _.clone(refImg), diffClusters: _.get(error, 'diffClusters', []), // TODO: extract diffOptions from config diffOptions: {current: actualImg.path, reference: refImg.path, ...DEFAULT_DIFF_OPTIONS} @@ -239,7 +239,7 @@ export class PlaywrightTestAdapter implements ReporterTestResult { status: SUCCESS, stateName: state, refImg, - expectedImg: refImg, + expectedImg: _.clone(refImg), ...(actualImg ? {actualImg} : {}) } satisfies ImageInfoSuccess; } diff --git a/lib/test-adapter/sqlite.ts b/lib/test-adapter/sqlite.ts index 9c156d642..658ba3ee7 100644 --- a/lib/test-adapter/sqlite.ts +++ b/lib/test-adapter/sqlite.ts @@ -32,9 +32,7 @@ export class SqliteTestAdapter implements ReporterTestResult { constructor(testResult: RawSuitesRow, attempt: number, options: SqliteTestAdapterOptions) { this._testResult = testResult; - this._parsedTestResult = {attempt}; - this._titleDelimiter = options.titleDelimiter; } @@ -110,7 +108,7 @@ export class SqliteTestAdapter implements ReporterTestResult { get meta(): Record { if (!_.has(this._parsedTestResult, 'meta')) { - this._parsedTestResult.meta = tryParseJson(this._testResult[DB_COLUMN_INDEXES.metaInfo]) as Record; + this._parsedTestResult.meta = tryParseJson(this._testResult[DB_COLUMN_INDEXES.metaInfo]) as Record ?? {}; } return this._parsedTestResult.meta as Record; diff --git a/lib/types.ts b/lib/types.ts index 5097baca0..f93ce6528 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -1,9 +1,8 @@ import type {LooksSameOptions, CoordBounds} from 'looks-same'; import type {default as Hermione, TestResult as HermioneTestResultOriginal} from 'hermione'; -import {DB_TYPES, DiffModeId, SaveFormat, SUITES_TABLE_COLUMNS, TestStatus, ViewMode} from './constants'; +import {DiffModeId, SaveFormat, SUITES_TABLE_COLUMNS, TestStatus, ViewMode} from './constants'; import type {HtmlReporter} from './plugin-api'; import {ImageDiffError, NoRefImageError} from './errors'; -import {EmptyObject, ValueOf} from 'type-fest'; declare module 'tmp' { export const tmpdir: string; @@ -181,36 +180,22 @@ export interface DbUrlsJsonData { jsonUrls: string[]; } -type DbType = ValueOf; - -type Length = - T extends { length: infer L } ? L : never; - -// Writable from type-fest didn't work here, because it transforms array to object -export type Mutable = { - -readonly [K in keyof T]: T[K] -} - -type ExtractType = T extends typeof DB_TYPES.int ? number : - T extends typeof DB_TYPES.text ? string : never; - -// This type accepts an array of objects shaped {name: string} and returns map of shape {: } -// Useful to produce precise type of db columns order, e.g. {suitePath: 0, suiteName: 1, ...} -export type NameToIndexMap = T extends [infer Head, ...infer Tail] ? - Tail extends {name: string}[] ? - Head extends {name: string} ? NameToIndexMap}, [Head, ...Processed]> : never - : never - : Result; - -// This type accepts an array of objects shaped {type: DbType} and returns map of shape {: } -// Useful to produce precise type of table row in db, similar to a tuple, e.g. [string, string, number, ...] -type IndexToTypeMap = T extends [infer Head, ...infer Tail] ? - Tail extends {type: DbType}[] ? - Head extends {type: DbType} ? IndexToTypeMap]: ExtractType}, [Head, ...Processed]> : never - : never - : Result; - -export type RawSuitesRow = IndexToTypeMap>; +export type RawSuitesRow = [ + suitePath: string, + suiteName: string, + name: string, + suiteUrl: string, + metaInfo: string, + history: string, + description: string, + error: string, + skipReason: string, + imagesInfo: string, + screenshot: number, + multipleTabs: number, + status: string, + timestamp: number, +]; export type LabeledSuitesRow = { [K in (typeof SUITES_TABLE_COLUMNS)[number]['name']]: string; diff --git a/test/unit/hermione.js b/test/unit/hermione.js index f418baf25..4befef487 100644 --- a/test/unit/hermione.js +++ b/test/unit/hermione.js @@ -159,12 +159,7 @@ describe('lib/hermione', () => { sandbox.stub(logger, 'log'); sandbox.stub(logger, 'warn'); - sandbox.stub(StaticReportBuilder.prototype, 'addSkipped').callsFake(_.identity); - sandbox.stub(StaticReportBuilder.prototype, 'addSuccess').callsFake(_.identity); - sandbox.stub(StaticReportBuilder.prototype, 'addError').callsFake(_.identity); - sandbox.stub(StaticReportBuilder.prototype, 'addFail').callsFake(_.identity); - sandbox.stub(StaticReportBuilder.prototype, 'addRetry').callsFake(_.identity); - sandbox.stub(StaticReportBuilder.prototype, 'saveStaticFiles'); + sandbox.stub(StaticReportBuilder.prototype, 'addTestResult').callsFake(_.identity); sandbox.stub(StaticReportBuilder.prototype, 'finalize'); sandbox.stub(SqliteClient.prototype, 'query'); @@ -203,7 +198,7 @@ describe('lib/hermione', () => { hermione.emit(events.TEST_PENDING, mkStubResult_({title: 'some-title'})); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addSkipped.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addTestResult.args[0][0].state, {name: 'some-title'}); }); it('should add passed test to result', async () => { @@ -211,7 +206,7 @@ describe('lib/hermione', () => { hermione.emit(events.TEST_PASS, mkStubResult_({title: 'some-title'})); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addSuccess.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addTestResult.args[0][0].state, {name: 'some-title'}); }); ['TEST_FAIL', 'RETRY'].forEach((event) => { @@ -223,7 +218,7 @@ describe('lib/hermione', () => { hermione.emit(events[event], testResult); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addFail.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addTestResult.args[0][0].state, {name: 'some-title'}); }); it(`errored assert view to result on ${event} event`, async () => { @@ -234,7 +229,7 @@ describe('lib/hermione', () => { hermione.emit(events[event], mkStubResult_({title: 'some-title', assertViewResults: [err]})); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addFail.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addTestResult.args[0][0].state, {name: 'some-title'}); }); it(`failed test to result on ${event} event`, async () => { @@ -250,7 +245,7 @@ describe('lib/hermione', () => { hermione.emit(events[event], testResult); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addFail.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addTestResult.args[0][0].state, {name: 'some-title'}); }); it(`failed test to result on ${event} event`, async () => { @@ -262,7 +257,7 @@ describe('lib/hermione', () => { hermione.emit(events[event], mkStubResult_({title: 'some-title', assertViewResults: [err]})); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addFail.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addTestResult.args[0][0].state, {name: 'some-title'}); }); }); }); diff --git a/test/unit/lib/common-utils.ts b/test/unit/lib/common-utils.ts index 995d651de..218a1000f 100644 --- a/test/unit/lib/common-utils.ts +++ b/test/unit/lib/common-utils.ts @@ -1,8 +1,13 @@ -import {determineFinalStatus, getError, hasDiff, getUrlWithBase} from 'lib/common-utils'; +import {determineFinalStatus, getError, hasDiff, getUrlWithBase, getDetailsFileName} from 'lib/common-utils'; import {RUNNING, QUEUED, ERROR, FAIL, UPDATED, SUCCESS, IDLE, SKIPPED} from 'lib/constants/test-statuses'; import {ErrorName} from 'lib/errors'; +import sinon from 'sinon'; describe('common-utils', () => { + const sandbox = sinon.sandbox.create(); + + afterEach(() => sandbox.restore()); + describe('getUrlWithBase', () => { it('should change host of the url', () => { const userUrl = 'https://example.com/test/123?a=1#hello'; @@ -155,4 +160,14 @@ describe('common-utils', () => { }); }); }); + + describe('getDetailsFileName', () => { + it('should compose correct file name from suite path, browser id and attempt', () => { + sandbox.stub(Date, 'now').returns('123456789'); + const testId = 'abcdef'; + const expected = `${testId}-bro_2_123456789.json`; + + assert.equal(getDetailsFileName(testId, 'bro', 1), expected); + }); + }); }); diff --git a/test/unit/lib/gui/tool-runner/index.js b/test/unit/lib/gui/tool-runner/index.js index f9949ab97..577498ab8 100644 --- a/test/unit/lib/gui/tool-runner/index.js +++ b/test/unit/lib/gui/tool-runner/index.js @@ -9,7 +9,8 @@ const {LOCAL_DATABASE_NAME} = require('lib/constants/database'); const {logger} = require('lib/common-utils'); const {stubTool, stubConfig, mkImagesInfo, mkState, mkSuite} = require('test/unit/utils'); const {SqliteClient} = require('lib/sqlite-client'); -const {PluginEvents, TestStatus} = require('lib/constants'); +const {PluginEvents, TestStatus, UPDATED} = require('lib/constants'); +const {Cache} = require('lib/cache'); describe('lib/gui/tool-runner/index', () => { const sandbox = sinon.createSandbox(); @@ -19,10 +20,10 @@ describe('lib/gui/tool-runner/index', () => { let hermione; let getTestsTreeFromDatabase; let looksSame; - let removeReferenceImage; - let revertReferenceImage; let toolRunnerUtils; let createTestRunner; + let getReferencePath; + let reporterHelpers; const mkTestCollection_ = (testsTree = {}) => { return { @@ -77,18 +78,33 @@ describe('lib/gui/tool-runner/index', () => { }; reportBuilder = sinon.createStubInstance(GuiReportBuilder); - reportBuilder.addUpdated.callsFake(_.identity); + reportBuilder.addTestResult.callsFake(_.identity); + reportBuilder.provideAttempt.callsFake(_.identity); subscribeOnToolEvents = sandbox.stub().named('reportSubscriber').resolves(); looksSame = sandbox.stub().named('looksSame').resolves({equal: true}); - removeReferenceImage = sandbox.stub().resolves(); - revertReferenceImage = sandbox.stub().resolves(); sandbox.stub(GuiReportBuilder, 'create').returns(reportBuilder); reportBuilder.getResult.returns({}); getTestsTreeFromDatabase = sandbox.stub().returns({}); + getReferencePath = sinon.stub().returns(''); + + const reporterHelpersOriginal = proxyquire('lib/reporter-helpers', { + './server-utils': { + copyFileAsync: sinon.stub().resolves(), + getCurrentAbsolutePath: sinon.stub(), + getReferencePath, + fileExists: sinon.stub(), + deleteFile: sinon.stub() + }, + './test-adapter/utils': { + copyAndUpdate: sinon.stub().callsFake(_.assign) + } + }); + reporterHelpers = _.clone(reporterHelpersOriginal); + ToolGuiReporter = proxyquire(`lib/gui/tool-runner`, { 'looks-same': looksSame, './runner': {createTestRunner}, @@ -96,14 +112,9 @@ describe('lib/gui/tool-runner/index', () => { './utils': toolRunnerUtils, '../../sqlite-client': {SqliteClient: {create: () => sinon.createStubInstance(SqliteClient)}}, '../../db-utils/server': {getTestsTreeFromDatabase}, - '../../reporter-helpers': { - updateReferenceImage: sandbox.stub().resolves(), - removeReferenceImage, - revertReferenceImage - } + '../../reporter-helpers': reporterHelpers }).ToolRunner; - sandbox.stub(reportBuilder, 'imageHandler').value({updateCacheExpectedPath: sinon.stub()}); sandbox.stub(logger, 'warn'); }); @@ -111,7 +122,7 @@ describe('lib/gui/tool-runner/index', () => { describe('initialize', () => { it('should set values added through api', () => { - const htmlReporter = {emit: sinon.stub(), values: {foo: 'bar'}}; + const htmlReporter = {emit: sinon.stub(), values: {foo: 'bar'}, config: {}, imagesSaver: {}}; hermione = stubTool(stubConfig(), {}, {}, htmlReporter); const gui = initGuiReporter(hermione); @@ -152,8 +163,7 @@ describe('lib/gui/tool-runner/index', () => { return gui.initialize() .then(() => { - assert.notCalled(reportBuilder.addSkipped); - assert.notCalled(reportBuilder.addIdle); + assert.notCalled(reportBuilder.addTestResult); }); }); @@ -165,8 +175,7 @@ describe('lib/gui/tool-runner/index', () => { return gui.initialize() .then(() => { - assert.notCalled(reportBuilder.addSkipped); - assert.notCalled(reportBuilder.addIdle); + assert.notCalled(reportBuilder.addTestResult); }); }); @@ -180,8 +189,7 @@ describe('lib/gui/tool-runner/index', () => { return gui.initialize() .then(() => { - assert.notCalled(reportBuilder.addSkipped); - assert.notCalled(reportBuilder.addIdle); + assert.notCalled(reportBuilder.addTestResult); }); }); @@ -192,7 +200,7 @@ describe('lib/gui/tool-runner/index', () => { const gui = initGuiReporter(hermione, {paths: ['foo']}); return gui.initialize() - .then(() => assert.calledOnce(reportBuilder.addSkipped)); + .then(() => assert.calledOnce(reportBuilder.addTestResult)); }); it('should add idle test to report', () => { @@ -202,7 +210,7 @@ describe('lib/gui/tool-runner/index', () => { const gui = initGuiReporter(hermione, {paths: ['foo']}); return gui.initialize() - .then(() => assert.calledOnce(reportBuilder.addIdle)); + .then(() => assert.calledOnce(reportBuilder.addTestResult)); }); it('should subscribe on events before read tests', () => { @@ -230,30 +238,31 @@ describe('lib/gui/tool-runner/index', () => { describe('updateReferenceImage', () => { describe('should emit "UPDATE_REFERENCE" event', () => { it('should emit "UPDATE_REFERENCE" event with state and reference data', async () => { - const tests = [{ + const testRefUpdateData = [{ id: 'some-id', fullTitle: () => 'some-title', browserId: 'yabro', suite: {path: ['suite1']}, state: {}, metaInfo: {}, - imagesInfo: [mkImagesInfo({ + imagesInfo: [{ + status: UPDATED, stateName: 'plain1', actualImg: { size: {height: 100, width: 200} } - })] + }] }]; const getScreenshotPath = sandbox.stub().returns('/ref/path1'); const config = stubConfig({ browsers: {yabro: {getScreenshotPath}} }); - const hermione = mkHermione_(config, {'some-title.yabro': tests[0]}); - const gui = initGuiReporter(hermione); + const hermione = mkHermione_(config, {'some-title.yabro': testRefUpdateData[0]}); + const gui = initGuiReporter(hermione, {pluginConfig: {path: 'report-path'}}); await gui.initialize(); - await gui.updateReferenceImage(tests); + await gui.updateReferenceImage(testRefUpdateData); assert.calledOnceWith(hermione.emit, 'updateReference', { refImg: {path: '/ref/path1', size: {height: 100, width: 200}}, @@ -270,18 +279,20 @@ describe('lib/gui/tool-runner/index', () => { state: {}, metaInfo: {}, imagesInfo: [ - mkImagesInfo({ + { + status: UPDATED, stateName: 'plain1', actualImg: { size: {height: 100, width: 200} } - }), - mkImagesInfo({ + }, + { + status: UPDATED, stateName: 'plain2', actualImg: { size: {height: 200, width: 300} } - }) + } ] }]; @@ -328,13 +339,13 @@ describe('lib/gui/tool-runner/index', () => { state: {}, metaInfo: {}, imagesInfo: [ - mkImagesInfo({ + { + status: TestStatus.UPDATED, stateName, actualImg: { size: {height: 100, width: 200} - }, - status: TestStatus.UPDATED - }) + } + } ] }]; @@ -350,15 +361,17 @@ describe('lib/gui/tool-runner/index', () => { }; it('should remove reference, if ReportBuilder.undoAcceptImages resolved "shouldRemoveReference"', async () => { + sandbox.stub(reporterHelpers, 'removeReferenceImage'); const stateName = 'plain'; const {gui, tests} = await mkUndoTestData_({shouldRemoveReference: true}, {stateName}); await gui.undoAcceptImages(tests); - assert.calledOnceWith(removeReferenceImage, sinon.match({fullName: 'some-title'}), 'plain'); + assert.calledOnceWith(reporterHelpers.removeReferenceImage, sinon.match({fullName: 'some-title'}), 'plain'); }); it('should revert reference, if ReportBuilder.undoAcceptImages resolved "shouldRevertReference"', async () => { + sandbox.stub(reporterHelpers, 'revertReferenceImage'); const stateName = 'plain'; const {gui, tests} = await mkUndoTestData_({ shouldRevertReference: true, removedResult: 'some-result' @@ -366,17 +379,21 @@ describe('lib/gui/tool-runner/index', () => { await gui.undoAcceptImages(tests); - assert.calledOnceWith(revertReferenceImage, 'some-result', sinon.match({fullName: 'some-title'}), 'plain'); + assert.calledOnceWith(reporterHelpers.revertReferenceImage, 'some-result', sinon.match({fullName: 'some-title'}), 'plain'); }); it('should update expected path', async () => { + sandbox.stub(Cache.prototype, 'set'); const stateName = 'plain'; const previousExpectedPath = 'previousExpectedPath'; const {gui, tests} = await mkUndoTestData_({previousExpectedPath}, {stateName}); await gui.undoAcceptImages(tests); - assert.calledOnceWith(reportBuilder.imageHandler.updateCacheExpectedPath, sinon.match.any, stateName, previousExpectedPath); + assert.calledOnce(Cache.prototype.set); + const args = Cache.prototype.set.firstCall.args; + assert.deepEqual(args[0], [{browserId: 'yabro', testPath: ['some-title']}, stateName]); + assert.deepEqual(args[1], previousExpectedPath); }); }); diff --git a/test/unit/lib/gui/tool-runner/report-subsciber.js b/test/unit/lib/gui/tool-runner/report-subsciber.js index 5616a8ae0..e863e6fcb 100644 --- a/test/unit/lib/gui/tool-runner/report-subsciber.js +++ b/test/unit/lib/gui/tool-runner/report-subsciber.js @@ -7,7 +7,7 @@ const {subscribeOnToolEvents} = require('lib/gui/tool-runner/report-subscriber') const {GuiReportBuilder} = require('lib/report-builder/gui'); const {ClientEvents} = require('lib/gui/constants'); const {stubTool, stubConfig} = require('test/unit/utils'); -const {HermioneTestAdapter} = require('lib/test-adapter'); +const {HermioneTestAdapter} = require('lib/test-adapter/hermione'); const {UNKNOWN_ATTEMPT} = require('lib/constants'); describe('lib/gui/tool-runner/hermione/report-subscriber', () => { @@ -34,13 +34,9 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { beforeEach(() => { reportBuilder = sinon.createStubInstance(GuiReportBuilder); - reportBuilder.addFail.callsFake(_.identity); - reportBuilder.addError.callsFake(_.identity); - reportBuilder.addRunning.callsFake(_.identity); - reportBuilder.addSkipped.callsFake(_.identity); + reportBuilder.addTestResult.callsFake(_.identity); sandbox.stub(GuiReportBuilder, 'create').returns(reportBuilder); - sandbox.stub(reportBuilder, 'imageHandler').value({saveTestImages: sinon.stub()}); sandbox.stub(HermioneTestAdapter.prototype, 'id').value('some-id'); client = new EventEmitter(); @@ -64,7 +60,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { const testResult = mkHermioneTestResult(); const mediator = sinon.spy().named('mediator'); - reportBuilder.addError.callsFake(() => Promise.delay(100).then(mediator).then(() => ({id: 'some-id'}))); + reportBuilder.addTestResult.callsFake(() => Promise.delay(100).then(mediator).then(() => ({id: 'some-id'}))); subscribeOnToolEvents(hermione, reportBuilder, client); hermione.emit(hermione.events.TEST_FAIL, testResult); @@ -79,7 +75,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { const hermione = mkHermione_(); const testResult = mkHermioneTestResult(); - reportBuilder.addRunning.resolves({id: 'some-id'}); + reportBuilder.addTestResult.resolves({id: 'some-id'}); reportBuilder.getTestBranch.withArgs('some-id').returns('test-tree-branch'); subscribeOnToolEvents(hermione, reportBuilder, client); @@ -99,7 +95,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { await hermione.emitAsync(hermione.events.TEST_PENDING, testResult); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.calledOnceWith(reportBuilder.addSkipped, sinon.match({ + assert.calledOnceWith(reportBuilder.addTestResult, sinon.match({ fullName: 'some-title', browserId: 'some-browser', attempt: UNKNOWN_ATTEMPT diff --git a/test/unit/lib/history-utils.js b/test/unit/lib/history-utils.js index d50622157..ed3a28285 100644 --- a/test/unit/lib/history-utils.js +++ b/test/unit/lib/history-utils.js @@ -64,10 +64,10 @@ describe('history-utils', () => { ]); }); - it('should return undefined if all history is not given', () => { + it('should return empty array if all history is not given', () => { const history = getCommandsHistory(undefined); - assert.isUndefined(history); + assert.deepEqual(history, []); }); }); }); diff --git a/test/unit/lib/local-images-saver.js b/test/unit/lib/local-image-file-saver.js similarity index 90% rename from test/unit/lib/local-images-saver.js rename to test/unit/lib/local-image-file-saver.js index 6d25e6dd6..db6945e38 100644 --- a/test/unit/lib/local-images-saver.js +++ b/test/unit/lib/local-image-file-saver.js @@ -12,9 +12,9 @@ describe('local-images-saver', () => { utils = _.clone(originalUtils); sandbox.stub(utils, 'copyFileAsync'); - imagesSaver = proxyquire('lib/local-images-saver', { + imagesSaver = proxyquire('lib/local-image-file-saver', { './server-utils': utils - }).LocalImagesSaver; + }).LocalImageFileSaver; }); afterEach(() => sandbox.restore()); diff --git a/test/unit/lib/merge-reports/index.js b/test/unit/lib/merge-reports/index.js index 15366049e..82cba7640 100644 --- a/test/unit/lib/merge-reports/index.js +++ b/test/unit/lib/merge-reports/index.js @@ -25,6 +25,8 @@ describe('lib/merge-reports', () => { htmlReporter = sinon.stub(); htmlReporter.events = {REPORT_SAVED: 'reportSaved'}; htmlReporter.emitAsync = sinon.stub(); + htmlReporter.imagesSaver = {saveImg: sinon.stub()}; + htmlReporter.config = {}; mergeReports = proxyquire('lib/merge-reports', { '../server-utils': serverUtils, diff --git a/test/unit/lib/report-builder/gui.js b/test/unit/lib/report-builder/gui.js index 0ac3487a2..8d773e23e 100644 --- a/test/unit/lib/report-builder/gui.js +++ b/test/unit/lib/report-builder/gui.js @@ -4,22 +4,22 @@ const fs = require('fs-extra'); const _ = require('lodash'); const proxyquire = require('proxyquire'); const serverUtils = require('lib/server-utils'); -const {HermioneTestAdapter} = require('lib/test-adapter'); +const {HermioneTestAdapter} = require('lib/test-adapter/hermione'); const {SqliteClient} = require('lib/sqlite-client'); const {GuiTestsTreeBuilder} = require('lib/tests-tree-builder/gui'); const {HtmlReporter} = require('lib/plugin-api'); -const {SUCCESS, FAIL, ERROR, SKIPPED, IDLE, RUNNING, UPDATED} = require('lib/constants/test-statuses'); +const {FAIL, UPDATED} = require('lib/constants/test-statuses'); const {LOCAL_DATABASE_NAME} = require('lib/constants/database'); -const {TestAttemptManager} = require('lib/test-attempt-manager'); -const {ImageDiffError} = require('../../utils'); const {ImagesInfoSaver} = require('lib/images-info-saver'); +const sinon = require('sinon'); +const {SKIPPED, SUCCESS, ERROR} = require('lib/constants'); const TEST_REPORT_PATH = 'test'; const TEST_DB_PATH = `${TEST_REPORT_PATH}/${LOCAL_DATABASE_NAME}`; describe('GuiReportBuilder', () => { const sandbox = sinon.sandbox.create(); - let hasImage, deleteFile, GuiReportBuilder, dbClient, testAttemptManager, copyAndUpdate; + let hasImage, deleteFile, GuiReportBuilder, dbClient, copyAndUpdate, imagesInfoSaver; const mkGuiReportBuilder_ = async ({toolConfig, pluginConfig} = {}) => { toolConfig = _.defaults(toolConfig || {}, {getAbsoluteUrl: _.noop}); @@ -36,9 +36,10 @@ describe('GuiReportBuilder', () => { HermioneTestAdapter.create = (obj) => obj; dbClient = await SqliteClient.create({htmlReporter, reportPath: TEST_REPORT_PATH}); - testAttemptManager = new TestAttemptManager(); + imagesInfoSaver = sinon.createStubInstance(ImagesInfoSaver); + imagesInfoSaver.save.callsFake(_.identity); - const reportBuilder = GuiReportBuilder.create(hermione.htmlReporter, pluginConfig, {dbClient, testAttemptManager}); + const reportBuilder = GuiReportBuilder.create(hermione.htmlReporter, pluginConfig, {dbClient, imagesInfoSaver}); const workers = {saveDiffTo: () => {}}; reportBuilder.registerWorkers(workers); @@ -120,182 +121,39 @@ describe('GuiReportBuilder', () => { sandbox.restore(); }); - describe('"addIdle" method', () => { - it(`should add "${IDLE}" status to result`, async () => { + describe('"addTestResult" method', () => { + it('should add skipped test results to skips', async () => { const reportBuilder = await mkGuiReportBuilder_(); - await reportBuilder.addIdle(stubTest_()); - - assert.equal(getTestResult_().status, IDLE); - }); - }); - - describe('"addRunning" method', () => { - it(`should add "${RUNNING}" status to result`, async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addRunning(stubTest_()); - - assert.equal(getTestResult_().status, RUNNING); - }); - }); - - describe('"addSkipped" method', () => { - it('should add skipped test to results', async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addSkipped(stubTest_({ - browserId: 'bro1', - skipReason: 'some skip comment', - fullName: 'suite-full-name' - })); - - assert.equal(getTestResult_().status, SKIPPED); - assert.deepEqual(reportBuilder.getResult().skips, [{ - suite: 'suite-full-name', - browser: 'bro1', - comment: 'some skip comment' - }]); - }); - }); - - describe('"addSuccess" method', () => { - it('should add success test to result', async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addSuccess(stubTest_({ - browserId: 'bro1' - })); - - assert.match(getTestResult_(), { - status: SUCCESS, - name: 'bro1' - }); - }); - }); - - describe('"addFail" method', () => { - it('should add failed test to result', async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addFail(stubTest_({ - browserId: 'bro1', - imageDir: 'some-image-dir' + await reportBuilder.addTestResult(stubTest_({ + status: SKIPPED, + fullName: 'some-name', + skipReason: 'some-reason', + browserId: 'some-browser' })); - assert.match(getTestResult_(), { - status: FAIL, - name: 'bro1' - }); - }); - }); - - describe('"addError" method', () => { - it('should add error test to result', async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addError(stubTest_({error: {message: 'some-error-message'}})); - - assert.match(getTestResult_(), { - status: ERROR, - error: {message: 'some-error-message'} - }); - }); - }); - - describe('"addRetry" method', () => { - it('should add "fail" status to result if test result has not equal images', async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addRetry(stubTest_({assertViewResults: [new ImageDiffError()]})); + const result = reportBuilder.getResult(); - assert.equal(getTestResult_().status, FAIL); + assert.deepEqual(result.skips[0], {suite: 'some-name', comment: 'some-reason', browser: 'some-browser'}); }); - it('should add "error" status to result if test result has no image', async () => { + it('should load images info from previous attempt', async () => { const reportBuilder = await mkGuiReportBuilder_(); + GuiTestsTreeBuilder.prototype.getImagesInfo.returns([ + {stateName: 'state-1', status: SUCCESS}, + {stateName: 'state-2', status: ERROR} + ]); - await reportBuilder.addRetry(stubTest_({assertViewResults: [{name: 'some-error-name'}]})); - - assert.equal(getTestResult_().status, ERROR); - }); - }); - - describe('"addUpdated" method', () => { - it(`should add "${UPDATED}" status to result if all images updated`, async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addUpdated(stubTest_({testPath: [], imagesInfo: [{status: UPDATED}]})); - - assert.equal(getTestResult_().status, UPDATED); - }); - - it(`should add "${UPDATED}" status even if result has errors`, async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - await reportBuilder.addUpdated(stubTest_({ - testPath: [], - error: {name: 'some-error', message: 'some-message'}, - imagesInfo: [{status: FAIL}], - attempt: 4 + copyAndUpdate.callsFake(_.assign); + const enrichedResult = await reportBuilder.addTestResult(stubTest_({ + status: UPDATED, + imagesInfo: [{stateName: 'state-2', status: UPDATED}] })); - assert.equal(getTestResult_().status, UPDATED); - }); - - it('should update test image for current state name', async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - const failedTest = stubTest_({ - imagesInfo: [ - {stateName: 'plain1', status: FAIL}, - {stateName: 'plain2', status: FAIL} - ] - }); - - const updatedTest = stubTest_({ - imagesInfo: [ - {stateName: 'plain1', status: UPDATED}, - {stateName: 'plain2', status: FAIL} - ], - testPath: [] - }); - - await reportBuilder.addFail(failedTest); - GuiTestsTreeBuilder.prototype.getImagesInfo.returns(failedTest.imagesInfo); - await reportBuilder.addUpdated(updatedTest); - - const updatedTestResult = GuiTestsTreeBuilder.prototype.addTestResult.secondCall.args[0]; - - assert.match(updatedTestResult.imagesInfo[0], {stateName: 'plain1', status: UPDATED}); - assert.match(updatedTestResult.imagesInfo[1], {stateName: 'plain2', status: FAIL}); - }); - - it('should update last test image if state name was not passed', async () => { - const reportBuilder = await mkGuiReportBuilder_(); - - const failedTest = stubTest_({ - imagesInfo: [ - {stateName: 'plain1', status: FAIL}, - {stateName: 'plain2', status: FAIL} - ] - }); - const updatedTest = stubTest_({ - id: 'result-2', - imagesInfo: [ - {status: UPDATED} - ], - testPath: [] - }); - - await reportBuilder.addFail(failedTest); - GuiTestsTreeBuilder.prototype.getImagesInfo.returns(failedTest.imagesInfo); - await reportBuilder.addUpdated(updatedTest, 'result-2'); - - const {imagesInfo} = GuiTestsTreeBuilder.prototype.addTestResult.secondCall.args[0]; - - assert.match(imagesInfo[0], {status: FAIL}); - assert.match(imagesInfo[1], {status: UPDATED}); + assert.deepEqual(enrichedResult.imagesInfo, [ + {stateName: 'state-1', status: SUCCESS}, + {stateName: 'state-2', status: UPDATED} + ]); }); }); @@ -522,33 +380,31 @@ describe('GuiReportBuilder', () => { it('"suiteUrl" field', async () => { const reportBuilder = await mkGuiReportBuilder_(); - await reportBuilder.addSuccess(stubTest_({ + await reportBuilder.addTestResult(stubTest_({ url: 'some-url' })); - assert.equal(getTestResult_().suiteUrl, 'some-url'); + assert.equal(getTestResult_().url, 'some-url'); }); it('"name" field as browser id', async () => { const reportBuilder = await mkGuiReportBuilder_(); - await reportBuilder.addSuccess(stubTest_({browserId: 'yabro'})); + await reportBuilder.addTestResult(stubTest_({browserId: 'yabro'})); - assert.equal(getTestResult_().name, 'yabro'); + assert.equal(getTestResult_().browserId, 'yabro'); }); it('"metaInfo" field', async () => { const reportBuilder = await mkGuiReportBuilder_(); - await reportBuilder.addSuccess(stubTest_({ - meta: {some: 'value', sessionId: '12345'}, - file: '/path/file.js', - url: '/test/url' + await reportBuilder.addTestResult(stubTest_({ + meta: {some: 'value', sessionId: '12345', file: '/path/file.js', url: '/test/url'} })); const expectedMetaInfo = {some: 'value', sessionId: '12345', file: '/path/file.js', url: '/test/url'}; - assert.deepEqual(getTestResult_().metaInfo, expectedMetaInfo); + assert.deepEqual(getTestResult_().meta, expectedMetaInfo); }); [ @@ -560,29 +416,11 @@ describe('GuiReportBuilder', () => { it(`add "${name}" field`, async () => { const reportBuilder = await mkGuiReportBuilder_(); - await reportBuilder.addSuccess(stubTest_({[name]: value})); + await reportBuilder.addTestResult(stubTest_({[name]: value})); assert.deepEqual(getTestResult_()[name], value); }); }); }); - - it('should pass test result without "errorDetails" if "saveErrorDetails" is not set', async () => { - const reportBuilder = await mkGuiReportBuilder_({pluginConfig: {saveErrorDetails: false}}); - const errorDetails = {title: 'some-title', filePath: 'some-path'}; - - await reportBuilder.addFail(stubTest_({errorDetails})); - - assert.isUndefined(getTestResult_().errorDetails); - }); - - it('should pass test result with "errorDetails" if "saveErrorDetails" is set', async () => { - const reportBuilder = await mkGuiReportBuilder_({pluginConfig: {saveErrorDetails: true}}); - const errorDetails = {title: 'some-title', filePath: 'some-path'}; - - await reportBuilder.addFail(stubTest_({errorDetails})); - - assert.deepEqual(getTestResult_().errorDetails, errorDetails); - }); }); }); diff --git a/test/unit/lib/report-builder/static.js b/test/unit/lib/report-builder/static.js index 49960e3ee..0f3326f5e 100644 --- a/test/unit/lib/report-builder/static.js +++ b/test/unit/lib/report-builder/static.js @@ -4,20 +4,18 @@ const fsOriginal = require('fs-extra'); const _ = require('lodash'); const Database = require('better-sqlite3'); const proxyquire = require('proxyquire'); +const sinon = require('sinon'); const {HtmlReporter} = require('lib/plugin-api'); -const {SUCCESS, FAIL, ERROR, SKIPPED} = require('lib/constants/test-statuses'); +const {ERROR} = require('lib/constants/test-statuses'); const {LOCAL_DATABASE_NAME} = require('lib/constants/database'); const {SqliteClient} = require('lib/sqlite-client'); -const {NoRefImageError, ImageDiffError} = require('../../utils'); -const sinon = require('sinon'); -const path = require('path'); const TEST_REPORT_PATH = 'test'; const TEST_DB_PATH = `${TEST_REPORT_PATH}/${LOCAL_DATABASE_NAME}`; describe('StaticReportBuilder', () => { const sandbox = sinon.sandbox.create(); - let StaticReportBuilder, htmlReporter, dbClient; + let StaticReportBuilder, htmlReporter, dbClient, workers, imagesInfoSaver; let cacheExpectedPaths = new Map(), cacheAllImages = new Map(), cacheDiffImages = new Map(); const fs = _.clone(fsOriginal); @@ -36,7 +34,7 @@ describe('StaticReportBuilder', () => { './server-utils': utils }); - const mkStaticReportBuilder_ = async ({pluginConfig, workers} = {}) => { + const mkStaticReportBuilder_ = async ({pluginConfig} = {}) => { pluginConfig = _.defaults(pluginConfig, {baseHost: '', path: TEST_REPORT_PATH, baseTestPath: ''}); htmlReporter = _.extend(HtmlReporter.create({baseHost: ''}), { @@ -47,9 +45,12 @@ describe('StaticReportBuilder', () => { }); dbClient = await SqliteClient.create({htmlReporter, reportPath: TEST_REPORT_PATH}); + imagesInfoSaver = sinon.createStubInstance(ImagesInfoSaver); - const reportBuilder = StaticReportBuilder.create(htmlReporter, pluginConfig, {dbClient}); - workers = workers ?? {saveDiffTo: () => {}}; + imagesInfoSaver.save.callsFake(_.identity); + + const reportBuilder = StaticReportBuilder.create(htmlReporter, pluginConfig, {dbClient, imagesInfoSaver}); + workers = {saveDiffTo: sinon.stub()}; reportBuilder.registerWorkers(workers); @@ -57,18 +58,14 @@ describe('StaticReportBuilder', () => { }; const stubTest_ = (opts = {}) => { - const {imagesInfo = []} = opts; - return _.defaultsDeep(opts, { state: {name: 'name-default'}, imageDir: '', - imagesInfo + imagesInfo: [] }); }; beforeEach(() => { - sandbox.stub(utils, 'hasImage').returns(true); - StaticReportBuilder = proxyquire('lib/report-builder/static', { 'fs-extra': fs, '../server-utils': utils, @@ -92,38 +89,8 @@ describe('StaticReportBuilder', () => { reportBuilder = await mkStaticReportBuilder_(); }); - it('should add skipped test', async () => { - await reportBuilder.addSkipped(stubTest_({status: SKIPPED})); - const db = new Database(TEST_DB_PATH); - - const [{status}] = db.prepare('SELECT * from suites').all(); - db.close(); - - assert.equal(status, SKIPPED); - }); - - it('should add success test', async () => { - await reportBuilder.addSuccess(stubTest_({status: SUCCESS})); - const db = new Database(TEST_DB_PATH); - - const [{status}] = db.prepare('SELECT * from suites').all(); - db.close(); - - assert.equal(status, SUCCESS); - }); - - it('should add failed test', async () => { - await reportBuilder.addFail(stubTest_({status: FAIL})); - const db = new Database(TEST_DB_PATH); - - const [{status}] = db.prepare('SELECT * from suites').all(); - db.close(); - - assert.equal(status, FAIL); - }); - - it('should add error test', async () => { - await reportBuilder.addError(stubTest_({status: ERROR})); + it('should use test results status', async () => { + await reportBuilder.addTestResult(stubTest_({status: ERROR})); const db = new Database(TEST_DB_PATH); const [{status}] = db.prepare('SELECT * from suites').all(); @@ -133,7 +100,7 @@ describe('StaticReportBuilder', () => { }); it('should use timestamp from test result when it is present', async () => { - await reportBuilder.addSuccess(stubTest_({timestamp: 100500})); + await reportBuilder.addTestResult(stubTest_({timestamp: 100500})); const db = new Database(TEST_DB_PATH); const [{timestamp}] = db.prepare('SELECT * from suites').all(); @@ -143,7 +110,7 @@ describe('StaticReportBuilder', () => { }); it('should use some current timestamp when test result misses one', async () => { - await reportBuilder.addSuccess(stubTest_()); + await reportBuilder.addTestResult(stubTest_()); const db = new Database(TEST_DB_PATH); const [{timestamp}] = db.prepare('SELECT * from suites').all(); @@ -154,117 +121,46 @@ describe('StaticReportBuilder', () => { }); describe('saving images', () => { - let reportBuilder, saveDiffTo; + let reportBuilder; beforeEach(async () => { - saveDiffTo = sinon.stub(); - - reportBuilder = await mkStaticReportBuilder_({workers: {saveDiffTo}}); - - sandbox.stub(utils, 'getReferencePath'); - sandbox.stub(utils, 'getCurrentPath'); - sandbox.stub(utils, 'getDiffPath').returns('diff'); - sandbox.stub(utils, 'copyFileAsync'); - sandbox.stub(utils, 'makeDirFor'); - - sandbox.stub(fs, 'readFile').resolves(Buffer.from('')); - }); - - it('should save image from passed test', async () => { - utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); - - await reportBuilder.addSuccess(stubTest_({assertViewResults: [{refImg: {path: 'ref/path'}, stateName: 'plain'}]})); - - assert.calledOnceWith(utils.copyFileAsync, 'ref/path', 'report/plain', {reportDir: 'test'}); - }); - - it('should save image from assert view error', async () => { - utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); - - const err = new NoRefImageError(); - err.stateName = 'plain'; - err.currImg = {path: 'current/path'}; - - await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); - - assert.calledOnceWith(utils.copyFileAsync, 'current/path', 'report/plain', {reportDir: 'test'}); - }); - - it('should save reference image from assert view fail', async () => { - utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); - - const err = new ImageDiffError(); - err.stateName = 'plain'; - err.refImg = {path: 'reference/path'}; - - await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); - - assert.calledWith(utils.copyFileAsync, 'reference/path', 'report/plain', {reportDir: 'test'}); - }); - - it('should save current image from assert view fail', async () => { - utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); - - const err = new ImageDiffError(); - err.stateName = 'plain'; - err.currImg = {path: 'current/path'}; - - await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); - - assert.calledWith(utils.copyFileAsync, 'current/path', 'report/plain', {reportDir: 'test'}); + reportBuilder = await mkStaticReportBuilder_(); }); - it('should save current diff image from assert view fail', async () => { - fs.readFile.resolves(Buffer.from('some-buff')); - utils.getDiffPath.callsFake(({stateName}) => `report/${stateName}`); + it('should use images info saver to save images', async () => { + const testResult = stubTest_({assertViewResults: [{refImg: {path: 'ref/path'}, stateName: 'plain'}]}); - const err = new ImageDiffError(); - err.stateName = 'plain'; + await reportBuilder.addTestResult(testResult); - await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); - - assert.calledWith( - saveDiffTo, sinon.match.instanceOf(ImageDiffError), sinon.match('/report/plain') - ); + assert.calledOnceWith(imagesInfoSaver.save, testResult, workers); }); }); describe('saving error details', () => { - let reportBuilder; - beforeEach(async () => { - reportBuilder = await mkStaticReportBuilder_({pluginConfig: {saveErrorDetails: true}}); - - sandbox.stub(utils, 'makeDirFor').resolves(); - sandbox.stub(utils, 'getDetailsFileName').returns('md5-bro-n-time'); - - sandbox.stub(fs, 'writeFile'); - sandbox.stub(fs, 'mkdirs'); + sandbox.stub(utils, 'saveErrorDetails').resolves(); }); - it('should do nothing if no error details are available', async () => { - await reportBuilder.addFail(stubTest_()); - - assert.notCalled(fs.writeFile); - }); + it('should not save error details if turned off', async () => { + const reportBuilder = await mkStaticReportBuilder_({ + pluginConfig: {saveErrorDetails: false, path: TEST_REPORT_PATH} + }); + const testResult = stubTest_({errorDetails: {filePath: 'some-path'}}); - it('should save error details to correct path', async () => { - await reportBuilder.addFail(stubTest_({errorDetails: {filePath: 'some-path'}})); + await reportBuilder.addTestResult(testResult); - assert.calledWithMatch(fs.writeFile, path.resolve(`${TEST_REPORT_PATH}/some-path`), sinon.match.any); + assert.notCalled(utils.saveErrorDetails); }); - it('should create directory for error details', async () => { - await reportBuilder.addFail(stubTest_({errorDetails: {filePath: `some-dir/some-path`}})); - - assert.calledOnceWith(fs.mkdirs, path.resolve(TEST_REPORT_PATH, 'some-dir')); - }); + it('should use server-utils to save error details if needed', async () => { + const reportBuilder = await mkStaticReportBuilder_({ + pluginConfig: {saveErrorDetails: true, path: TEST_REPORT_PATH} + }); + const testResult = stubTest_({errorDetails: {filePath: 'some-path'}}); - it('should save error details', async () => { - const data = {foo: 'bar'}; - await reportBuilder.addFail(stubTest_({errorDetails: {filePath: 'some-path', data}})); + await reportBuilder.addTestResult(testResult); - assert.calledWith(fs.writeFile, sinon.match.any, JSON.stringify(data, null, 2)); + assert.calledOnceWith(utils.saveErrorDetails, testResult, TEST_REPORT_PATH); }); }); diff --git a/test/unit/lib/server-utils.js b/test/unit/lib/server-utils.js index 50f680c50..efdb98374 100644 --- a/test/unit/lib/server-utils.js +++ b/test/unit/lib/server-utils.js @@ -1,15 +1,24 @@ 'use strict'; const path = require('path'); -const fs = require('fs-extra'); + const Promise = require('bluebird'); -const utils = require('lib/server-utils'); +const _ = require('lodash'); +const proxyquire = require('proxyquire'); +const sinon = require('sinon'); const {IMAGES_PATH} = require('lib/constants/paths'); const testStatuses = require('lib/constants/test-statuses'); describe('server-utils', () => { const sandbox = sinon.sandbox.create(); + const fsOriginal = require('fs-extra'); + const fs = _.clone(fsOriginal); + + const utils = proxyquire('lib/server-utils', { + 'fs-extra': fs + }); + afterEach(() => sandbox.restore()); [ @@ -44,11 +53,11 @@ describe('server-utils', () => { it('should add state name to the path if it was passed', () => { const test = { imageDir: 'some/dir', - browserId: 'bro', - stateName: 'plain' + browserId: 'bro' }; + const stateName = 'plain'; - const resultPath = utils[`get${testData.name}Path`](test); + const resultPath = utils[`get${testData.name}Path`](test, stateName); assert.equal(resultPath, path.join(IMAGES_PATH, 'some', 'dir', `plain/bro~${testData.prefix}_0.png`)); }); @@ -121,13 +130,43 @@ describe('server-utils', () => { }); }); - describe('getDetailsFileName', () => { - it('should compose correct file name from suite path, browser id and attempt', () => { - sandbox.stub(Date, 'now').returns('123456789'); - const testId = 'abcdef'; - const expected = `${testId}-bro_2_123456789.json`; + describe('saveErrorDetails', () => { + const TEST_REPORT_PATH = 'report-path'; + + beforeEach(() => { + sandbox.stub(fs, 'writeFile').resolves(); + sandbox.stub(fs, 'mkdirs').resolves(); + }); + + it('should do nothing if no error details are available', async () => { + await utils.saveErrorDetails({}, ''); + + assert.notCalled(fs.writeFile); + }); + + it('should save error details to correct path', async () => { + const testResult = {errorDetails: {filePath: 'some-path'}}; + + await utils.saveErrorDetails(testResult, TEST_REPORT_PATH); + + assert.calledWithMatch(fs.writeFile, path.resolve(`${TEST_REPORT_PATH}/some-path`), sinon.match.any); + }); + + it('should create directory for error details', async () => { + const testResult = {errorDetails: {filePath: `some-dir/some-path`}}; + + await utils.saveErrorDetails(testResult, TEST_REPORT_PATH); + + assert.calledOnceWith(fs.mkdirs, path.resolve(TEST_REPORT_PATH, 'some-dir')); + }); + + it('should save error details', async () => { + const data = {foo: 'bar'}; + const testResult = {errorDetails: {filePath: 'some-path', data}}; + + await utils.saveErrorDetails(testResult, TEST_REPORT_PATH); - assert.equal(utils.getDetailsFileName(testId, 'bro', 1), expected); + assert.calledWith(fs.writeFile, sinon.match.any, JSON.stringify(data, null, 2)); }); }); diff --git a/test/unit/lib/test-adapter/hermione.ts b/test/unit/lib/test-adapter/hermione.ts index cf31927d3..1662974cc 100644 --- a/test/unit/lib/test-adapter/hermione.ts +++ b/test/unit/lib/test-adapter/hermione.ts @@ -6,7 +6,8 @@ import tmpOriginal from 'tmp'; import {TestStatus} from 'lib/constants/test-statuses'; import {ERROR_DETAILS_PATH} from 'lib/constants/paths'; -import {HermioneTestAdapter, HermioneTestAdapterOptions, ReporterTestResult} from 'lib/test-adapter'; +import {ReporterTestResult} from 'lib/test-adapter'; +import {HermioneTestAdapter, HermioneTestAdapterOptions} from 'lib/test-adapter/hermione'; import {HermioneTestResult} from 'lib/types'; import * as originalUtils from 'lib/server-utils'; import * as originalCommonUtils from 'lib/common-utils'; diff --git a/test/unit/lib/test-adapter/playwright.ts b/test/unit/lib/test-adapter/playwright.ts index 3102301bc..b25d0dbd0 100644 --- a/test/unit/lib/test-adapter/playwright.ts +++ b/test/unit/lib/test-adapter/playwright.ts @@ -57,115 +57,12 @@ describe('PlaywrightTestAdapter', () => { sandbox.restore(); }); - describe('assertViewResults', () => { - it('should return an IMAGE_DIFF result when error is IMAGE_DIFF and all images are present', () => { - const testCaseStub = mkTestCase(); - const testResultStub = mkTestResult(); - const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions()); - - const results = adapter.assertViewResults as ImageDiffError[]; - - assert.lengthOf(results, 1); - assert.strictEqual(results[0].name, ErrorName.IMAGE_DIFF); - assert.strictEqual(results[0].stateName, 'state1'); - assert.strictEqual(results[0].refImg?.path, 'state1' + ImageTitleEnding.Expected); - assert.strictEqual(results[0].diffImg?.path, 'state1' + ImageTitleEnding.Diff); - assert.strictEqual(results[0].currImg?.path, 'state1' + ImageTitleEnding.Actual); - }); - - it('should return a NO_REF_IMAGE result when error is NO_REF_IMAGE and only actual image is present', () => { - const testCaseStub = mkTestCase(); - const testResultStub = mkTestResult({ - attachments: [createAttachment('state1' + ImageTitleEnding.Actual)], - errors: [{name: ErrorName.NO_REF_IMAGE, message: 'snapshot doesn\'t exist: some.png.', stack: 'error-stack'}] as any - }); - const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions()); - - const results = adapter.assertViewResults as NoRefImageError[]; - - assert.lengthOf(results, 1); - assert.strictEqual(results[0].name, ErrorName.NO_REF_IMAGE); - assert.strictEqual(results[0].stateName, 'state1'); - assert.strictEqual(results[0].currImg?.path, 'state1' + ImageTitleEnding.Actual); - }); - - it('should have diff clusters', () => { - const testCaseStub = mkTestCase(); - const testResultStub = mkTestResult({ - errors: [{message: 'Screenshot comparison failed', meta: { - type: 'ImageDiffError', - snapshotName: 'state1.png', - diffClusters: [{left: 0, top: 0, right: 1, bottom: 1}] - }} as PwtImageDiffError] - }); - - const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions()); - - const results = adapter.assertViewResults as ImageDiffError[]; - - assert.deepEqual(results[0].diffClusters, [{left: 0, top: 0, right: 1, bottom: 1}]); - }); - - it('should detect multiple different errors from toMatchScreenshot', () => { - const testCaseStub = mkTestCase(); - const testResultStub = mkTestResult({ - attachments: [ - createAttachment('state1' + ImageTitleEnding.Expected), - createAttachment('state1' + ImageTitleEnding.Diff), - createAttachment('state1' + ImageTitleEnding.Actual), - createAttachment('state2' + ImageTitleEnding.Actual) - ], - errors: [ - { - message: 'Screenshot comparison failed', - meta: { - type: 'ImageDiffError', - snapshotName: 'state1.png', - diffClusters: [{left: 0, top: 0, right: 1, bottom: 1}] - } - } as PwtImageDiffError, - { - message: '', - meta: { - type: 'NoRefImageError', - snapshotName: 'state2.png' - } - } as PwtNoRefImageError - ] - }); - - const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions()); - - const results = adapter.assertViewResults as ImageDiffError[]; - - assert.deepEqual(results[0].name, 'ImageDiffError'); - assert.deepEqual(results[1].name, 'NoRefImageError'); - }); - - it('should return refImg, if provided', () => { - const testCaseStub = mkTestCase(); - const testResultStub = { - status: 'success', - attachments: [createAttachment('state1' + ImageTitleEnding.Expected)], - steps: [] - } as unknown as TestResult; - const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions()); - - const results = adapter.assertViewResults as ImageDiffError[]; - - assert.lengthOf(results, 1); - assert.isUndefined(results[0].name); - assert.strictEqual(results[0].stateName, 'state1'); - assert.strictEqual(results[0].refImg?.path, 'state1' + ImageTitleEnding.Expected); - }); - }); - describe('attempt', () => { it('should return suite attempt', () => { // eslint-disable-next-line no-new - new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); - const adapter2 = new PlaywrightTestAdapter(mkTestCase({titlePath: sinon.stub().returns(['another-title'])}), mkTestResult(), mkAdapterOptions()); - const adapter3 = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); + new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter2 = new PlaywrightTestAdapter(mkTestCase({titlePath: sinon.stub().returns(['another-title'])}), mkTestResult()); + const adapter3 = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); assert.equal(adapter3.attempt, 1); assert.equal(adapter2.attempt, 0); @@ -175,8 +72,8 @@ describe('PlaywrightTestAdapter', () => { const testResult = mkTestResult({status: 'skipped'}); // eslint-disable-next-line no-new - new PlaywrightTestAdapter(mkTestCase(), testResult, mkAdapterOptions()); - const adapter2 = new PlaywrightTestAdapter(mkTestCase(), testResult, mkAdapterOptions()); + new PlaywrightTestAdapter(mkTestCase(), testResult); + const adapter2 = new PlaywrightTestAdapter(mkTestCase(), testResult); assert.equal(adapter2.attempt, 0); }); @@ -184,7 +81,7 @@ describe('PlaywrightTestAdapter', () => { describe('browserId', () => { it('should return browserId', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); assert.equal(adapter.browserId, 'some-browser'); }); @@ -192,7 +89,7 @@ describe('PlaywrightTestAdapter', () => { describe('error', () => { it('should return undefined if there are no errors', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors: []}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors: []})); const {error} = adapter; @@ -202,7 +99,7 @@ describe('PlaywrightTestAdapter', () => { it('should return an error with name NO_REF_IMAGE for snapshot missing errors', () => { const errorMessage = 'A snapshot doesn\'t exist: image-name.png.'; const errors = [{message: errorMessage}]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); const {error} = adapter; @@ -213,7 +110,7 @@ describe('PlaywrightTestAdapter', () => { it('should return an error with name IMAGE_DIFF for screenshot comparison failures', () => { const errorMessage = 'Screenshot comparison failed'; const errors = [{message: errorMessage}]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); const {error} = adapter; @@ -225,7 +122,7 @@ describe('PlaywrightTestAdapter', () => { const errorMessage = 'Some error occurred'; const errorStack = 'Error: Some error occurred at some-file.ts:10:15'; const errors = [{message: errorMessage, stack: errorStack}]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); const {error} = adapter; @@ -237,7 +134,7 @@ describe('PlaywrightTestAdapter', () => { {message: 'First error', stack: 'Error: First error at some-file.ts:5:10'}, {message: 'Second error', stack: 'Error: Second error at another-file.ts:15:20'} ]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); const expectedMessage = JSON.stringify(errors.map(err => err.message)); const expectedStack = JSON.stringify(errors.map(err => err.stack)); @@ -250,7 +147,7 @@ describe('PlaywrightTestAdapter', () => { describe('file', () => { it('should return file path', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); assert.strictEqual(adapter.file, 'test-file-path'); }); @@ -258,7 +155,7 @@ describe('PlaywrightTestAdapter', () => { describe('fullName', () => { it('should return fullName', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); assert.strictEqual(adapter.fullName, 'describe › test'); }); @@ -270,7 +167,7 @@ describe('PlaywrightTestAdapter', () => { {title: 'Step1', duration: 100}, {title: 'Step2', duration: 200} ]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({steps} as any), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({steps} as any)); const expectedHistory = ['Step1 <- 100ms\n', 'Step2 <- 200ms\n']; assert.deepEqual(adapter.history, expectedHistory); @@ -279,7 +176,7 @@ describe('PlaywrightTestAdapter', () => { describe('id', () => { it('should return id', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); assert.strictEqual(adapter.id, 'describe test some-browser 0'); }); @@ -287,7 +184,7 @@ describe('PlaywrightTestAdapter', () => { describe('imageDir', () => { it('should return imageDir', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); assert.strictEqual(adapter.imageDir, '4050de5'); }); @@ -307,31 +204,31 @@ describe('PlaywrightTestAdapter', () => { describe('status', () => { it('should return SUCCESS for PASSED PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.PASSED}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.PASSED})); assert.equal(adapter.status, TestStatus.SUCCESS); }); it('should return FAIL for FAILED PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.FAILED}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.FAILED})); assert.equal(adapter.status, TestStatus.FAIL); }); it('should return FAIL for TIMED_OUT PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.TIMED_OUT}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.TIMED_OUT})); assert.equal(adapter.status, TestStatus.FAIL); }); it('should return FAIL for INTERRUPTED PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.INTERRUPTED}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.INTERRUPTED})); assert.equal(adapter.status, TestStatus.FAIL); }); it('should return SKIPPED for any other PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.SKIPPED}), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.SKIPPED})); assert.equal(adapter.status, TestStatus.SKIPPED); }); @@ -339,7 +236,7 @@ describe('PlaywrightTestAdapter', () => { describe('testPath', () => { it('should return testPath', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); assert.deepEqual(adapter.testPath, ['describe', 'test']); }); diff --git a/test/unit/lib/tests-tree-builder/base.js b/test/unit/lib/tests-tree-builder/base.js index fb0ce8e36..25365f4e3 100644 --- a/test/unit/lib/tests-tree-builder/base.js +++ b/test/unit/lib/tests-tree-builder/base.js @@ -10,15 +10,12 @@ describe('ResultsTreeBuilder', () => { const sandbox = sinon.sandbox.create(); let ResultsTreeBuilder, builder, determineFinalStatus; - const mkTestResult_ = (result) => { - return _.defaults(result, {imagesInfo: [], metaInfo: {}}); - }; - const mkFormattedResult_ = (result) => { return _.defaults(result, { testPath: ['default-parent-suite', 'default-child-suite'], browserId: 'default-browser', - attempt: 0 + attempt: 0, + meta: {browserVersion: BrowserVersions.UNKNOWN} }); }; @@ -35,8 +32,8 @@ describe('ResultsTreeBuilder', () => { describe('"sortTree" method', () => { it('should sort ids of root suites', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s2']})); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s2']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1']})); builder.sortTree(); @@ -44,8 +41,8 @@ describe('ResultsTreeBuilder', () => { }); it('should sort ids of child suites', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1', 'ch2']})); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1', 'ch1']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1', 'ch2']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1', 'ch1']})); builder.sortTree(); @@ -53,8 +50,8 @@ describe('ResultsTreeBuilder', () => { }); it('should sort ids of browsers', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b2'})); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1'})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1'], browserId: 'b2'})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1'], browserId: 'b1'})); builder.sortTree(); @@ -65,21 +62,21 @@ describe('ResultsTreeBuilder', () => { describe('"addTestResult" method', () => { describe('"suites" field in the tree', () => { it('should collect all suite root ids', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1', 's2']})); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s3', 's4']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1', 's2']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s3', 's4']})); assert.deepEqual(builder.tree.suites.allRootIds, ['s1', 's3']); }); it('should collect all suite ids', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1', 's2']})); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s3', 's4']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1', 's2']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s3', 's4']})); assert.deepEqual(builder.tree.suites.allIds, ['s1', 's1 s2', 's3', 's3 s4']); }); it('should correctly init root suite', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1', 's2']})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1', 's2']})); assert.deepEqual( builder.tree.suites.byId['s1'], @@ -96,7 +93,7 @@ describe('ResultsTreeBuilder', () => { }); it('should correctly init child suite', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1', 's2'], browserId: 'b1'})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1', 's2'], browserId: 'b1'})); assert.deepEqual( builder.tree.suites.byId['s1 s2'], @@ -115,14 +112,14 @@ describe('ResultsTreeBuilder', () => { describe('"browsers" field in the tree', () => { it('should collect all browser ids', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1'})); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({testPath: ['s2'], browserId: 'b2'})); + builder.addTestResult(mkFormattedResult_({testPath: ['s1'], browserId: 'b1'})); + builder.addTestResult(mkFormattedResult_({testPath: ['s2'], browserId: 'b2'})); assert.deepEqual(builder.tree.browsers.allIds, ['s1 b1', 's2 b2']); }); it('should correctly init browser', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({ + builder.addTestResult(mkFormattedResult_({ testPath: ['s1'], browserId: 'b1', attempt: 0 })); @@ -139,24 +136,21 @@ describe('ResultsTreeBuilder', () => { }); it('should collect all browser versions from results in browser', () => { - const result1 = mkTestResult_({metaInfo: {browserVersion: '1'}}); - const result2 = mkTestResult_({metaInfo: {browserVersion: '1'}}); - - builder.addTestResult(result1, mkFormattedResult_({ - testPath: ['s1'], browserId: 'b1', attempt: 0 + builder.addTestResult(mkFormattedResult_({ + testPath: ['s1'], browserId: 'b1', attempt: 0, meta: {browserVersion: '1'} })); - builder.addTestResult(result2, mkFormattedResult_({ - testPath: ['s1'], browserId: 'b1', attempt: 1 + builder.addTestResult(mkFormattedResult_({ + testPath: ['s1'], browserId: 'b1', attempt: 1, meta: {browserVersion: '1'} })); assert.deepEqual(builder.tree.browsers.byId['s1 b1'].version, '1'); }); it('should collect all ids to test results in browser', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({ + builder.addTestResult(mkFormattedResult_({ testPath: ['s1'], browserId: 'b1', attempt: 0 })); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({ + builder.addTestResult(mkFormattedResult_({ testPath: ['s1'], browserId: 'b1', attempt: 1 })); @@ -166,10 +160,10 @@ describe('ResultsTreeBuilder', () => { describe('"results" field in the tree', () => { it('should collect all test result ids', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_({ + builder.addTestResult(mkFormattedResult_({ testPath: ['s1'], browserId: 'b1', attempt: 0 })); - builder.addTestResult(mkTestResult_(), mkFormattedResult_({ + builder.addTestResult(mkFormattedResult_({ testPath: ['s2'], browserId: 'b2', attempt: 0 })); @@ -177,26 +171,26 @@ describe('ResultsTreeBuilder', () => { }); it('should correctly init test result', () => { - builder.addTestResult(mkTestResult_(), mkFormattedResult_( + builder.addTestResult(mkFormattedResult_( {testPath: ['s1'], browserId: 'b1', attempt: 0} )); assert.deepEqual( - builder.tree.results.byId['s1 b1 0'], + _.pick(builder.tree.results.byId['s1 b1 0'], ['attempt', 'id', 'parentId', 'imageIds', 'metaInfo']), { attempt: 0, id: 's1 b1 0', parentId: 's1 b1', imageIds: [], - metaInfo: {} + metaInfo: {browserVersion: BrowserVersions.UNKNOWN} } ); }); it('should collect all ids to images in test result', () => { + const imagesInfo = [{stateName: 'img1'}, {stateName: 'img2'}]; builder.addTestResult( - mkTestResult_({imagesInfo: [{stateName: 'img1'}, {stateName: 'img2'}]}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0, imagesInfo}) ); assert.deepEqual( @@ -208,13 +202,13 @@ describe('ResultsTreeBuilder', () => { describe('"images" field in the tree', () => { it('should collect all images ids', () => { + const imagesInfo1 = [{stateName: 'img1'}]; builder.addTestResult( - mkTestResult_({imagesInfo: [{stateName: 'img1'}]}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0, imagesInfo: imagesInfo1}) ); + const imagesInfo2 = [{status: ERROR}]; builder.addTestResult( - mkTestResult_({imagesInfo: [{status: ERROR}]}), - mkFormattedResult_({testPath: ['s2'], browserId: 'b2', attempt: 0}) + mkFormattedResult_({testPath: ['s2'], browserId: 'b2', attempt: 0, imagesInfo: imagesInfo2}) ); assert.deepEqual(builder.tree.images.allIds, ['s1 b1 0 img1', `s2 b2 0 ${ERROR}_0`]); @@ -223,8 +217,7 @@ describe('ResultsTreeBuilder', () => { it('should correctly init image results', () => { const imagesInfo = [{stateName: 'img1', foo: 'bar'}, {status: ERROR, bar: 'baz'}]; builder.addTestResult( - mkTestResult_({imagesInfo}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0, imagesInfo}) ); assert.deepEqual( @@ -249,8 +242,7 @@ describe('ResultsTreeBuilder', () => { describe('determine statuses for suites', () => { it('should call "determineFinalStatus" with test result status', () => { builder.addTestResult( - mkTestResult_({status: SUCCESS}), - mkFormattedResult_({testPath: ['s1']}) + mkFormattedResult_({status: SUCCESS, testPath: ['s1']}) ); assert.calledOnceWith(determineFinalStatus, [SUCCESS]); @@ -258,12 +250,10 @@ describe('ResultsTreeBuilder', () => { it('should call "determineFinalStatus" with test result status from last attempt', () => { builder.addTestResult( - mkTestResult_({status: FAIL}), - mkFormattedResult_({testPath: ['s1'], attempt: 0}) + mkFormattedResult_({status: FAIL, testPath: ['s1'], attempt: 0}) ); builder.addTestResult( - mkTestResult_({status: SUCCESS}), - mkFormattedResult_({testPath: ['s1'], attempt: 1}) + mkFormattedResult_({status: SUCCESS, testPath: ['s1'], attempt: 1}) ); assert.calledWith(determineFinalStatus.lastCall, [SUCCESS]); @@ -271,12 +261,10 @@ describe('ResultsTreeBuilder', () => { it('should call "determineFinalStatus" with all test statuses from each browser', () => { builder.addTestResult( - mkTestResult_({status: FAIL}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1'}) + mkFormattedResult_({status: FAIL, testPath: ['s1'], browserId: 'b1'}) ); builder.addTestResult( - mkTestResult_({status: SUCCESS}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b2'}) + mkFormattedResult_({status: SUCCESS, testPath: ['s1'], browserId: 'b2'}) ); assert.calledWith(determineFinalStatus.secondCall, [FAIL, SUCCESS]); @@ -286,12 +274,10 @@ describe('ResultsTreeBuilder', () => { determineFinalStatus.withArgs([FAIL]).returns('s1 s2 status'); determineFinalStatus.withArgs([ERROR]).returns('s1 s3 status'); builder.addTestResult( - mkTestResult_({status: FAIL}), - mkFormattedResult_({testPath: ['s1', 's2']}) + mkFormattedResult_({status: FAIL, testPath: ['s1', 's2']}) ); builder.addTestResult( - mkTestResult_({status: ERROR}), - mkFormattedResult_({testPath: ['s1', 's3']}) + mkFormattedResult_({status: ERROR, testPath: ['s1', 's3']}) ); assert.calledWith(determineFinalStatus.getCall(3), ['s1 s2 status', 's1 s3 status']); diff --git a/test/unit/lib/tests-tree-builder/gui.js b/test/unit/lib/tests-tree-builder/gui.js index e20961a1d..839e5832f 100644 --- a/test/unit/lib/tests-tree-builder/gui.js +++ b/test/unit/lib/tests-tree-builder/gui.js @@ -8,15 +8,12 @@ const {ToolName} = require('lib/constants'); describe('GuiResultsTreeBuilder', () => { let builder; - const mkTestResult_ = (result) => { - return _.defaults(result, {status: IDLE, imagesInfo: [], metaInfo: {}}); - }; - const mkFormattedResult_ = (result) => { return _.defaults(result, { testPath: ['default-parent-suite', 'default-child-suite'], browserId: 'default-browser', - attempt: 0 + attempt: 0, + meta: {browserVersion: 'some-version'} }); }; @@ -28,9 +25,9 @@ describe('GuiResultsTreeBuilder', () => { describe('"getImagesInfo" method', () => { it('should return images from tree for passed test result id', () => { - const formattedRes = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 0}); const imagesInfo = [{stateName: 'image-1'}, {stateName: 'image-2'}]; - builder.addTestResult(mkTestResult_({imagesInfo}), formattedRes); + const formattedRes = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 0, imagesInfo}); + builder.addTestResult(formattedRes); const gotImagesInfo = builder.getImagesInfo('s b 0'); @@ -49,12 +46,10 @@ describe('GuiResultsTreeBuilder', () => { it('should not reuse browser result if browser ids are not matched', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) ); builder.addTestResult( - mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b2', attempt: 0}) ); @@ -67,12 +62,10 @@ describe('GuiResultsTreeBuilder', () => { it('should reuse browser result from the passed tree if browser ids matched', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) ); builder.addTestResult( - mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) ); @@ -86,13 +79,11 @@ describe('GuiResultsTreeBuilder', () => { it('should not reuse result if browser ids does not matched', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_({status: FAIL}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({status: FAIL, testPath: ['s1'], browserId: 'b1', attempt: 0}) ); builder.addTestResult( - mkTestResult_({status: IDLE}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b2', attempt: 0}) + mkFormattedResult_({status: IDLE, testPath: ['s1'], browserId: 'b2', attempt: 0}) ); builder.reuseTestsTree(srcBuilder.tree); @@ -104,17 +95,14 @@ describe('GuiResultsTreeBuilder', () => { it('should reuse all results from the passed tree if browser ids matched', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_({status: FAIL}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({status: FAIL, testPath: ['s1'], browserId: 'b1', attempt: 0}) ); srcBuilder.addTestResult( - mkTestResult_({status: SUCCESS}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 1}) + mkFormattedResult_({status: SUCCESS, testPath: ['s1'], browserId: 'b1', attempt: 1}) ); builder.addTestResult( - mkTestResult_({status: IDLE}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({status: IDLE, testPath: ['s1'], browserId: 'b1', attempt: 0}) ); builder.reuseTestsTree(srcBuilder.tree); @@ -126,12 +114,10 @@ describe('GuiResultsTreeBuilder', () => { it('should register reused result ids', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 1}) ); builder.addTestResult( - mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) ); @@ -145,13 +131,11 @@ describe('GuiResultsTreeBuilder', () => { it('should not reuse images if browser ids does not matched', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_({imagesInfo: [{stateName: 'img1'}]}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0, imagesInfo: [{stateName: 'img1'}]}) ); builder.addTestResult( - mkTestResult_({imagesInfo: [{stateName: 'img1'}]}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b2', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b2', attempt: 0, imagesInfo: [{stateName: 'img1'}]}) ); builder.reuseTestsTree(srcBuilder.tree); @@ -162,14 +146,14 @@ describe('GuiResultsTreeBuilder', () => { it('should reuse all images from the passed tree if browser ids matched', () => { const srcBuilder = mkGuiTreeBuilder(); + const imagesInfo1 = [{stateName: 'img1'}, {stateName: 'img2'}]; srcBuilder.addTestResult( - mkTestResult_({imagesInfo: [{stateName: 'img1'}, {stateName: 'img2'}]}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0, imagesInfo: imagesInfo1}) ); + const imagesInfo2 = [{stateName: 'img1'}]; builder.addTestResult( - mkTestResult_({imagesInfo: [{stateName: 'img1'}]}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0, imagesInfo: imagesInfo2}) ); builder.reuseTestsTree(srcBuilder.tree); @@ -180,13 +164,12 @@ describe('GuiResultsTreeBuilder', () => { it('should register reused images ids', () => { const srcBuilder = mkGuiTreeBuilder(); + const imagesInfo = [{stateName: 'img1'}, {stateName: 'img2'}]; srcBuilder.addTestResult( - mkTestResult_({imagesInfo: [{stateName: 'img1'}, {stateName: 'img2'}]}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0, imagesInfo}) ); builder.addTestResult( - mkTestResult_(), mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) ); @@ -200,13 +183,11 @@ describe('GuiResultsTreeBuilder', () => { it('should not reuse suite status if browser ids does not matched', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_({status: FAIL}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({status: FAIL, testPath: ['s1'], browserId: 'b1', attempt: 0}) ); builder.addTestResult( - mkTestResult_({status: IDLE}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b2', attempt: 0}) + mkFormattedResult_({status: IDLE, testPath: ['s1'], browserId: 'b2', attempt: 0}) ); builder.reuseTestsTree(srcBuilder.tree); @@ -217,13 +198,11 @@ describe('GuiResultsTreeBuilder', () => { it('should reuse suite status from passed tree with if browser ids matched', () => { const srcBuilder = mkGuiTreeBuilder(); srcBuilder.addTestResult( - mkTestResult_({status: FAIL}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({status: FAIL, testPath: ['s1'], browserId: 'b1', attempt: 0}) ); builder.addTestResult( - mkTestResult_({status: IDLE}), - mkFormattedResult_({testPath: ['s1'], browserId: 'b1', attempt: 0}) + mkFormattedResult_({status: IDLE, testPath: ['s1'], browserId: 'b1', attempt: 0}) ); builder.reuseTestsTree(srcBuilder.tree); @@ -236,8 +215,7 @@ describe('GuiResultsTreeBuilder', () => { describe('"getTestBranch" method', () => { it('should return "suites" as array for root suite', () => { builder.addTestResult( - mkTestResult_({status: IDLE}), - mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 0}) + mkFormattedResult_({status: IDLE, testPath: ['s'], browserId: 'b', attempt: 0}) ); const {suites} = builder.getTestBranch('s b 0'); @@ -248,10 +226,15 @@ describe('GuiResultsTreeBuilder', () => { describe('"getResultDataToUnacceptImage" method', () => { it('should return "shouldRemoveResult: true" if it is the only updated image in result', () => { - const formattedRes1 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 0}); - const formattedRes2 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 1}); - builder.addTestResult(mkTestResult_({imagesInfo: [{stateName: 'foo', status: FAIL}]}), formattedRes1); - builder.addTestResult(mkTestResult_({imagesInfo: [{stateName: 'foo', status: UPDATED}]}), formattedRes2); + const formattedRes1 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 0, imagesInfo: [ + {stateName: 'foo', status: FAIL}] + }); + const formattedRes2 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 1, imagesInfo: [ + {stateName: 'foo', status: UPDATED}] + }); + + builder.addTestResult(formattedRes1); + builder.addTestResult(formattedRes2); const {shouldRemoveResult} = builder.getResultDataToUnacceptImage('s b 1', 'foo'); @@ -261,10 +244,11 @@ describe('GuiResultsTreeBuilder', () => { it('should return "shouldRemoveResult: false" if it is not the only updated image in result', () => { const imagesInfo1 = [{stateName: 'foo', status: FAIL}, {stateName: 'bar', status: FAIL}]; const imagesInfo2 = [{stateName: 'foo', status: UPDATED}, {stateName: 'bar', status: UPDATED}]; - const formattedRes1 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 0}); - const formattedRes2 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 1}); - builder.addTestResult(mkTestResult_({imagesInfo: imagesInfo1}), formattedRes1); - builder.addTestResult(mkTestResult_({imagesInfo: imagesInfo2}), formattedRes2); + const formattedRes1 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 0, imagesInfo: imagesInfo1}); + const formattedRes2 = mkFormattedResult_({testPath: ['s'], browserId: 'b', attempt: 1, imagesInfo: imagesInfo2}); + + builder.addTestResult(formattedRes1); + builder.addTestResult(formattedRes2); const {shouldRemoveResult} = builder.getResultDataToUnacceptImage('s b 1', 'foo'); diff --git a/test/unit/lib/tests-tree-builder/static.js b/test/unit/lib/tests-tree-builder/static.js index 6270e7085..310608c72 100644 --- a/test/unit/lib/tests-tree-builder/static.js +++ b/test/unit/lib/tests-tree-builder/static.js @@ -48,24 +48,38 @@ describe('StaticResultsTreeBuilder', () => { ]; }; - const formatToTestResult = (result, data = {}) => { + const formatDbResultToTestResult = (result, overrides = {}) => { return { description: result.description, imagesInfo: result.imagesInfo, - metaInfo: result.metaInfo, + meta: result.metaInfo, history: result.history, multipleTabs: result.multipleTabs, - name: result.name, - screenshot: result.screenshot, + browserId: result.name, status: result.status, - suiteUrl: result.suiteUrl, + url: result.suiteUrl, skipReason: result.skipReason, error: result.error, timestamp: result.timestamp, - ...data + ...overrides }; }; + const formatTestResult = (result) => _.pick(result, [ + 'attempt', + 'description', + 'imagesInfo', + 'meta', + 'history', + 'multipleTabs', + 'browserId', + 'status', + 'url', + 'skipReason', + 'error', + 'timestamp' + ]); + beforeEach(() => { sandbox.stub(StaticTestsTreeBuilder.prototype, 'addTestResult'); sandbox.stub(StaticTestsTreeBuilder.prototype, 'sortTree'); @@ -84,35 +98,34 @@ describe('StaticResultsTreeBuilder', () => { builder.build(rows); - assert.calledWith( - StaticTestsTreeBuilder.prototype.addTestResult.firstCall, - sinon.match(formatToTestResult(dataFromDb1, {attempt: 0})), - sinon.match({browserId: 'yabro', testPath: ['s1'], attempt: 0}) - ); - assert.calledWith( - StaticTestsTreeBuilder.prototype.addTestResult.secondCall, - sinon.match(formatToTestResult(dataFromDb2, {attempt: 0})), - sinon.match({browserId: 'yabro', testPath: ['s2'], attempt: 0}) - ); + assert.calledTwice(StaticTestsTreeBuilder.prototype.addTestResult); + + const actualTestResults = StaticTestsTreeBuilder.prototype.addTestResult + .getCalls() + .map(_.property('args.0')) + .map(formatTestResult); + const expectedTestResults = [dataFromDb1, dataFromDb2].map(r => formatDbResultToTestResult(r, {attempt: 0})); + assert.deepEqual(actualTestResults, expectedTestResults); }); - it('the same test with increase attempt', () => { + it('the same test with increased attempt', () => { const dataFromDb1 = mkDataFromDb_({suitePath: ['s1'], name: 'yabro', timestamp: 10}); const dataFromDb2 = mkDataFromDb_({suitePath: ['s1'], name: 'yabro', timestamp: 20}); const rows = [mkDataRowFromDb_(dataFromDb1), mkDataRowFromDb_(dataFromDb2)]; builder.build(rows); - assert.calledWith( - StaticTestsTreeBuilder.prototype.addTestResult.firstCall, - sinon.match(formatToTestResult(dataFromDb1, {attempt: 0})), - sinon.match({browserId: 'yabro', testPath: ['s1'], attempt: 0}) - ); - assert.calledWith( - StaticTestsTreeBuilder.prototype.addTestResult.secondCall, - sinon.match(formatToTestResult(dataFromDb2, {attempt: 1})), - sinon.match({browserId: 'yabro', testPath: ['s1'], attempt: 1}) - ); + assert.calledTwice(StaticTestsTreeBuilder.prototype.addTestResult); + + const actualTestResults = StaticTestsTreeBuilder.prototype.addTestResult + .getCalls() + .map(_.property('args.0')) + .map(formatTestResult); + const expectedTestResults = [ + formatDbResultToTestResult(dataFromDb1, {attempt: 0}), + formatDbResultToTestResult(dataFromDb2, {attempt: 1}) + ]; + assert.deepEqual(actualTestResults, expectedTestResults); }); }); diff --git a/test/unit/utils.js b/test/unit/utils.js index 1131994db..b77989de7 100644 --- a/test/unit/utils.js +++ b/test/unit/utils.js @@ -40,6 +40,9 @@ function stubTool(config = stubConfig(), events = {}, errors = {}, htmlReporter) return false; }; + sinon.stub(tool.htmlReporter, 'imagesSaver').value({saveImg: sinon.stub()}); + sinon.stub(tool.htmlReporter, 'config').value({}); + return tool; } From a2be74fd85d5d8042173aac2b421631fc9a9fba5 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Tue, 9 Jan 2024 02:30:07 +0300 Subject: [PATCH 6/7] test: fixed and added some pwt unit tests --- lib/test-adapter/cache/hermione.ts | 1 - lib/test-adapter/cache/playwright.ts | 1 - lib/test-adapter/playwright.ts | 41 +++---- lib/types.ts | 7 +- playwright.ts | 4 +- test/unit/lib/test-adapter/hermione.ts | 3 - test/unit/lib/test-adapter/playwright.ts | 129 ++++++++++++++--------- 7 files changed, 101 insertions(+), 85 deletions(-) delete mode 100644 lib/test-adapter/cache/hermione.ts delete mode 100644 lib/test-adapter/cache/playwright.ts diff --git a/lib/test-adapter/cache/hermione.ts b/lib/test-adapter/cache/hermione.ts deleted file mode 100644 index d040676b2..000000000 --- a/lib/test-adapter/cache/hermione.ts +++ /dev/null @@ -1 +0,0 @@ -export const testsAttempts: Map = new Map(); diff --git a/lib/test-adapter/cache/playwright.ts b/lib/test-adapter/cache/playwright.ts deleted file mode 100644 index d040676b2..000000000 --- a/lib/test-adapter/cache/playwright.ts +++ /dev/null @@ -1 +0,0 @@ -export const testsAttempts: Map = new Map(); diff --git a/lib/test-adapter/playwright.ts b/lib/test-adapter/playwright.ts index 9eede117d..36612e733 100644 --- a/lib/test-adapter/playwright.ts +++ b/lib/test-adapter/playwright.ts @@ -5,8 +5,7 @@ import _ from 'lodash'; import stripAnsi from 'strip-ansi'; import {ReporterTestResult} from './index'; -import {testsAttempts} from './cache/playwright'; -import {getError, getShortMD5, isImageDiffError, isNoRefImageError, mkTestId} from '../common-utils'; +import {getError, getShortMD5, isImageDiffError, isNoRefImageError} from '../common-utils'; import {ERROR, FAIL, PWT_TITLE_DELIMITER, SUCCESS, TestStatus} from '../constants'; import {ErrorName} from '../errors'; import { @@ -18,7 +17,6 @@ import { ImageSize, TestError } from '../types'; -import * as utils from '../server-utils'; import type {CoordBounds} from 'looks-same'; export type PlaywrightAttachment = PlaywrightTestResult['attachments'][number]; @@ -46,7 +44,7 @@ export enum ImageTitleEnding { const ANY_IMAGE_ENDING_REGEXP = new RegExp(Object.values(ImageTitleEnding).map(ending => `${ending}$`).join('|')); -const DEFAULT_DIFF_OPTIONS = { +export const DEFAULT_DIFF_OPTIONS = { diffColor: '#ff00ff' } satisfies Partial; @@ -90,7 +88,7 @@ const extractErrorStack = (result: PlaywrightTestResult): string => { return JSON.stringify(result.errors.map(err => stripAnsi(err.stack || ''))); }; -const extractImageError = (result: PlaywrightTestResult, {state, expectedAttachment, diffAttachment, actualAttachment} : { +const extractToMatchScreenshotError = (result: PlaywrightTestResult, {state, expectedAttachment, diffAttachment, actualAttachment} : { state: string; expectedAttachment?: PlaywrightAttachment; diffAttachment?: PlaywrightAttachment; @@ -107,7 +105,6 @@ const extractImageError = (result: PlaywrightTestResult, {state, expectedAttachm return {name: ErrorName.IMAGE_DIFF, message: '', diffClusters: imageDiffError?.meta?.diffClusters}; } - // only supports toMatchScreenshot const errors = (result.errors || []) as PwtNoRefImageError[]; const noRefImageError = errors.find(err => { return err.meta?.type === ErrorName.NO_REF_IMAGE && err.meta.snapshotName === snapshotName; @@ -135,16 +132,11 @@ export class PlaywrightTestAdapter implements ReporterTestResult { private readonly _testResult: PlaywrightTestResult; private _attempt: number; - constructor(testCase: PlaywrightTestCase, testResult: PlaywrightTestResult) { + constructor(testCase: PlaywrightTestCase, testResult: PlaywrightTestResult, attempt: number) { this._testCase = testCase; this._testResult = testResult; - const testId = mkTestId(this.fullName, this.browserId); - if (utils.shouldUpdateAttempt(this.status)) { - testsAttempts.set(testId, _.isUndefined(testsAttempts.get(testId)) ? 0 : testsAttempts.get(testId) as number + 1); - } - - this._attempt = testsAttempts.get(testId) || 0; + this._attempt = attempt; } get attempt(): number { @@ -165,7 +157,7 @@ export class PlaywrightTestAdapter implements ReporterTestResult { const result: TestError = {name: ErrorName.GENERAL_ERROR, message}; const stack = extractErrorStack(this._testResult); - if (stack) { + if (!_.isNil(stack)) { result.stack = stack; } @@ -210,36 +202,35 @@ export class PlaywrightTestAdapter implements ReporterTestResult { const diffAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Diff)); const actualAttachment = attachments.find(a => a.name?.endsWith(ImageTitleEnding.Actual)); - const [refImg, diffImg, actualImg] = [expectedAttachment, diffAttachment, actualAttachment].map(getImageData); + const [expectedImg, diffImg, actualImg] = [expectedAttachment, diffAttachment, actualAttachment].map(getImageData); - const error = extractImageError(this._testResult, {state, expectedAttachment, diffAttachment, actualAttachment}) || this.error; + const error = extractToMatchScreenshotError(this._testResult, {state, expectedAttachment, diffAttachment, actualAttachment}) || this.error; - if (error?.name === ErrorName.IMAGE_DIFF && refImg && diffImg && actualImg) { + // We don't provide refImg here, because on some pwt versions it's impossible to provide correct path: + // older pwt versions had test-results directory in expected path instead of project directory. + if (error?.name === ErrorName.IMAGE_DIFF && expectedImg && diffImg && actualImg) { return { status: FAIL, stateName: state, - refImg, diffImg, actualImg, - expectedImg: _.clone(refImg), + expectedImg, diffClusters: _.get(error, 'diffClusters', []), // TODO: extract diffOptions from config - diffOptions: {current: actualImg.path, reference: refImg.path, ...DEFAULT_DIFF_OPTIONS} + diffOptions: {current: actualImg.path, reference: expectedImg.path, ...DEFAULT_DIFF_OPTIONS} } satisfies ImageInfoDiff; - } else if (error?.name === ErrorName.NO_REF_IMAGE && refImg && actualImg) { + } else if (error?.name === ErrorName.NO_REF_IMAGE && actualImg) { return { status: ERROR, stateName: state, error: _.pick(error, ['message', 'name', 'stack']), - refImg, actualImg } satisfies ImageInfoNoRef; - } else if (!error && refImg) { + } else if (!error && expectedImg) { return { status: SUCCESS, stateName: state, - refImg, - expectedImg: _.clone(refImg), + expectedImg, ...(actualImg ? {actualImg} : {}) } satisfies ImageInfoSuccess; } diff --git a/lib/types.ts b/lib/types.ts index f93ce6528..7fb67261a 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -65,7 +65,8 @@ export interface TestError { export interface ImageInfoDiff { status: TestStatus.FAIL; stateName: string; - refImg: ImageFile; + // Ref image is absent in pwt test results + refImg?: ImageFile; diffClusters?: CoordBounds[]; expectedImg: ImageFile; actualImg: ImageFile; @@ -81,6 +82,7 @@ interface AssertViewSuccess { export interface ImageInfoSuccess { status: TestStatus.SUCCESS; stateName: string; + // Ref image may be absent in pwt test results refImg?: ImageFile; diffClusters?: CoordBounds[]; expectedImg: ImageFile; @@ -101,7 +103,8 @@ export interface ImageInfoNoRef { status: TestStatus.ERROR; error?: TestError; stateName: string; - refImg: ImageFile; + // Ref image may be absent in pwt test results + refImg?: ImageFile; actualImg: ImageFile; } diff --git a/playwright.ts b/playwright.ts index 93454caa7..3efc83ab1 100644 --- a/playwright.ts +++ b/playwright.ts @@ -10,7 +10,7 @@ import {StaticReportBuilder} from './lib/report-builder/static'; import {HtmlReporter} from './lib/plugin-api'; import {ReporterConfig, TestSpecByPath} from './lib/types'; import {parseConfig} from './lib/config'; -import {PluginEvents, ToolName} from './lib/constants'; +import {PluginEvents, ToolName, UNKNOWN_ATTEMPT} from './lib/constants'; import {RegisterWorkers} from './lib/workers/create-workers'; import {PlaywrightTestAdapter} from './lib/test-adapter/playwright'; import {SqliteClient} from './lib/sqlite-client'; @@ -68,7 +68,7 @@ class MyReporter implements Reporter { const staticReportBuilder = this._staticReportBuilder as StaticReportBuilder; - const formattedResult = new PlaywrightTestAdapter(test, result); + const formattedResult = new PlaywrightTestAdapter(test, result, UNKNOWN_ATTEMPT); await staticReportBuilder.addTestResult(formattedResult); }); diff --git a/test/unit/lib/test-adapter/hermione.ts b/test/unit/lib/test-adapter/hermione.ts index 1662974cc..be8bc8a4a 100644 --- a/test/unit/lib/test-adapter/hermione.ts +++ b/test/unit/lib/test-adapter/hermione.ts @@ -23,7 +23,6 @@ describe('HermioneTestAdapter', () => { let commonUtils: sinon.SinonStubbedInstance; let fs: sinon.SinonStubbedInstance; let tmp: typeof tmpOriginal; - let hermioneCache: typeof import('lib/test-adapter/cache/hermione'); let testAdapterUtils: sinon.SinonStubbedInstance; const mkHermioneTestResultAdapter = ( @@ -41,7 +40,6 @@ describe('HermioneTestAdapter', () => { beforeEach(() => { tmp = {tmpdir: 'default/dir'} as typeof tmpOriginal; fs = sinon.stub(_.clone(fsOriginal)); - hermioneCache = {testsAttempts: new Map()}; getSuitePath = sandbox.stub(); getCommandsHistory = sandbox.stub(); @@ -65,7 +63,6 @@ describe('HermioneTestAdapter', () => { '../plugin-utils': {getSuitePath}, '../history-utils': {getCommandsHistory}, '../server-utils': utils, - './cache/hermione': hermioneCache, './utils': testAdapterUtils }).HermioneTestAdapter; sandbox.stub(utils, 'getCurrentPath').returns(''); diff --git a/test/unit/lib/test-adapter/playwright.ts b/test/unit/lib/test-adapter/playwright.ts index b25d0dbd0..fa1afac99 100644 --- a/test/unit/lib/test-adapter/playwright.ts +++ b/test/unit/lib/test-adapter/playwright.ts @@ -2,15 +2,20 @@ import sinon from 'sinon'; import _ from 'lodash'; import proxyquire from 'proxyquire'; import {TestCase, TestResult} from '@playwright/test/reporter'; -import {ImageTitleEnding, PlaywrightAttachment, PlaywrightTestAdapterOptions, PwtImageDiffError, PwtNoRefImageError, PwtTestStatus} from 'lib/test-adapter/playwright'; -import {ErrorName, ImageDiffError, NoRefImageError} from 'lib/errors'; -import {TestStatus} from 'lib/constants'; +import { + DEFAULT_DIFF_OPTIONS, + ImageTitleEnding, + PlaywrightAttachment, + PwtTestStatus +} from 'lib/test-adapter/playwright'; +import {ErrorName} from 'lib/errors'; +import {ERROR, FAIL, TestStatus, UNKNOWN_ATTEMPT} from 'lib/constants'; +import {ImageInfoDiff, ImageInfoNoRef} from 'lib/types'; describe('PlaywrightTestAdapter', () => { let sandbox: sinon.SinonSandbox; let PlaywrightTestAdapter: typeof import('lib/test-adapter/playwright').PlaywrightTestAdapter; let imageSizeStub: sinon.SinonStub; - let playwrightCache: typeof import('lib/test-adapter/cache/playwright'); const createAttachment = (path: string): PlaywrightAttachment => ({ contentType: 'image/png', @@ -36,20 +41,13 @@ describe('PlaywrightTestAdapter', () => { steps: [] } as any); - const mkAdapterOptions = (overrides: Partial = {}): PlaywrightTestAdapterOptions => _.defaults(overrides, { - imagesInfoFormatter: sinon.stub() - } as any); - beforeEach(() => { sandbox = sinon.createSandbox(); - playwrightCache = {testsAttempts: new Map()}; - imageSizeStub = sinon.stub().returns({height: 100, width: 200}); PlaywrightTestAdapter = proxyquire('lib/test-adapter/playwright', { - 'image-size': imageSizeStub, - './cache/playwright': playwrightCache + 'image-size': imageSizeStub }).PlaywrightTestAdapter; }); @@ -59,29 +57,15 @@ describe('PlaywrightTestAdapter', () => { describe('attempt', () => { it('should return suite attempt', () => { - // eslint-disable-next-line no-new - new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); - const adapter2 = new PlaywrightTestAdapter(mkTestCase({titlePath: sinon.stub().returns(['another-title'])}), mkTestResult()); - const adapter3 = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter = new PlaywrightTestAdapter(mkTestCase({titlePath: sinon.stub().returns(['another-title'])}), mkTestResult(), 3); - assert.equal(adapter3.attempt, 1); - assert.equal(adapter2.attempt, 0); - }); - - it('should not increment attempt for skipped tests', () => { - const testResult = mkTestResult({status: 'skipped'}); - - // eslint-disable-next-line no-new - new PlaywrightTestAdapter(mkTestCase(), testResult); - const adapter2 = new PlaywrightTestAdapter(mkTestCase(), testResult); - - assert.equal(adapter2.attempt, 0); + assert.equal(adapter.attempt, 3); }); }); describe('browserId', () => { it('should return browserId', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), UNKNOWN_ATTEMPT); assert.equal(adapter.browserId, 'some-browser'); }); @@ -89,7 +73,7 @@ describe('PlaywrightTestAdapter', () => { describe('error', () => { it('should return undefined if there are no errors', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors: []})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors: []}), UNKNOWN_ATTEMPT); const {error} = adapter; @@ -99,7 +83,7 @@ describe('PlaywrightTestAdapter', () => { it('should return an error with name NO_REF_IMAGE for snapshot missing errors', () => { const errorMessage = 'A snapshot doesn\'t exist: image-name.png.'; const errors = [{message: errorMessage}]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); const {error} = adapter; @@ -110,7 +94,7 @@ describe('PlaywrightTestAdapter', () => { it('should return an error with name IMAGE_DIFF for screenshot comparison failures', () => { const errorMessage = 'Screenshot comparison failed'; const errors = [{message: errorMessage}]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); const {error} = adapter; @@ -122,7 +106,7 @@ describe('PlaywrightTestAdapter', () => { const errorMessage = 'Some error occurred'; const errorStack = 'Error: Some error occurred at some-file.ts:10:15'; const errors = [{message: errorMessage, stack: errorStack}]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); const {error} = adapter; @@ -134,7 +118,7 @@ describe('PlaywrightTestAdapter', () => { {message: 'First error', stack: 'Error: First error at some-file.ts:5:10'}, {message: 'Second error', stack: 'Error: Second error at another-file.ts:15:20'} ]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); const expectedMessage = JSON.stringify(errors.map(err => err.message)); const expectedStack = JSON.stringify(errors.map(err => err.stack)); @@ -147,7 +131,7 @@ describe('PlaywrightTestAdapter', () => { describe('file', () => { it('should return file path', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), UNKNOWN_ATTEMPT); assert.strictEqual(adapter.file, 'test-file-path'); }); @@ -155,7 +139,7 @@ describe('PlaywrightTestAdapter', () => { describe('fullName', () => { it('should return fullName', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), UNKNOWN_ATTEMPT); assert.strictEqual(adapter.fullName, 'describe › test'); }); @@ -167,7 +151,7 @@ describe('PlaywrightTestAdapter', () => { {title: 'Step1', duration: 100}, {title: 'Step2', duration: 200} ]; - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({steps} as any)); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({steps} as any), UNKNOWN_ATTEMPT); const expectedHistory = ['Step1 <- 100ms\n', 'Step2 <- 200ms\n']; assert.deepEqual(adapter.history, expectedHistory); @@ -176,7 +160,7 @@ describe('PlaywrightTestAdapter', () => { describe('id', () => { it('should return id', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), 0); assert.strictEqual(adapter.id, 'describe test some-browser 0'); }); @@ -184,51 +168,94 @@ describe('PlaywrightTestAdapter', () => { describe('imageDir', () => { it('should return imageDir', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), UNKNOWN_ATTEMPT); assert.strictEqual(adapter.imageDir, '4050de5'); }); }); describe('imagesInfo', () => { - it('should call formatter', () => { - const getImagesInfoStub = sinon.stub(); - const options = mkAdapterOptions({imagesInfoFormatter: {getImagesInfo: getImagesInfoStub}}); - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), options); + it('should correctly format diff result', () => { + const errors = [{message: 'Screenshot comparison failed', stack: ''}]; + const attachments: TestResult['attachments'] = [ + {name: `header-actual.png`, path: 'test-results/header-actual.png', contentType: 'image/png'}, + {name: `header-expected.png`, path: 'project-dir/header-expected.png', contentType: 'image/png'}, + {name: `header-diff.png`, path: 'test-results/header-diff.png', contentType: 'image/png'}, + {name: 'screenshot', path: 'test-results/test-name-1.png', contentType: 'image/png'} + ]; - adapter.imagesInfo; + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({attachments, errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.imagesInfo.length, 2); + assert.deepEqual(adapter.imagesInfo.find(info => (info as ImageInfoDiff).stateName === undefined), { + status: ERROR, + actualImg: {path: `test-results/test-name-1.png`, size: {height: 100, width: 200}} + }); + assert.deepEqual(adapter.imagesInfo.find(info => (info as ImageInfoDiff).stateName === 'header'), { + status: FAIL, + stateName: 'header', + actualImg: {path: `test-results/header-actual.png`, size: {height: 100, width: 200}}, + expectedImg: {path: 'project-dir/header-expected.png', size: {height: 100, width: 200}}, + diffImg: {path: 'test-results/header-diff.png', size: {height: 100, width: 200}}, + diffClusters: [], + diffOptions: {current: 'test-results/header-actual.png', reference: 'project-dir/header-expected.png', ...DEFAULT_DIFF_OPTIONS} + }); + }); + + it('should correctly format no ref result', () => { + const errors = [{message: 'snapshot doesn\'t exist at some.png', stack: ''}]; + const attachments: TestResult['attachments'] = [ + {name: `header-actual.png`, path: 'test-results/header-actual.png', contentType: 'image/png'}, + {name: 'screenshot', path: 'test-results/test-name-1.png', contentType: 'image/png'} + ]; - assert.calledOnceWith(getImagesInfoStub, adapter); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({attachments, errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.imagesInfo.length, 2); + assert.deepEqual(adapter.imagesInfo.find(info => (info as ImageInfoNoRef).stateName === undefined), { + status: ERROR, + actualImg: {path: `test-results/test-name-1.png`, size: {height: 100, width: 200}} + }); + assert.deepEqual(adapter.imagesInfo.find(info => (info as ImageInfoNoRef).stateName === 'header'), { + status: ERROR, + stateName: 'header', + error: { + name: ErrorName.NO_REF_IMAGE, + message: 'snapshot doesn\'t exist at some.png', + stack: '' + }, + actualImg: {path: `test-results/header-actual.png`, size: {height: 100, width: 200}} + }); }); }); describe('status', () => { it('should return SUCCESS for PASSED PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.PASSED})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.PASSED}), UNKNOWN_ATTEMPT); assert.equal(adapter.status, TestStatus.SUCCESS); }); it('should return FAIL for FAILED PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.FAILED})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.FAILED}), UNKNOWN_ATTEMPT); assert.equal(adapter.status, TestStatus.FAIL); }); it('should return FAIL for TIMED_OUT PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.TIMED_OUT})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.TIMED_OUT}), UNKNOWN_ATTEMPT); assert.equal(adapter.status, TestStatus.FAIL); }); it('should return FAIL for INTERRUPTED PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.INTERRUPTED})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.INTERRUPTED}), UNKNOWN_ATTEMPT); assert.equal(adapter.status, TestStatus.FAIL); }); it('should return SKIPPED for any other PwtTestStatus', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.SKIPPED})); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.SKIPPED}), UNKNOWN_ATTEMPT); assert.equal(adapter.status, TestStatus.SKIPPED); }); @@ -236,7 +263,7 @@ describe('PlaywrightTestAdapter', () => { describe('testPath', () => { it('should return testPath', () => { - const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult()); + const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), UNKNOWN_ATTEMPT); assert.deepEqual(adapter.testPath, ['describe', 'test']); }); From 74fd3af23d67a0f0c64686076d59007b299a1b30 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Tue, 9 Jan 2024 17:34:35 +0300 Subject: [PATCH 7/7] fix: create directory before saving buffer --- lib/images-info-saver.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/images-info-saver.ts b/lib/images-info-saver.ts index f8c5c4581..0ba8e7879 100644 --- a/lib/images-info-saver.ts +++ b/lib/images-info-saver.ts @@ -149,6 +149,7 @@ export class ImagesInfoSaver extends EventEmitter2 { private async _saveImage(imageData: ImageFile | ImageBuffer | ImageBase64, destPath: string): Promise { const sourceFilePath = isImageBufferData(imageData) || isBase64Image(imageData) ? getTempPath(destPath) : imageData.path; if (isImageBufferData(imageData)) { + await makeDirFor(sourceFilePath); await fs.writeFile(sourceFilePath, Buffer.from(imageData.buffer)); } else if (isBase64Image(imageData)) { await makeDirFor(sourceFilePath);