From 182022557400a93d2877ce71fa34c7cb25fafb97 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Tue, 12 Dec 2023 01:14:11 +0300 Subject: [PATCH] fix: fix unit and e2e tests, further refactoring --- hermione.ts | 108 ++++++------- lib/common-utils.ts | 22 +-- lib/constants/tests.ts | 4 + lib/gui/tool-runner/index.ts | 66 +++----- lib/gui/tool-runner/report-subscriber.ts | 68 +++----- lib/image-handler.ts | 45 +++--- lib/report-builder/gui.ts | 56 ++++--- lib/report-builder/static.ts | 85 +++++++--- lib/reporter-helpers.ts | 5 +- lib/server-utils.ts | 14 ++ lib/sqlite-client.ts | 1 + lib/static/components/retry-switcher/item.jsx | 14 +- lib/static/modules/reducers/tree/index.js | 2 +- lib/test-adapter/hermione.ts | 37 +---- lib/test-adapter/index.ts | 1 - lib/test-adapter/playwright.ts | 4 - lib/test-adapter/reporter.ts | 121 ++++++++++++++ lib/test-adapter/utils/index.ts | 34 +++- lib/tests-tree-builder/gui.ts | 15 -- lib/tests-tree-builder/static.ts | 1 + lib/types.ts | 4 +- playwright.ts | 22 +-- test/unit/hermione.js | 113 +------------ test/unit/lib/gui/tool-runner/index.js | 16 +- .../lib/gui/tool-runner/report-subsciber.js | 47 ++---- test/unit/lib/report-builder/gui.js | 81 ++++++---- test/unit/lib/report-builder/static.js | 148 +++++++++++++++++- test/unit/lib/test-adapter/hermione.ts | 61 ++------ test/unit/lib/tests-tree-builder/base.js | 18 +-- test/unit/lib/tests-tree-builder/static.js | 16 +- test/unit/utils.js | 22 ++- 31 files changed, 701 insertions(+), 550 deletions(-) create mode 100644 lib/test-adapter/reporter.ts diff --git a/hermione.ts b/hermione.ts index eabb750f1..385c3a700 100644 --- a/hermione.ts +++ b/hermione.ts @@ -6,37 +6,42 @@ import PQueue from 'p-queue'; import {CommanderStatic} from '@gemini-testing/commander'; import {cliCommands} from './lib/cli-commands'; -import {hasDiff} from './lib/common-utils'; +import {hasFailedImages} from './lib/common-utils'; import {parseConfig} from './lib/config'; -import {ERROR, FAIL, SUCCESS, ToolName} from './lib/constants'; +import {SKIPPED, SUCCESS, TestStatus, ToolName, UNKNOWN_ATTEMPT} from './lib/constants'; import {HtmlReporter} from './lib/plugin-api'; import {StaticReportBuilder} from './lib/report-builder/static'; import {formatTestResult, logPathToHtmlReport, logError} from './lib/server-utils'; import {SqliteClient} from './lib/sqlite-client'; -import {HermioneTestAdapter, ReporterTestResult} from './lib/test-adapter'; -import {TestAttemptManager} from './lib/test-attempt-manager'; -import {HtmlReporterApi, ReporterConfig, ReporterOptions} from './lib/types'; +import {HtmlReporterApi, ImageInfoFull, ReporterOptions} from './lib/types'; import {createWorkers, CreateWorkersRunner} from './lib/workers/create-workers'; -let workers: ReturnType; - export = (hermione: Hermione, opts: Partial): void => { - if (hermione.isWorker()) { + if (hermione.isWorker() || !opts.enabled) { return; } const config = parseConfig(opts); - if (!config.enabled) { - return; - } - const htmlReporter = HtmlReporter.create(config, {toolName: ToolName.Hermione}); (hermione as Hermione & HtmlReporterApi).htmlReporter = htmlReporter; let isCliCommandLaunched = false; let handlingTestResults: Promise; + let staticReportBuilder: StaticReportBuilder; + + const withMiddleware = unknown>(fn: T): + (...args: Parameters) => ReturnType | undefined => { + return (...args: unknown[]) => { + // If any CLI command was launched, e.g. merge-reports, we need to interrupt regular flow + if (isCliCommandLaunched) { + return; + } + + return fn.call(undefined, ...args) as ReturnType; + }; + }; hermione.on(hermione.events.CLI, (commander: CommanderStatic) => { _.values(cliCommands).forEach((command: string) => { @@ -49,30 +54,27 @@ export = (hermione: Hermione, opts: Partial): void => { }); }); - hermione.on(hermione.events.INIT, async () => { - if (isCliCommandLaunched) { - return; - } - + hermione.on(hermione.events.INIT, withMiddleware(async () => { const dbClient = await SqliteClient.create({htmlReporter, reportPath: config.path}); - const testAttemptManager = new TestAttemptManager(); - const staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient, testAttemptManager}); + staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient}); handlingTestResults = Promise.all([ staticReportBuilder.saveStaticFiles(), - handleTestResults(hermione, staticReportBuilder, config) + handleTestResults(hermione, staticReportBuilder) ]).then(async () => { await staticReportBuilder.finalize(); }).then(async () => { await htmlReporter.emitAsync(htmlReporter.events.REPORT_SAVED, {reportPath: config.path}); }); - }); - hermione.on(hermione.events.RUNNER_START, (runner) => { - workers = createWorkers(runner as unknown as CreateWorkersRunner); - }); + htmlReporter.emit(htmlReporter.events.DATABASE_CREATED, dbClient.getRawConnection()); + })); - hermione.on(hermione.events.RUNNER_END, async () => { + hermione.on(hermione.events.RUNNER_START, withMiddleware((runner) => { + staticReportBuilder.registerWorkers(createWorkers(runner as unknown as CreateWorkersRunner)); + })); + + hermione.on(hermione.events.RUNNER_END, withMiddleware(async () => { try { await handlingTestResults; @@ -80,57 +82,47 @@ export = (hermione: Hermione, opts: Partial): void => { } catch (e: unknown) { logError(e as Error); } - }); + })); }; -async function handleTestResults(hermione: Hermione, reportBuilder: StaticReportBuilder, pluginConfig: ReporterConfig): Promise { - const {path: reportPath} = pluginConfig; - const {imageHandler} = reportBuilder; - - const failHandler = async (testResult: HermioneTestResult): Promise => { - const status = hasDiff(testResult.assertViewResults as {name?: string}[]) ? FAIL : ERROR; - const attempt = reportBuilder.testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, status); - const formattedResult = formatTestResult(testResult, status, attempt, reportBuilder); - - const actions: Promise[] = [imageHandler.saveTestImages(formattedResult, workers)]; - - if (pluginConfig.saveErrorDetails && formattedResult.errorDetails) { - actions.push((formattedResult as HermioneTestAdapter).saveErrorDetails(reportPath)); - } - - await Promise.all(actions); - - return formattedResult; - }; - - const addFail = (formattedResult: ReporterTestResult): ReporterTestResult => { - return reportBuilder.addFail(formattedResult); - }; - +async function handleTestResults(hermione: Hermione, reportBuilder: StaticReportBuilder): Promise { return new Promise((resolve, reject) => { const queue = new PQueue({concurrency: os.cpus().length}); const promises: Promise[] = []; hermione.on(hermione.events.TEST_PASS, testResult => { promises.push(queue.add(async () => { - const attempt = reportBuilder.testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, FAIL); - const formattedResult = formatTestResult(testResult, SUCCESS, attempt, reportBuilder); - await imageHandler.saveTestImages(formattedResult, workers); - - return reportBuilder.addSuccess(formattedResult); + const formattedResult = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder); + await reportBuilder.addSuccess(formattedResult); }).catch(reject)); }); hermione.on(hermione.events.RETRY, testResult => { - promises.push(queue.add(() => failHandler(testResult).then(addFail)).catch(reject)); + promises.push(queue.add(async () => { + const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; + + const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); + + await reportBuilder.addFail(formattedResult); + }).catch(reject)); }); hermione.on(hermione.events.TEST_FAIL, testResult => { - promises.push(queue.add(() => failHandler(testResult).then(addFail)).catch(reject)); + promises.push(queue.add(async () => { + const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; + + const formattedResult = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); + + await reportBuilder.addFail(formattedResult); + }).catch(reject)); }); hermione.on(hermione.events.TEST_PENDING, testResult => { - promises.push(queue.add(() => failHandler(testResult as HermioneTestResult).then((testResult) => reportBuilder.addSkipped(testResult)).catch(reject))); + promises.push(queue.add(async () => { + const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder); + + await reportBuilder.addSkipped(formattedResult); + }).catch(reject)); }); hermione.on(hermione.events.RUNNER_END, () => { diff --git a/lib/common-utils.ts b/lib/common-utils.ts index 50f499362..9fdd4628b 100644 --- a/lib/common-utils.ts +++ b/lib/common-utils.ts @@ -5,7 +5,7 @@ import axios, {AxiosRequestConfig} from 'axios'; import {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED, TestStatus} from './constants'; import {UNCHECKED, INDETERMINATE, CHECKED} from './constants/checked-statuses'; -import {ImageData, ImageBase64, ImageInfoFull, TestError, ImageInfoError} from './types'; +import {ImageData, ImageBase64, ImageInfoFull, TestError, ImageInfoFail} from './types'; import {ErrorName, ImageDiffError, NoRefImageError} from './errors'; import {ReporterTestResult} from './test-adapter'; export const getShortMD5 = (str: string): string => { @@ -104,17 +104,17 @@ export const hasNoRefImageErrors = ({assertViewResults = []}: {assertViewResults return assertViewResults.some((assertViewResult) => isNoRefImageError(assertViewResult)); }; -export const hasFailedImages = (result: {imagesInfo?: ImageInfoFull[]}): boolean => { - const {imagesInfo = []} = result; - +export const hasFailedImages = (imagesInfo: ImageInfoFull[] = []): boolean => { return imagesInfo.some((imageInfo: ImageInfoFull) => { - return !isAssertViewError((imageInfo as ImageInfoError).error) && - (isErrorStatus(imageInfo.status) || isFailStatus(imageInfo.status)); + return (imageInfo as ImageInfoFail).stateName && + (isErrorStatus(imageInfo.status) || isFailStatus(imageInfo.status) || isNoRefImageError(imageInfo) || isImageDiffError(imageInfo)); }); }; -export const hasResultFails = (testResult: {status: TestStatus, imagesInfo?: ImageInfoFull[]}): boolean => { - return hasFailedImages(testResult) || isErrorStatus(testResult.status) || isFailStatus(testResult.status); +export const hasUnrelatedToScreenshotsErrors = (error: TestError): boolean => { + return !isNoRefImageError(error) && + !isImageDiffError(error) && + !isAssertViewError(error); }; export const getError = (error?: TestError): undefined | Pick => { @@ -131,7 +131,11 @@ export const hasDiff = (assertViewResults: {name?: string}[]): boolean => { /* This method tries to determine true status of testResult by using fields like error, imagesInfo */ export const determineStatus = (testResult: Pick): TestStatus => { - if (!hasFailedImages(testResult) && !isSkippedStatus(testResult.status) && isEmpty(testResult.error)) { + if ( + !hasFailedImages(testResult.imagesInfo) && + !isSkippedStatus(testResult.status) && + (!testResult.error || !hasUnrelatedToScreenshotsErrors(testResult.error)) + ) { return SUCCESS; } diff --git a/lib/constants/tests.ts b/lib/constants/tests.ts index 072614418..808755def 100644 --- a/lib/constants/tests.ts +++ b/lib/constants/tests.ts @@ -1 +1,5 @@ +export const HERMIONE_TITLE_DELIMITER = ' '; + export const PWT_TITLE_DELIMITER = ' › '; + +export const UNKNOWN_ATTEMPT = -1; diff --git a/lib/gui/tool-runner/index.ts b/lib/gui/tool-runner/index.ts index 0fa45ca8c..6117dbcc0 100644 --- a/lib/gui/tool-runner/index.ts +++ b/lib/gui/tool-runner/index.ts @@ -12,7 +12,7 @@ import {createTestRunner} from './runner'; import {subscribeOnToolEvents} from './report-subscriber'; import {GuiReportBuilder, GuiReportBuilderResult} from '../../report-builder/gui'; import {EventSource} from '../event-source'; -import {logger, getShortMD5} from '../../common-utils'; +import {logger, getShortMD5, isUpdatedStatus} from '../../common-utils'; import * as reporterHelper from '../../reporter-helpers'; import { UPDATED, @@ -22,7 +22,7 @@ import { ToolName, DATABASE_URLS_JSON_NAME, LOCAL_DATABASE_NAME, - PluginEvents + PluginEvents, UNKNOWN_ATTEMPT } from '../../constants'; import {formatId, mkFullTitle, mergeDatabasesForReuse, filterByEqualDiffSizes} from './utils'; import {getTestsTreeFromDatabase} from '../../db-utils/server'; @@ -43,7 +43,8 @@ import {TestBranch, TestEqualDiffsData, TestRefUpdateData} from '../../tests-tre import {ReporterTestResult} from '../../test-adapter'; import {ImagesInfoFormatter} from '../../image-handler'; import {SqliteClient} from '../../sqlite-client'; -import {TestAttemptManager} from '../../test-attempt-manager'; +import PQueue from 'p-queue'; +import os from 'os'; type ToolRunnerArgs = [paths: string[], hermione: Hermione & HtmlReporterApi, configs: GuiConfigs]; @@ -55,7 +56,7 @@ interface HermioneTestExtended extends HermioneTest { imagesInfo: Pick[]; } -type HermioneTestPlain = Pick; +type HermioneTestPlain = Pick; export interface UndoAcceptImagesResult { updatedImages: TreeImage[]; @@ -72,8 +73,6 @@ const formatTestResultUnsafe = ( return formatTestResult(test as HermioneTestResult, status, attempt, {imageHandler}); }; -const HERMIONE_TITLE_DELIMITER = ' '; - export class ToolRunner { private _testFiles: string[]; private _hermione: Hermione & HtmlReporterApi; @@ -86,7 +85,6 @@ export class ToolRunner { private _eventSource: EventSource; protected _reportBuilder: GuiReportBuilder | null; private _tests: Record; - private readonly _testAttemptManager: TestAttemptManager; static create(this: new (...args: ToolRunnerArgs) => T, ...args: ToolRunnerArgs): T { return new this(...args); @@ -107,7 +105,6 @@ export class ToolRunner { this._reportBuilder = null; this._tests = {}; - this._testAttemptManager = new TestAttemptManager(); } get config(): HermioneConfig { @@ -123,7 +120,7 @@ export class ToolRunner { const dbClient = await SqliteClient.create({htmlReporter: this._hermione.htmlReporter, reportPath: this._reportPath, reuse: true}); - this._reportBuilder = GuiReportBuilder.create(this._hermione.htmlReporter, this._pluginConfig, {dbClient, testAttemptManager: this._testAttemptManager}); + this._reportBuilder = GuiReportBuilder.create(this._hermione.htmlReporter, this._pluginConfig, {dbClient}); this._subscribeOnEvents(); this._collection = await this._readTests(); @@ -188,13 +185,11 @@ export class ToolRunner { return Promise.all(tests.map(async (test): Promise => { const updateResult = this._prepareTestResult(test); - const fullName = [...test.suite.path, test.state.name].join(HERMIONE_TITLE_DELIMITER); - const updateAttempt = reportBuilder.testAttemptManager.registerAttempt({fullName, browserId: test.browserId}, UPDATED); - const formattedResult = formatTestResultUnsafe(updateResult, UPDATED, updateAttempt, reportBuilder); + const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT, reportBuilder); - const failResultId = formatTestResultUnsafe(updateResult, UPDATED, updateAttempt - 1, reportBuilder).id; + const formattedResult = await reportBuilder.addUpdated(formattedResultWithoutAttempt); - updateResult.attempt = updateAttempt; + updateResult.attempt = formattedResult.attempt; await Promise.all(updateResult.imagesInfo.map(async (imageInfo) => { const {stateName} = imageInfo; @@ -205,26 +200,22 @@ export class ToolRunner { this._emitUpdateReference(result, stateName); })); - reportBuilder.addUpdated(formattedResult, failResultId); - return reportBuilder.getTestBranch(formattedResult.id); })); } async undoAcceptImages(tests: TestRefUpdateData[]): Promise { - const updatedImages: TreeImage[] = [], removedResults: string[] = []; + const updatedImages: TreeImage[] = [], removedResultIds: string[] = []; const reportBuilder = this._ensureReportBuilder(); await Promise.all(tests.map(async (test) => { const updateResult = this._prepareTestResult(test); - const fullName = [...test.suite.path, test.state.name].join(' '); - const attempt = reportBuilder.testAttemptManager.getCurrentAttempt({fullName, browserId: test.browserId}); - const formattedResult = formatTestResultUnsafe(updateResult, UPDATED, attempt, reportBuilder); + const formattedResultWithoutAttempt = formatTestResultUnsafe(updateResult, UPDATED, UNKNOWN_ATTEMPT, reportBuilder); await Promise.all(updateResult.imagesInfo.map(async (imageInfo) => { const {stateName} = imageInfo; - const undoResultData = reportBuilder.undoAcceptImage(formattedResult, stateName); + const undoResultData = reportBuilder.undoAcceptImage(formattedResultWithoutAttempt, stateName); if (undoResultData === null) { return; } @@ -234,18 +225,19 @@ export class ToolRunner { removedResult, previousExpectedPath, shouldRemoveReference, - shouldRevertReference + shouldRevertReference, + newResult } = undoResultData; updatedImage && updatedImages.push(updatedImage); - removedResult && removedResults.push(removedResult); + removedResult && removedResultIds.push(removedResult.id); if (shouldRemoveReference) { - await reporterHelper.removeReferenceImage(formattedResult, stateName); + await reporterHelper.removeReferenceImage(newResult, stateName); } if (shouldRevertReference && removedResult) { - await reporterHelper.revertReferenceImage(removedResult, formattedResult, stateName); + await reporterHelper.revertReferenceImage(removedResult, newResult, stateName); } if (previousExpectedPath && (updateResult as HermioneTest).fullTitle) { @@ -257,7 +249,7 @@ export class ToolRunner { })); })); - return {updatedImages, removedResults}; + return {updatedImages, removedResults: removedResultIds}; } async findEqualDiffs(images: TestEqualDiffsData[]): Promise { @@ -312,6 +304,7 @@ export class ToolRunner { protected async _handleRunnableCollection(): Promise { const reportBuilder = this._ensureReportBuilder(); + const queue = new PQueue({concurrency: os.cpus().length}); this._ensureTestCollection().eachTest((test, browserId) => { if (test.disabled || this._isSilentlySkipped(test)) { @@ -322,14 +315,14 @@ export class ToolRunner { const testId = formatId(test.id.toString(), browserId); this._tests[testId] = _.extend(test, {browserId}); - const attempt = 0; if (test.pending) { - reportBuilder.addSkipped(formatTestResultUnsafe(test, SKIPPED, attempt, reportBuilder)); + queue.add(async () => reportBuilder.addSkipped(formatTestResultUnsafe(test, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder))); } else { - reportBuilder.addIdle(formatTestResultUnsafe(test, IDLE, attempt, reportBuilder)); + queue.add(async () => reportBuilder.addIdle(formatTestResultUnsafe(test, IDLE, UNKNOWN_ATTEMPT, reportBuilder))); } }); + await queue.onIdle(); await this._fillTestsTree(); } @@ -338,7 +331,7 @@ export class ToolRunner { } protected _subscribeOnEvents(): void { - subscribeOnToolEvents(this._hermione, this._ensureReportBuilder(), this._eventSource, this._reportPath); + subscribeOnToolEvents(this._hermione, this._ensureReportBuilder(), this._eventSource); } protected _prepareTestResult(test: TestRefUpdateData): HermioneTestExtended | HermioneTestPlain { @@ -356,7 +349,7 @@ export class ToolRunner { const path = this._hermione.config.browsers[browserId].getScreenshotPath(rawTest, stateName); const refImg = {path, size: actualImg.size}; - assertViewResults.push({stateName, refImg, currImg: actualImg}); + assertViewResults.push({stateName, refImg, currImg: actualImg, isUpdated: isUpdatedStatus(imageInfo.status)}); return _.extend(imageInfo, {expectedImg: refImg}); }); @@ -367,8 +360,7 @@ export class ToolRunner { imagesInfo, sessionId, attempt, - meta: {url}, - updated: true + meta: {url} }); // _.merge can't fully clone test object since hermione@7+ @@ -393,14 +385,6 @@ export class ToolRunner { if (testsTree && !_.isEmpty(testsTree)) { reportBuilder.reuseTestsTree(testsTree); - - // Fill test attempt manager with data from db - for (const [, testResult] of Object.entries(testsTree.results.byId)) { - this._testAttemptManager.registerAttempt({ - fullName: testResult.suitePath.join(HERMIONE_TITLE_DELIMITER), - browserId: testResult.name - }, testResult.status, testResult.attempt); - } } this._tree = {...reportBuilder.getResult(), autoRun}; diff --git a/lib/gui/tool-runner/report-subscriber.ts b/lib/gui/tool-runner/report-subscriber.ts index 344bb6982..a307aaa1c 100644 --- a/lib/gui/tool-runner/report-subscriber.ts +++ b/lib/gui/tool-runner/report-subscriber.ts @@ -5,32 +5,17 @@ import {ClientEvents} from '../constants'; import {getSuitePath} from '../../plugin-utils'; import {createWorkers, CreateWorkersRunner} from '../../workers/create-workers'; import {logError, formatTestResult} from '../../server-utils'; -import {hasDiff} from '../../common-utils'; -import {TestStatus, RUNNING, SUCCESS, SKIPPED} from '../../constants'; +import {hasFailedImages} from '../../common-utils'; +import {TestStatus, RUNNING, SUCCESS, SKIPPED, UNKNOWN_ATTEMPT} from '../../constants'; import {GuiReportBuilder} from '../../report-builder/gui'; import {EventSource} from '../event-source'; -import {HermioneTestResult} from '../../types'; -import {HermioneTestAdapter, ReporterTestResult} from '../../test-adapter'; -import {ImageDiffError} from '../../errors'; +import {HermioneTestResult, ImageInfoFull} from '../../types'; -let workers: ReturnType; - -export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiReportBuilder, client: EventSource, reportPath: string): void => { +export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiReportBuilder, client: EventSource): void => { const queue = new PQueue({concurrency: os.cpus().length}); - const {imageHandler, testAttemptManager} = reportBuilder; - - async function failHandler(formattedResult: ReporterTestResult): Promise { - const actions: Promise[] = [imageHandler.saveTestImages(formattedResult, workers)]; - - if (formattedResult.errorDetails) { - actions.push((formattedResult as HermioneTestAdapter).saveErrorDetails(reportPath)); - } - - await Promise.all(actions); - } hermione.on(hermione.events.RUNNER_START, (runner) => { - workers = createWorkers(runner as unknown as CreateWorkersRunner); + reportBuilder.registerWorkers(createWorkers(runner as unknown as CreateWorkersRunner)); }); hermione.on(hermione.events.SUITE_BEGIN, (suite) => { @@ -45,22 +30,21 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo }); hermione.on(hermione.events.TEST_BEGIN, (data) => { - const attempt = testAttemptManager.registerAttempt({fullName: data.fullTitle(), browserId: data.browserId}, RUNNING); - const formattedResult = formatTestResult(data as HermioneTestResult, RUNNING, attempt, reportBuilder); + queue.add(async () => { + const formattedResultWithoutAttempt = formatTestResult(data as HermioneTestResult, RUNNING, UNKNOWN_ATTEMPT, reportBuilder); - reportBuilder.addRunning(formattedResult); - const testBranch = reportBuilder.getTestBranch(formattedResult.id); + const formattedResult = await reportBuilder.addRunning(formattedResultWithoutAttempt); + const testBranch = reportBuilder.getTestBranch(formattedResult.id); - return client.emit(ClientEvents.BEGIN_STATE, testBranch); + return client.emit(ClientEvents.BEGIN_STATE, testBranch); + }); }); hermione.on(hermione.events.TEST_PASS, (testResult) => { queue.add(async () => { - const attempt = testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, SUCCESS); - const formattedResult = formatTestResult(testResult, SUCCESS, attempt, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult, SUCCESS, UNKNOWN_ATTEMPT, reportBuilder); - await imageHandler.saveTestImages(formattedResult, workers); - reportBuilder.addSuccess(formattedResult); + const formattedResult = await reportBuilder.addSuccess(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); @@ -69,13 +53,11 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo hermione.on(hermione.events.RETRY, (testResult) => { queue.add(async () => { - const status = hasDiff(testResult.assertViewResults as ImageDiffError[]) ? TestStatus.FAIL : TestStatus.ERROR; - const attempt = testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, status); + const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - const formattedResult = formatTestResult(testResult, status, attempt, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); - await failHandler(formattedResult); - reportBuilder.addRetry(formattedResult); + const formattedResult = await reportBuilder.addRetry(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); @@ -84,15 +66,13 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo hermione.on(hermione.events.TEST_FAIL, (testResult) => { queue.add(async () => { - const status = hasDiff(testResult.assertViewResults as ImageDiffError[]) ? TestStatus.FAIL : TestStatus.ERROR; - const attempt = testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, status); + const status = hasFailedImages(testResult.assertViewResults as ImageInfoFull[]) ? TestStatus.FAIL : TestStatus.ERROR; - const formattedResult = formatTestResult(testResult, status, attempt, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult, status, UNKNOWN_ATTEMPT, reportBuilder); - await failHandler(formattedResult); - status === TestStatus.FAIL - ? reportBuilder.addFail(formattedResult) - : reportBuilder.addError(formattedResult); + const formattedResult = status === TestStatus.FAIL + ? await reportBuilder.addFail(formattedResultWithoutAttempt) + : await reportBuilder.addError(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); @@ -101,11 +81,9 @@ export const subscribeOnToolEvents = (hermione: Hermione, reportBuilder: GuiRepo hermione.on(hermione.events.TEST_PENDING, async (testResult) => { queue.add(async () => { - const attempt = testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, SKIPPED); - const formattedResult = formatTestResult(testResult as HermioneTestResult, SKIPPED, attempt, reportBuilder); + const formattedResultWithoutAttempt = formatTestResult(testResult as HermioneTestResult, SKIPPED, UNKNOWN_ATTEMPT, reportBuilder); - await failHandler(formattedResult); - reportBuilder.addSkipped(formattedResult); + const formattedResult = await reportBuilder.addSkipped(formattedResultWithoutAttempt); const testBranch = reportBuilder.getTestBranch(formattedResult.id); client.emit(ClientEvents.TEST_RESULT, testBranch); diff --git a/lib/image-handler.ts b/lib/image-handler.ts index 791d07a40..c53badaf9 100644 --- a/lib/image-handler.ts +++ b/lib/image-handler.ts @@ -17,7 +17,7 @@ import { ImagesSaver, ImageInfoPageSuccess } from './types'; -import {ERROR, FAIL, PluginEvents, SUCCESS, TestStatus, UPDATED} from './constants'; +import {ERROR, FAIL, PluginEvents, SUCCESS, TestStatus, UNKNOWN_ATTEMPT, UPDATED} from './constants'; import { getError, getShortMD5, @@ -142,7 +142,7 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter { const imagesInfo: ImageInfoFull[] = testResult.assertViewResults?.map((assertResult): ImageInfoFull => { let status: TestStatus, error: {message: string; stack?: string;} | undefined; - if (testResult.isUpdated === true) { + if (assertResult.isUpdated === true) { status = UPDATED; } else if (isImageDiffError(assertResult)) { status = FAIL; @@ -258,33 +258,38 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter { private _getExpectedPath(testResult: ReporterTestResultPlain, stateName?: string): {path: string, reused: boolean} { const key = this._getExpectedKey(testResult, stateName); + let result: {path: string; reused: boolean}; if (testResult.status === UPDATED) { const expectedPath = utils.getReferencePath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName}); - cacheExpectedPaths.set(key, expectedPath); - - return {path: expectedPath, reused: false}; - } - if (cacheExpectedPaths.has(key)) { - return {path: cacheExpectedPaths.get(key) as string, reused: true}; - } - - const imageInfo = this._imageStore.getLastImageInfoFromDb(testResult, stateName); + result = {path: expectedPath, reused: false}; + } else if (cacheExpectedPaths.has(key)) { + result = {path: cacheExpectedPaths.get(key) as string, reused: true}; + } else { + const imageInfo = this._imageStore.getLastImageInfoFromDb(testResult, stateName); - if (imageInfo && (imageInfo as ImageInfoFail).expectedImg) { - const expectedPath = (imageInfo as ImageInfoFail).expectedImg.path; + if (imageInfo && (imageInfo as ImageInfoFail).expectedImg) { + const expectedPath = (imageInfo as ImageInfoFail).expectedImg.path; - cacheExpectedPaths.set(key, expectedPath); - - return {path: expectedPath, reused: true}; + result = {path: expectedPath, reused: true}; + } else { + const expectedPath = utils.getReferencePath({ + attempt: testResult.attempt, + browserId: testResult.browserId, + imageDir: testResult.imageDir, + stateName + }); + + result = {path: expectedPath, reused: false}; + } } - const expectedPath = utils.getReferencePath({attempt: testResult.attempt, browserId: testResult.browserId, imageDir: testResult.imageDir, stateName}); - - cacheExpectedPaths.set(key, expectedPath); + if (testResult.attempt !== UNKNOWN_ATTEMPT) { + cacheExpectedPaths.set(key, result.path); + } - return {path: expectedPath, reused: false}; + return result; } private _getImgFromStorage(imgPath: string): string { diff --git a/lib/report-builder/gui.ts b/lib/report-builder/gui.ts index 0b17c7b2d..074406f41 100644 --- a/lib/report-builder/gui.ts +++ b/lib/report-builder/gui.ts @@ -2,7 +2,7 @@ import * as _ from 'lodash'; import {StaticReportBuilder} from './static'; import {GuiTestsTreeBuilder, TestBranch, TestEqualDiffsData, TestRefUpdateData} from '../tests-tree-builder/gui'; import { - IDLE, RUNNING, UPDATED, TestStatus, DB_COLUMNS, ToolName + IDLE, RUNNING, UPDATED, TestStatus, DB_COLUMNS, ToolName, HERMIONE_TITLE_DELIMITER } from '../constants'; import {ConfigForStaticFile, getConfigForStaticFile} from '../server-utils'; import {ReporterTestResult} from '../test-adapter'; @@ -16,11 +16,11 @@ import {copyAndUpdate} from '../test-adapter/utils'; interface UndoAcceptImageResult { updatedImage: TreeImage | undefined; - removedResult: string | undefined; + removedResult: ReporterTestResult | undefined; previousExpectedPath: string | null; shouldRemoveReference: boolean; shouldRevertReference: boolean; - newTestResult: ReporterTestResult; + newResult: ReporterTestResult; } export interface GuiReportBuilderResult { @@ -43,16 +43,16 @@ export class GuiReportBuilder extends StaticReportBuilder { this._skips = []; } - addIdle(result: ReporterTestResult): ReporterTestResult { + async addIdle(result: ReporterTestResult): Promise { return this._addTestResult(result, {status: IDLE}); } - addRunning(result: ReporterTestResult): ReporterTestResult { + async addRunning(result: ReporterTestResult): Promise { return this._addTestResult(result, {status: RUNNING}); } - addSkipped(result: ReporterTestResult): ReporterTestResult { - const formattedResult = super.addSkipped(result); + override async addSkipped(result: ReporterTestResult): Promise { + const formattedResult = await super.addSkipped(result); const { fullName: suite, skipReason: comment, @@ -64,8 +64,8 @@ export class GuiReportBuilder extends StaticReportBuilder { return formattedResult; } - addUpdated(result: ReporterTestResult, failResultId: string): ReporterTestResult { - return this._addTestResult(result, {status: UPDATED}, {failResultId}); + async addUpdated(result: ReporterTestResult): Promise { + return this._addTestResult(result, {status: UPDATED}); } setApiValues(values: HtmlReporterValues): this { @@ -75,6 +75,14 @@ export class GuiReportBuilder extends StaticReportBuilder { reuseTestsTree(tree: Tree): void { this._testsTree.reuseTestsTree(tree); + + // Fill test attempt manager with data from db + for (const [, testResult] of Object.entries(tree.results.byId)) { + this._testAttemptManager.registerAttempt({ + fullName: testResult.suitePath.join(HERMIONE_TITLE_DELIMITER), + browserId: testResult.name + }, testResult.status, testResult.attempt); + } } getResult(): GuiReportBuilderResult { @@ -104,7 +112,11 @@ export class GuiReportBuilder extends StaticReportBuilder { return this._testsTree.getImageDataToFindEqualDiffs(imageIds); } - undoAcceptImage(testResult: ReporterTestResult, stateName: string): UndoAcceptImageResult | null { + undoAcceptImage(testResultWithoutAttempt: ReporterTestResult, stateName: string): UndoAcceptImageResult | null { + const attempt = this._testAttemptManager.getCurrentAttempt(testResultWithoutAttempt); + const imagesInfoFormatter = this.imageHandler; + const testResult = copyAndUpdate(testResultWithoutAttempt, {attempt}, {imagesInfoFormatter}); + const resultId = testResult.id; const suitePath = testResult.testPath; const browserName = testResult.browserId; @@ -127,21 +139,19 @@ export class GuiReportBuilder extends StaticReportBuilder { const shouldRemoveReference = _.isNull(previousImageRefImgSize); const shouldRevertReference = !shouldRemoveReference; - let updatedImage: TreeImage | undefined, removedResult: string | undefined, newTestResult: ReporterTestResult; + let updatedImage: TreeImage | undefined, removedResult: ReporterTestResult | undefined; if (shouldRemoveResult) { this._testsTree.removeTestResult(resultId); this._testAttemptManager.removeAttempt(testResult); - newTestResult = copyAndUpdate(testResult, {attempt: this._testAttemptManager.getCurrentAttempt(testResult)}); - - removedResult = resultId; + removedResult = testResult; } else { updatedImage = this._testsTree.updateImageInfo(imageId, previousImage); - - newTestResult = testResult; } + const newResult = copyAndUpdate(testResult, {attempt: this._testAttemptManager.getCurrentAttempt(testResult)}, {imagesInfoFormatter}); + this._deleteTestResultFromDb({where: [ `${DB_COLUMNS.SUITE_PATH} = ?`, `${DB_COLUMNS.NAME} = ?`, @@ -150,11 +160,11 @@ export class GuiReportBuilder extends StaticReportBuilder { `json_extract(${DB_COLUMNS.IMAGES_INFO}, '$[0].stateName') = ?` ].join(' AND ')}, JSON.stringify(suitePath), browserName, status, timestamp.toString(), stateName); - return {updatedImage, removedResult, previousExpectedPath, shouldRemoveReference, shouldRevertReference, newTestResult}; + return {updatedImage, removedResult, previousExpectedPath, shouldRemoveReference, shouldRevertReference, newResult}; } - protected override _addTestResult(formattedResult: ReporterTestResult, props: {status: TestStatus} & Partial, opts: {failResultId?: string} = {}): ReporterTestResult { - super._addTestResult(formattedResult, props); + protected override async _addTestResult(formattedResultOriginal: ReporterTestResult, props: {status: TestStatus} & Partial): Promise { + const formattedResult = await super._addTestResult(formattedResultOriginal, props); const testResult = this._createTestResult(formattedResult, { ...props, @@ -162,22 +172,24 @@ export class GuiReportBuilder extends StaticReportBuilder { attempt: formattedResult.attempt }); - this._extendTestWithImagePaths(testResult, formattedResult, opts); + this._extendTestWithImagePaths(testResult, formattedResult); this._testsTree.addTestResult(testResult, formattedResult); return formattedResult; } - private _extendTestWithImagePaths(testResult: PreparedTestResult, formattedResult: ReporterTestResult, opts: {failResultId?: string} = {}): void { + private _extendTestWithImagePaths(testResult: PreparedTestResult, formattedResult: ReporterTestResult): void { const newImagesInfo = formattedResult.imagesInfo; + const imagesInfoFormatter = this._imageHandler; if (testResult.status !== UPDATED) { _.set(testResult, 'imagesInfo', newImagesInfo); return; } - const failImagesInfo = opts.failResultId ? this._testsTree.getImagesInfo(opts.failResultId) : []; + const failResultId = copyAndUpdate(formattedResult, {attempt: formattedResult.attempt - 1}, {imagesInfoFormatter}).id; + const failImagesInfo = this._testsTree.getImagesInfo(failResultId); if (failImagesInfo.length) { testResult.imagesInfo = _.clone(failImagesInfo); diff --git a/lib/report-builder/static.ts b/lib/report-builder/static.ts index 05c28429f..ee7bb8743 100644 --- a/lib/report-builder/static.ts +++ b/lib/report-builder/static.ts @@ -12,11 +12,11 @@ import { SUCCESS, TestStatus, LOCAL_DATABASE_NAME, - PluginEvents + PluginEvents, UNKNOWN_ATTEMPT, UPDATED } from '../constants'; import type {PreparedTestResult, SqliteClient} from '../sqlite-client'; import {ReporterTestResult} from '../test-adapter'; -import {hasImage, saveStaticFilesToReportDir, writeDatabaseUrlsFile} from '../server-utils'; +import {hasImage, saveErrorDetails, saveStaticFilesToReportDir, writeDatabaseUrlsFile} from '../server-utils'; import {ReporterConfig} from '../types'; import {HtmlReporter} from '../plugin-api'; import {ImageHandler} from '../image-handler'; @@ -25,12 +25,13 @@ import {getUrlWithBase, getError, getRelativeUrl, hasDiff, hasNoRefImageErrors} import {getTestFromDb} from '../db-utils/server'; import {ImageDiffError} from '../errors'; import {TestAttemptManager} from '../test-attempt-manager'; +import {copyAndUpdate} from '../test-adapter/utils'; +import {RegisterWorkers} from '../workers/create-workers'; const ignoredStatuses = [RUNNING, IDLE]; interface StaticReportBuilderOptions { dbClient: SqliteClient; - testAttemptManager: TestAttemptManager; } export class StaticReportBuilder { @@ -39,6 +40,7 @@ export class StaticReportBuilder { protected _dbClient: SqliteClient; protected _imageHandler: ImageHandler; protected _testAttemptManager: TestAttemptManager; + private _workers: RegisterWorkers<['saveDiffTo']> | null; static create( this: new (htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, options: StaticReportBuilderOptions) => T, @@ -49,17 +51,19 @@ export class StaticReportBuilder { return new this(htmlReporter, pluginConfig, options); } - constructor(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, {dbClient, testAttemptManager}: StaticReportBuilderOptions) { + constructor(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, {dbClient}: StaticReportBuilderOptions) { this._htmlReporter = htmlReporter; this._pluginConfig = pluginConfig; this._dbClient = dbClient; - this._testAttemptManager = testAttemptManager; + this._testAttemptManager = new TestAttemptManager(); const imageStore = new SqliteImageStore(this._dbClient); this._imageHandler = new ImageHandler(imageStore, htmlReporter.imagesSaver, {reportPath: pluginConfig.path}); + this._workers = null; + this._htmlReporter.on(PluginEvents.IMAGES_SAVER_UPDATED, (newImagesSaver) => { this._imageHandler.setImagesSaver(newImagesSaver); }); @@ -71,10 +75,6 @@ export class StaticReportBuilder { return this._imageHandler; } - get testAttemptManager(): TestAttemptManager { - return this._testAttemptManager; - } - async saveStaticFiles(): Promise { const destPath = this._pluginConfig.path; @@ -84,26 +84,26 @@ export class StaticReportBuilder { ]); } - addSkipped(result: ReporterTestResult): ReporterTestResult { + async addSkipped(result: ReporterTestResult): Promise { return this._addTestResult(result, { status: SKIPPED, skipReason: result.skipReason }); } - addSuccess(result: ReporterTestResult): ReporterTestResult { + async addSuccess(result: ReporterTestResult): Promise { return this._addTestResult(result, {status: SUCCESS}); } - addFail(result: ReporterTestResult): ReporterTestResult { + async addFail(result: ReporterTestResult): Promise { return this._addFailResult(result); } - addError(result: ReporterTestResult): ReporterTestResult { + async addError(result: ReporterTestResult): Promise { return this._addErrorResult(result); } - addRetry(result: ReporterTestResult): ReporterTestResult { + async addRetry(result: ReporterTestResult): Promise { if (hasDiff(result.assertViewResults as ImageDiffError[])) { return this._addFailResult(result); } else { @@ -111,15 +111,64 @@ export class StaticReportBuilder { } } - protected _addFailResult(formattedResult: ReporterTestResult): ReporterTestResult { + registerWorkers(workers: RegisterWorkers<['saveDiffTo']>): void { + this._workers = workers; + } + + private _ensureWorkers(): RegisterWorkers<['saveDiffTo']> { + if (!this._workers) { + throw new Error('You must register workers before using report builder.' + + 'Make sure registerWorkers() was called before adding any test results.'); + } + + return this._workers; + } + + /** If passed test result doesn't have attempt, this method registers new attempt and sets attempt number */ + private _provideAttempt(testResultOriginal: ReporterTestResult): ReporterTestResult { + let formattedResult = testResultOriginal; + + if (testResultOriginal.attempt === UNKNOWN_ATTEMPT) { + const imagesInfoFormatter = this._imageHandler; + const attempt = this._testAttemptManager.registerAttempt(testResultOriginal, testResultOriginal.status); + formattedResult = copyAndUpdate(testResultOriginal, {attempt}, {imagesInfoFormatter}); + } + + return formattedResult; + } + + private async _saveTestResultData(testResult: ReporterTestResult): Promise { + if ([IDLE, RUNNING, UPDATED].includes(testResult.status)) { + return; + } + + const actions: Promise[] = []; + + if (!_.isEmpty(testResult.assertViewResults)) { + actions.push(this._imageHandler.saveTestImages(testResult, this._ensureWorkers())); + } + + if (this._pluginConfig.saveErrorDetails && testResult.errorDetails) { + actions.push(saveErrorDetails(testResult, this._pluginConfig.path)); + } + + await Promise.all(actions); + } + + protected async _addFailResult(formattedResult: ReporterTestResult): Promise { return this._addTestResult(formattedResult, {status: FAIL}); } - protected _addErrorResult(formattedResult: ReporterTestResult): ReporterTestResult { + protected async _addErrorResult(formattedResult: ReporterTestResult): Promise { return this._addTestResult(formattedResult, {status: ERROR}); } - protected _addTestResult(formattedResult: ReporterTestResult, props: {status: TestStatus} & Partial): ReporterTestResult { + protected async _addTestResult(formattedResultOriginal: ReporterTestResult, props: {status: TestStatus} & Partial): Promise { + const formattedResult = this._provideAttempt(formattedResultOriginal); + + // Test result data has to be saved before writing to db, because user may save data to custom location + await this._saveTestResultData(formattedResult); + formattedResult.image = hasImage(formattedResult); const testResult = this._createTestResult(formattedResult, _.extend(props, { @@ -154,7 +203,7 @@ export class StaticReportBuilder { const testResult: PreparedTestResult = Object.assign({ suiteUrl, name: browserId, metaInfo, description, history, imagesInfo, screenshot: Boolean(screenshot), multipleTabs, - suitePath: testPath + suitePath: testPath, suiteName: _.last(testPath) as string }, props); const error = getError(result.error); diff --git a/lib/reporter-helpers.ts b/lib/reporter-helpers.ts index f0d278a41..f5d4ec2a2 100644 --- a/lib/reporter-helpers.ts +++ b/lib/reporter-helpers.ts @@ -29,10 +29,11 @@ export const updateReferenceImage = async (testResult: ReporterTestResult, repor ]); }; -export const revertReferenceImage = async (referenceId: string, testResult: ReporterTestResult, stateName: string): Promise => { +export const revertReferenceImage = async (removedResult: ReporterTestResult, newResult: ReporterTestResult, stateName: string): Promise => { + const referenceId = removedResult.id; const referenceHash = mkReferenceHash(referenceId, stateName); const oldReferencePath = path.resolve(tmp.tmpdir, referenceHash); - const referencePath = ImageHandler.getRefImg(testResult.assertViewResults, stateName)?.path; + const referencePath = ImageHandler.getRefImg(newResult.assertViewResults, stateName)?.path; if (!referencePath) { return; diff --git a/lib/server-utils.ts b/lib/server-utils.ts index ccd28fab3..1db16430e 100644 --- a/lib/server-utils.ts +++ b/lib/server-utils.ts @@ -319,3 +319,17 @@ export const formatTestResult = ( ): ReporterTestResult => { return new HermioneTestAdapter(rawResult, {attempt, status, imagesInfoFormatter: imageHandler}); }; + +export const saveErrorDetails = async (testResult: ReporterTestResult, reportPath: string): Promise => { + if (!testResult.errorDetails) { + return; + } + + const detailsFilePath = path.resolve(reportPath, testResult.errorDetails.filePath); + const detailsData = _.isObject(testResult.errorDetails.data) + ? JSON.stringify(testResult.errorDetails.data, null, 2) + : testResult.errorDetails.data; + + await makeDirFor(detailsFilePath); + await fs.writeFile(detailsFilePath, detailsData); +}; diff --git a/lib/sqlite-client.ts b/lib/sqlite-client.ts index d399aeb01..4575056f2 100644 --- a/lib/sqlite-client.ts +++ b/lib/sqlite-client.ts @@ -45,6 +45,7 @@ export interface PreparedTestResult { timestamp: number; errorDetails?: ErrorDetails; suitePath: string[]; + suiteName: string; } export interface DbTestResult { diff --git a/lib/static/components/retry-switcher/item.jsx b/lib/static/components/retry-switcher/item.jsx index 326d46172..802b88c54 100644 --- a/lib/static/components/retry-switcher/item.jsx +++ b/lib/static/components/retry-switcher/item.jsx @@ -5,10 +5,8 @@ import {connect} from 'react-redux'; import {get} from 'lodash'; import {ERROR} from '../../../constants'; import { - isAssertViewError, - isFailStatus, - isImageDiffError, - isNoRefImageError + hasUnrelatedToScreenshotsErrors, + isFailStatus } from '../../../common-utils'; class RetrySwitcherItem extends Component { @@ -46,7 +44,7 @@ export default connect( const {status, attempt, error} = result; return { - status: hasUnrelatedToScreenshotsErrors(status, error) ? `${status}_${ERROR}` : status, + status: isFailStatus(status) && hasUnrelatedToScreenshotsErrors(error) ? `${status}_${ERROR}` : status, attempt, keyToGroupTestsBy, matchedSelectedGroup @@ -54,9 +52,3 @@ export default connect( } )(RetrySwitcherItem); -function hasUnrelatedToScreenshotsErrors(status, error) { - return isFailStatus(status) && - !isNoRefImageError(error) && - !isImageDiffError(error) && - !isAssertViewError(error); -} diff --git a/lib/static/modules/reducers/tree/index.js b/lib/static/modules/reducers/tree/index.js index bdcac6e56..0c511a47a 100644 --- a/lib/static/modules/reducers/tree/index.js +++ b/lib/static/modules/reducers/tree/index.js @@ -1,4 +1,4 @@ -import {findLast, isEmpty, pick} from 'lodash'; +import {findLast, isEmpty} from 'lodash'; import {produce} from 'immer'; import actionNames from '../../action-names'; import { diff --git a/lib/test-adapter/hermione.ts b/lib/test-adapter/hermione.ts index 5e99154d8..f307998d9 100644 --- a/lib/test-adapter/hermione.ts +++ b/lib/test-adapter/hermione.ts @@ -1,11 +1,9 @@ import _ from 'lodash'; -import fs from 'fs-extra'; import path from 'path'; import {getCommandsHistory} from '../history-utils'; -import {ERROR_DETAILS_PATH, TestStatus} from '../constants'; +import {TestStatus} from '../constants'; import {wrapLinkByTag} from '../common-utils'; -import * as utils from '../server-utils'; import { AssertViewResult, ErrorDetails, @@ -16,6 +14,7 @@ import { import {ImagesInfoFormatter} from '../image-handler'; import {ReporterTestResult} from './index'; import {getSuitePath} from '../plugin-utils'; +import {extractErrorDetails} from './utils'; const getSkipComment = (suite: HermioneTestResult | HermioneSuite): string | null | undefined => { return suite.skipReason || suite.parent && getSkipComment(suite.parent); @@ -116,10 +115,6 @@ export class HermioneTestAdapter implements ReporterTestResult { return this.testPath.concat(this.browserId, this.attempt.toString()).join(' '); } - get isUpdated(): boolean | undefined { - return this._testResult.updated; - } - get screenshot(): ImageBase64 | undefined { return _.get(this._testResult, 'err.screenshot'); } @@ -137,19 +132,7 @@ export class HermioneTestAdapter implements ReporterTestResult { return this._errorDetails; } - const details = _.get(this._testResult, 'err.details', null); - - if (details) { - this._errorDetails = { - title: details.title || 'error details', - data: details.data, - filePath: `${ERROR_DETAILS_PATH}/${utils.getDetailsFileName( - this._testResult.id, this._testResult.browserId, this.attempt - )}` - }; - } else { - this._errorDetails = null; - } + this._errorDetails = extractErrorDetails(this); return this._errorDetails; } @@ -175,18 +158,4 @@ export class HermioneTestAdapter implements ReporterTestResult { this._timestamp = timestamp; } } - - async saveErrorDetails(reportPath: string): Promise { - if (!this.errorDetails) { - return; - } - - const detailsFilePath = path.resolve(reportPath, this.errorDetails.filePath); - const detailsData = _.isObject(this.errorDetails.data) - ? JSON.stringify(this.errorDetails.data, null, 2) - : this.errorDetails.data; - - await utils.makeDirFor(detailsFilePath); - await fs.writeFile(detailsFilePath, detailsData); - } } diff --git a/lib/test-adapter/index.ts b/lib/test-adapter/index.ts index 71378e66a..ffc2b3456 100644 --- a/lib/test-adapter/index.ts +++ b/lib/test-adapter/index.ts @@ -17,7 +17,6 @@ export interface ReporterTestResult { image?: boolean; readonly imageDir: string; readonly imagesInfo: ImageInfoFull[] | undefined; - readonly isUpdated?: boolean; readonly meta: Record; readonly multipleTabs: boolean; readonly screenshot: ImageBase64 | ImageData | null | undefined; diff --git a/lib/test-adapter/playwright.ts b/lib/test-adapter/playwright.ts index d65ba8bc7..be38db2f6 100644 --- a/lib/test-adapter/playwright.ts +++ b/lib/test-adapter/playwright.ts @@ -241,10 +241,6 @@ export class PlaywrightTestAdapter implements ReporterTestResult { return this._imagesInfoFormatter.getImagesInfo(this); } - get isUpdated(): boolean { - return false; - } - get meta(): Record { return Object.fromEntries(this._testCase.annotations.map(a => [a.type, a.description ?? ''])); } diff --git a/lib/test-adapter/reporter.ts b/lib/test-adapter/reporter.ts new file mode 100644 index 000000000..15f402d4f --- /dev/null +++ b/lib/test-adapter/reporter.ts @@ -0,0 +1,121 @@ +import {TestStatus} from '../constants'; +import {AssertViewResult, TestError, ErrorDetails, ImageInfoFull, ImageBase64, ImageData} from '../types'; +import {ReporterTestResult} from './index'; +import {ImagesInfoFormatter} from '../image-handler'; +import _ from 'lodash'; +import {extractErrorDetails} from './utils'; +import {getShortMD5} from '../common-utils'; + +interface ReporterTestAdapterOptions { + imagesInfoFormatter: ImagesInfoFormatter; +} + +// This class is primarily useful when cloning ReporterTestResult. +// It allows to override some properties while keeping computable +// properties valid, e.g. id + +export class ReporterTestAdapter implements ReporterTestResult { + private _testResult: ReporterTestResult; + private _imagesInfoFormatter: ImagesInfoFormatter; + private _errorDetails: ErrorDetails | null; + + constructor(testResult: ReporterTestResult, {imagesInfoFormatter}: ReporterTestAdapterOptions) { + this._testResult = testResult; + this._imagesInfoFormatter = imagesInfoFormatter; + this._errorDetails = null; + } + + get assertViewResults(): AssertViewResult[] { + return this._testResult.assertViewResults; + } + + get attempt(): number { + return this._testResult.attempt; + } + + get browserId(): string { + return this._testResult.browserId; + } + + get description(): string | undefined { + return this._testResult.description; + } + + get error(): TestError | undefined { + return this._testResult.error; + } + + get errorDetails(): ErrorDetails | null { + if (!_.isNil(this._errorDetails)) { + return this._errorDetails; + } + + this._errorDetails = extractErrorDetails(this); + + return this._errorDetails; + } + + get file(): string { + return this._testResult.file; + } + + get fullName(): string { + return this._testResult.fullName; + } + + get history(): string[] { + return this._testResult.history; + } + + get id(): string { + return this.testPath.concat(this.browserId, this.attempt.toString()).join(' '); + } + + get imageDir(): string { + return getShortMD5(this.fullName); + } + + get imagesInfo(): ImageInfoFull[] | undefined { + return this._imagesInfoFormatter.getImagesInfo(this); + } + + get meta(): Record { + return this._testResult.meta; + } + + get multipleTabs(): boolean { + return this._testResult.multipleTabs; + } + + get screenshot(): ImageBase64 | ImageData | null | undefined { + return this.error?.screenshot; + } + + get sessionId(): string { + return this._testResult.sessionId; + } + + get skipReason(): string | undefined { + return this._testResult.skipReason; + } + + get state(): {name: string;} { + return {name: this.testPath.at(-1) as string}; + } + + get status(): TestStatus { + return this._testResult.status; + } + + get testPath(): string[] { + return this._testResult.testPath; + } + + get timestamp(): number | undefined { + return this._testResult.timestamp; + } + + get url(): string | undefined { + return this._testResult.url; + } +} diff --git a/lib/test-adapter/utils/index.ts b/lib/test-adapter/utils/index.ts index 76d96f36d..48e5c7d29 100644 --- a/lib/test-adapter/utils/index.ts +++ b/lib/test-adapter/utils/index.ts @@ -1,8 +1,17 @@ import _ from 'lodash'; import {ReporterTestResult} from '../index'; import {TupleToUnion} from 'type-fest'; +import {ErrorDetails} from '../../types'; +import {ERROR_DETAILS_PATH} from '../../constants'; +import * as utils from '../../server-utils'; +import {ReporterTestAdapter} from '../reporter'; +import {ImagesInfoFormatter} from '../../image-handler'; -export const copyAndUpdate = (original: ReporterTestResult, updates: Partial): ReporterTestResult => { +export const copyAndUpdate = ( + original: ReporterTestResult, + updates: Partial, + {imagesInfoFormatter}: {imagesInfoFormatter: ImagesInfoFormatter} +): ReporterTestResult => { const keys = [ 'assertViewResults', 'attempt', @@ -17,7 +26,6 @@ export const copyAndUpdate = (original: ReporterTestResult, updates: Partial { + const details = testResult.error?.details ?? null; + + if (details) { + return { + title: details.title || 'error details', + data: details.data, + filePath: `${ERROR_DETAILS_PATH}/${utils.getDetailsFileName( + testResult.imageDir, testResult.browserId, testResult.attempt + )}` + }; + } + + return null; }; diff --git a/lib/tests-tree-builder/gui.ts b/lib/tests-tree-builder/gui.ts index 0a660ece4..83c74ec59 100644 --- a/lib/tests-tree-builder/gui.ts +++ b/lib/tests-tree-builder/gui.ts @@ -2,7 +2,6 @@ import _ from 'lodash'; import {BaseTestsTreeBuilder, Tree, TreeImage, TreeResult, TreeSuite} from './base'; import {TestStatus, UPDATED} from '../constants'; import {isUpdatedStatus} from '../common-utils'; -import {ReporterTestResult} from '../test-adapter'; import {ImageInfoFail, ImageInfoWithState} from '../types'; interface SuiteBranch { @@ -235,18 +234,4 @@ export class GuiTestsTreeBuilder extends BaseTestsTreeBuilder { this._reuseSuiteStatus(testsTree, suite.parentId); } - - protected _buildBrowserId(formattedResult: Pick): string { - const {testPath, browserId: browserName} = formattedResult; - const suiteId = this._buildId(testPath); - const browserId = this._buildId(suiteId, browserName); - - return browserId; - } - - protected _buildResultId(formattedResult: ReporterTestResult, attempt: number): string { - const browserId = this._buildBrowserId(formattedResult); - - return `${browserId} ${attempt}`; - } } diff --git a/lib/tests-tree-builder/static.ts b/lib/tests-tree-builder/static.ts index b053d46f4..2c1ffdb5e 100644 --- a/lib/tests-tree-builder/static.ts +++ b/lib/tests-tree-builder/static.ts @@ -201,6 +201,7 @@ function mkTestResult(row: RawSuitesRow, data: {attempt: number}): ParsedSuitesR name: row[DB_COLUMN_INDEXES.name] as string, screenshot: Boolean(row[DB_COLUMN_INDEXES.screenshot]), status: row[DB_COLUMN_INDEXES.status] as TestStatus, + suiteName: row[DB_COLUMN_INDEXES.suiteName] as string, suitePath: JSON.parse(row[DB_COLUMN_INDEXES.suitePath] as string), suiteUrl: row[DB_COLUMN_INDEXES.suiteUrl] as string, skipReason: row[DB_COLUMN_INDEXES.skipReason] as string, diff --git a/lib/types.ts b/lib/types.ts index d74b76ee8..8c1d5db43 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -12,7 +12,6 @@ export {Suite as HermioneSuite} from 'hermione'; export interface HermioneTestResult extends HermioneTestResultOriginal { timestamp?: number; - updated?: boolean; } export interface ImagesSaver { @@ -98,7 +97,7 @@ export type ImageInfo = | Omit | Omit; -export type AssertViewResult = AssertViewSuccess | ImageDiffError | NoRefImageError; +export type AssertViewResult = (AssertViewSuccess | ImageDiffError | NoRefImageError) & {isUpdated?: boolean}; export interface TestError { name: string; @@ -133,6 +132,7 @@ export interface ParsedSuitesRow { screenshot: boolean; skipReason?: string; status: TestStatus; + suiteName: string; suitePath: string[]; suiteUrl: string; timestamp: number; diff --git a/playwright.ts b/playwright.ts index 285628a1b..03464bd9e 100644 --- a/playwright.ts +++ b/playwright.ts @@ -1,7 +1,10 @@ import {promisify} from 'util'; + +import {EventEmitter} from 'events'; import _ from 'lodash'; -import type {Reporter, TestCase, TestResult as PwtTestResult} from '@playwright/test/reporter'; +import PQueue from 'p-queue'; import workerFarm, {Workers} from 'worker-farm'; +import type {Reporter, TestCase, TestResult as PwtTestResult} from '@playwright/test/reporter'; import {StaticReportBuilder} from './lib/report-builder/static'; import {HtmlReporter} from './lib/plugin-api'; @@ -9,11 +12,8 @@ import {ReporterConfig} from './lib/types'; import {parseConfig} from './lib/config'; import {PluginEvents, TestStatus, ToolName} from './lib/constants'; import {RegisterWorkers} from './lib/workers/create-workers'; -import {EventEmitter} from 'events'; import {PlaywrightTestAdapter, getStatus} from './lib/test-adapter/playwright'; -import PQueue from 'p-queue'; import {SqliteClient} from './lib/sqlite-client'; -import {TestAttemptManager} from './lib/test-attempt-manager'; export {ReporterConfig} from './lib/types'; @@ -39,9 +39,10 @@ class MyReporter implements Reporter { this._initPromise = (async (htmlReporter: HtmlReporter, config: ReporterConfig): Promise => { const dbClient = await SqliteClient.create({htmlReporter, reportPath: config.path}); - const testAttemptManager = new TestAttemptManager(); - this._staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient, testAttemptManager}); + this._staticReportBuilder = StaticReportBuilder.create(htmlReporter, config, {dbClient}); + this._staticReportBuilder.registerWorkers(workers); + await this._staticReportBuilder.saveStaticFiles(); })(this._htmlReporter, this._config); @@ -59,16 +60,15 @@ class MyReporter implements Reporter { if (status === TestStatus.FAIL) { if (formattedResult.status === TestStatus.FAIL) { - staticReportBuilder.addFail(formattedResult); + await staticReportBuilder.addFail(formattedResult); } else { - staticReportBuilder.addError(formattedResult); + await staticReportBuilder.addError(formattedResult); } } else if (status === TestStatus.SUCCESS) { - staticReportBuilder.addSuccess(formattedResult); + await staticReportBuilder.addSuccess(formattedResult); } else if (status === TestStatus.SKIPPED) { - staticReportBuilder.addSkipped(formattedResult); + await staticReportBuilder.addSkipped(formattedResult); } - await staticReportBuilder.imageHandler.saveTestImages(formattedResult, this._workers); }); } diff --git a/test/unit/hermione.js b/test/unit/hermione.js index e54082ae8..6dd21fdbf 100644 --- a/test/unit/hermione.js +++ b/test/unit/hermione.js @@ -5,7 +5,7 @@ const Database = require('better-sqlite3'); const fsOriginal = require('fs-extra'); const proxyquire = require('proxyquire').noPreserveCache(); const {logger} = require('lib/common-utils'); -const {stubTool} = require('./utils'); +const {stubTool, NoRefImageError, ImageDiffError} = require('./utils'); const mkSqliteDb = () => { const instance = sinon.createStubInstance(Database); @@ -75,24 +75,6 @@ describe('lib/hermione', () => { AFTER_TESTS_READ: 'afterTestsRead' }; - class ImageDiffError extends Error { - name = 'ImageDiffError'; - constructor() { - super(); - this.stateName = ''; - this.currImg = { - path: '' - }; - this.refImg = { - path: '' - }; - } - } - - class NoRefImageError extends Error { - name = 'NoRefImageError'; - } - function mkHermione_() { return stubTool({ forBrowser: sinon.stub().returns({ @@ -178,11 +160,11 @@ describe('lib/hermione', () => { sandbox.stub(logger, 'log'); sandbox.stub(logger, 'warn'); - sandbox.stub(StaticReportBuilder.prototype, 'addSkipped'); - sandbox.stub(StaticReportBuilder.prototype, 'addSuccess'); - sandbox.stub(StaticReportBuilder.prototype, 'addError'); - sandbox.stub(StaticReportBuilder.prototype, 'addFail'); - sandbox.stub(StaticReportBuilder.prototype, 'addRetry'); + sandbox.stub(StaticReportBuilder.prototype, 'addSkipped').callsFake(_.identity); + sandbox.stub(StaticReportBuilder.prototype, 'addSuccess').callsFake(_.identity); + sandbox.stub(StaticReportBuilder.prototype, 'addError').callsFake(_.identity); + sandbox.stub(StaticReportBuilder.prototype, 'addFail').callsFake(_.identity); + sandbox.stub(StaticReportBuilder.prototype, 'addRetry').callsFake(_.identity); sandbox.stub(StaticReportBuilder.prototype, 'saveStaticFiles'); sandbox.stub(StaticReportBuilder.prototype, 'finalize'); @@ -242,7 +224,7 @@ describe('lib/hermione', () => { hermione.emit(events[event], testResult); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addError.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addFail.args[0][0].state, {name: 'some-title'}); }); it(`errored assert view to result on ${event} event`, async () => { @@ -253,7 +235,7 @@ describe('lib/hermione', () => { hermione.emit(events[event], mkStubResult_({title: 'some-title', assertViewResults: [err]})); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.deepEqual(StaticReportBuilder.prototype.addError.args[0][0].state, {name: 'some-title'}); + assert.deepEqual(StaticReportBuilder.prototype.addFail.args[0][0].state, {name: 'some-title'}); }); it(`failed test to result on ${event} event`, async () => { @@ -285,83 +267,4 @@ describe('lib/hermione', () => { }); }); }); - - it('should save image from passed test', async () => { - utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); - - await initReporter_({path: '/absolute'}); - const testData = mkStubResult_({assertViewResults: [{refImg: {path: 'ref/path'}, stateName: 'plain'}]}); - hermione.emit(events.TEST_PASS, testData); - await hermione.emitAsync(events.RUNNER_END); - - assert.calledOnceWith(utils.copyFileAsync, 'ref/path', 'report/plain', {reportDir: '/absolute'}); - }); - - it('should save image from assert view error', async () => { - utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); - await initReporter_({path: '/absolute'}); - const err = new NoRefImageError(); - err.stateName = 'plain'; - err.currImg = {path: 'current/path'}; - - hermione.emit(events.RETRY, mkStubResult_({assertViewResults: [err]})); - await hermione.emitAsync(events.RUNNER_END); - - assert.calledOnceWith(utils.copyFileAsync, 'current/path', 'report/plain', {reportDir: '/absolute'}); - }); - - it('should save reference image from assert view fail', async () => { - utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); - await initReporter_({path: '/absolute'}); - await stubWorkers(); - - const err = new ImageDiffError(); - err.stateName = 'plain'; - err.refImg = {path: 'reference/path'}; - - hermione.emit(events.TEST_FAIL, mkStubResult_({assertViewResults: [err]})); - await hermione.emitAsync(events.RUNNER_END); - - assert.calledWith(utils.copyFileAsync, 'reference/path', 'report/plain', {reportDir: '/absolute'}); - }); - - it('should save current image from assert view fail', async () => { - utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); - await initReporter_({path: '/absolute'}); - await hermione.emitAsync(events.RUNNER_START, { - registerWorkers: () => { - return {saveDiffTo: sandbox.stub()}; - } - }); - const err = new ImageDiffError(); - err.stateName = 'plain'; - err.currImg = {path: 'current/path'}; - - hermione.emit(events.TEST_FAIL, mkStubResult_({assertViewResults: [err]})); - await hermione.emitAsync(events.RUNNER_END); - - assert.calledWith(utils.copyFileAsync, 'current/path', 'report/plain', {reportDir: '/absolute'}); - }); - - it('should save current diff image from assert view fail', async () => { - fs.readFile.resolves(Buffer.from('some-buff')); - utils.getDiffPath.callsFake(({stateName}) => `report/${stateName}`); - const saveDiffTo = sinon.stub().resolves(); - const err = new ImageDiffError(); - err.stateName = 'plain'; - - await initReporter_(); - - await hermione.emitAsync(events.RUNNER_START, { - registerWorkers: () => { - return {saveDiffTo}; - } - }); - hermione.emit(events.TEST_FAIL, mkStubResult_({assertViewResults: [err]})); - await hermione.emitAsync(events.RUNNER_END); - - assert.calledWith( - saveDiffTo, sinon.match.instanceOf(ImageDiffError), sinon.match('/report/plain') - ); - }); }); diff --git a/test/unit/lib/gui/tool-runner/index.js b/test/unit/lib/gui/tool-runner/index.js index 759627530..f9949ab97 100644 --- a/test/unit/lib/gui/tool-runner/index.js +++ b/test/unit/lib/gui/tool-runner/index.js @@ -9,8 +9,7 @@ const {LOCAL_DATABASE_NAME} = require('lib/constants/database'); const {logger} = require('lib/common-utils'); const {stubTool, stubConfig, mkImagesInfo, mkState, mkSuite} = require('test/unit/utils'); const {SqliteClient} = require('lib/sqlite-client'); -const {TestAttemptManager} = require('lib/test-attempt-manager'); -const {PluginEvents} = require('lib/constants'); +const {PluginEvents, TestStatus} = require('lib/constants'); describe('lib/gui/tool-runner/index', () => { const sandbox = sinon.createSandbox(); @@ -24,7 +23,6 @@ describe('lib/gui/tool-runner/index', () => { let revertReferenceImage; let toolRunnerUtils; let createTestRunner; - let testAttemptManager; const mkTestCollection_ = (testsTree = {}) => { return { @@ -73,15 +71,13 @@ describe('lib/gui/tool-runner/index', () => { createTestRunner = sinon.stub(); - testAttemptManager = new TestAttemptManager(); - toolRunnerUtils = { findTestResult: sandbox.stub(), formatId: sandbox.stub().returns('some-id') }; reportBuilder = sinon.createStubInstance(GuiReportBuilder); - sandbox.stub(reportBuilder, 'testAttemptManager').get(() => testAttemptManager); + reportBuilder.addUpdated.callsFake(_.identity); subscribeOnToolEvents = sandbox.stub().named('reportSubscriber').resolves(); looksSame = sandbox.stub().named('looksSame').resolves({equal: true}); @@ -320,7 +316,10 @@ describe('lib/gui/tool-runner/index', () => { const mkUndoTestData_ = async (stubResult, {stateName = 'plain'} = {}) => { reportBuilder.undoAcceptImage.withArgs(sinon.match({ fullName: 'some-title' - }), 'plain').returns(stubResult); + }), 'plain').returns({ + newResult: {fullName: 'some-title'}, + ...stubResult + }); const tests = [{ id: 'some-id', fullTitle: () => 'some-title', @@ -333,7 +332,8 @@ describe('lib/gui/tool-runner/index', () => { stateName, actualImg: { size: {height: 100, width: 200} - } + }, + status: TestStatus.UPDATED }) ] }]; diff --git a/test/unit/lib/gui/tool-runner/report-subsciber.js b/test/unit/lib/gui/tool-runner/report-subsciber.js index 858e6685b..5616a8ae0 100644 --- a/test/unit/lib/gui/tool-runner/report-subsciber.js +++ b/test/unit/lib/gui/tool-runner/report-subsciber.js @@ -8,13 +8,11 @@ const {GuiReportBuilder} = require('lib/report-builder/gui'); const {ClientEvents} = require('lib/gui/constants'); const {stubTool, stubConfig} = require('test/unit/utils'); const {HermioneTestAdapter} = require('lib/test-adapter'); -const {ErrorName} = require('lib/errors'); -const {TestAttemptManager} = require('lib/test-attempt-manager'); -const {FAIL} = require('lib/constants'); +const {UNKNOWN_ATTEMPT} = require('lib/constants'); describe('lib/gui/tool-runner/hermione/report-subscriber', () => { const sandbox = sinon.createSandbox(); - let reportBuilder, testAttemptManager; + let reportBuilder; let client; const events = { @@ -36,12 +34,13 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { beforeEach(() => { reportBuilder = sinon.createStubInstance(GuiReportBuilder); - - testAttemptManager = new TestAttemptManager(); + reportBuilder.addFail.callsFake(_.identity); + reportBuilder.addError.callsFake(_.identity); + reportBuilder.addRunning.callsFake(_.identity); + reportBuilder.addSkipped.callsFake(_.identity); sandbox.stub(GuiReportBuilder, 'create').returns(reportBuilder); sandbox.stub(reportBuilder, 'imageHandler').value({saveTestImages: sinon.stub()}); - sandbox.stub(reportBuilder, 'testAttemptManager').value(testAttemptManager); sandbox.stub(HermioneTestAdapter.prototype, 'id').value('some-id'); client = new EventEmitter(); @@ -65,7 +64,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { const testResult = mkHermioneTestResult(); const mediator = sinon.spy().named('mediator'); - reportBuilder.imageHandler.saveTestImages.callsFake(() => Promise.delay(100).then(mediator)); + reportBuilder.addError.callsFake(() => Promise.delay(100).then(mediator).then(() => ({id: 'some-id'}))); subscribeOnToolEvents(hermione, reportBuilder, client); hermione.emit(hermione.events.TEST_FAIL, testResult); @@ -76,16 +75,18 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { }); describe('TEST_BEGIN', () => { - it('should emit "BEGIN_STATE" event for client with correct data', () => { + it('should emit "BEGIN_STATE" event for client with correct data', async () => { const hermione = mkHermione_(); const testResult = mkHermioneTestResult(); + reportBuilder.addRunning.resolves({id: 'some-id'}); reportBuilder.getTestBranch.withArgs('some-id').returns('test-tree-branch'); subscribeOnToolEvents(hermione, reportBuilder, client); hermione.emit(hermione.events.TEST_BEGIN, testResult); + await hermione.emitAsync(hermione.events.RUNNER_END); - assert.calledOnceWith(client.emit, ClientEvents.BEGIN_STATE, 'test-tree-branch'); + assert.calledWith(client.emit, ClientEvents.BEGIN_STATE, 'test-tree-branch'); }); }); @@ -101,7 +102,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { assert.calledOnceWith(reportBuilder.addSkipped, sinon.match({ fullName: 'some-title', browserId: 'some-browser', - attempt: 0 + attempt: UNKNOWN_ATTEMPT })); }); @@ -120,30 +121,6 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { }); describe('TEST_FAIL', () => { - it('should add correct attempt', async () => { - const hermione = mkHermione_(); - const testResult = mkHermioneTestResult({assertViewResults: [{name: ErrorName.IMAGE_DIFF}]}); - - testAttemptManager.registerAttempt({fullName: testResult.fullTitle(), browserId: testResult.browserId}, FAIL); - - subscribeOnToolEvents(hermione, reportBuilder, client); - hermione.emit(hermione.events.TEST_FAIL, testResult); - await hermione.emitAsync(hermione.events.RUNNER_END); - - assert.calledWithMatch(reportBuilder.addFail, {attempt: 1}); - }); - - it('should save images before fail adding', async () => { - const hermione = mkHermione_(); - const testResult = mkHermioneTestResult({assertViewResults: [{name: ErrorName.IMAGE_DIFF}]}); - - subscribeOnToolEvents(hermione, reportBuilder, client); - hermione.emit(hermione.events.TEST_FAIL, testResult); - await hermione.emitAsync(hermione.events.RUNNER_END); - - assert.callOrder(reportBuilder.imageHandler.saveTestImages, reportBuilder.addFail); - }); - it('should emit "TEST_RESULT" event for client with test data', async () => { const hermione = mkHermione_(); const testResult = mkHermioneTestResult(); diff --git a/test/unit/lib/report-builder/gui.js b/test/unit/lib/report-builder/gui.js index 55ebe87a0..a269c6c71 100644 --- a/test/unit/lib/report-builder/gui.js +++ b/test/unit/lib/report-builder/gui.js @@ -10,15 +10,16 @@ const {GuiTestsTreeBuilder} = require('lib/tests-tree-builder/gui'); const {HtmlReporter} = require('lib/plugin-api'); const {SUCCESS, FAIL, ERROR, SKIPPED, IDLE, RUNNING, UPDATED} = require('lib/constants/test-statuses'); const {LOCAL_DATABASE_NAME} = require('lib/constants/database'); -const {ErrorName} = require('lib/errors'); const {TestAttemptManager} = require('lib/test-attempt-manager'); +const {ImageDiffError} = require('../../utils'); +const {ImageHandler} = require('lib/image-handler'); const TEST_REPORT_PATH = 'test'; const TEST_DB_PATH = `${TEST_REPORT_PATH}/${LOCAL_DATABASE_NAME}`; describe('GuiReportBuilder', () => { const sandbox = sinon.sandbox.create(); - let hasImage, deleteFile, GuiReportBuilder, dbClient, testAttemptManager; + let hasImage, deleteFile, GuiReportBuilder, dbClient, testAttemptManager, copyAndUpdate; const mkGuiReportBuilder_ = async ({toolConfig, pluginConfig} = {}) => { toolConfig = _.defaults(toolConfig || {}, {getAbsoluteUrl: _.noop}); @@ -39,6 +40,9 @@ describe('GuiReportBuilder', () => { const reportBuilder = GuiReportBuilder.create(hermione.htmlReporter, pluginConfig, {dbClient, testAttemptManager}); + const workers = {saveDiffTo: () => {}}; + reportBuilder.registerWorkers(workers); + return reportBuilder; }; @@ -79,15 +83,23 @@ describe('GuiReportBuilder', () => { sandbox.stub(fs, 'writeFileSync'); sandbox.stub(serverUtils, 'prepareCommonJSData'); + copyAndUpdate = sandbox.stub().callsFake(_.identity); + + const imageHandler = sandbox.createStubInstance(ImageHandler); + hasImage = sandbox.stub().returns(true); deleteFile = sandbox.stub().resolves(); GuiReportBuilder = proxyquire('lib/report-builder/gui', { './static': { StaticReportBuilder: proxyquire('lib/report-builder/static', { - '../sqlite-client': {SqliteClient} + '../sqlite-client': {SqliteClient}, + '../image-handler': {ImageHandler: function() { + return imageHandler; + }} }).StaticReportBuilder }, - '../server-utils': {hasImage, deleteFile} + '../server-utils': {hasImage, deleteFile}, + '../test-adapter/utils': {copyAndUpdate} }).GuiReportBuilder; sandbox.stub(GuiTestsTreeBuilder, 'create').returns(Object.create(GuiTestsTreeBuilder.prototype)); @@ -112,7 +124,7 @@ describe('GuiReportBuilder', () => { it(`should add "${IDLE}" status to result`, async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addIdle(stubTest_()); + await reportBuilder.addIdle(stubTest_()); assert.equal(getTestResult_().status, IDLE); }); @@ -122,7 +134,7 @@ describe('GuiReportBuilder', () => { it(`should add "${RUNNING}" status to result`, async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addRunning(stubTest_()); + await reportBuilder.addRunning(stubTest_()); assert.equal(getTestResult_().status, RUNNING); }); @@ -132,7 +144,7 @@ describe('GuiReportBuilder', () => { it('should add skipped test to results', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addSkipped(stubTest_({ + await reportBuilder.addSkipped(stubTest_({ browserId: 'bro1', skipReason: 'some skip comment', fullName: 'suite-full-name' @@ -151,7 +163,7 @@ describe('GuiReportBuilder', () => { it('should add success test to result', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addSuccess(stubTest_({ + await reportBuilder.addSuccess(stubTest_({ browserId: 'bro1' })); @@ -166,7 +178,7 @@ describe('GuiReportBuilder', () => { it('should add failed test to result', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addFail(stubTest_({ + await reportBuilder.addFail(stubTest_({ browserId: 'bro1', imageDir: 'some-image-dir' })); @@ -182,7 +194,7 @@ describe('GuiReportBuilder', () => { it('should add error test to result', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addError(stubTest_({error: {message: 'some-error-message'}})); + await reportBuilder.addError(stubTest_({error: {message: 'some-error-message'}})); assert.match(getTestResult_(), { status: ERROR, @@ -195,7 +207,7 @@ describe('GuiReportBuilder', () => { it('should add "fail" status to result if test result has not equal images', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addRetry(stubTest_({assertViewResults: [{name: ErrorName.IMAGE_DIFF}]})); + await reportBuilder.addRetry(stubTest_({assertViewResults: [new ImageDiffError()]})); assert.equal(getTestResult_().status, FAIL); }); @@ -203,31 +215,32 @@ describe('GuiReportBuilder', () => { it('should add "error" status to result if test result has no image', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addRetry(stubTest_({assertViewResults: [{name: 'some-error-name'}]})); + await reportBuilder.addRetry(stubTest_({assertViewResults: [{name: 'some-error-name'}]})); assert.equal(getTestResult_().status, ERROR); }); }); describe('"addUpdated" method', () => { - it(`should add "${SUCCESS}" status to result if all images updated`, async () => { + it(`should add "${UPDATED}" status to result if all images updated`, async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addUpdated(stubTest_({imagesInfo: [{status: UPDATED}]})); + await reportBuilder.addUpdated(stubTest_({testPath: [], imagesInfo: [{status: UPDATED}]})); - assert.equal(getTestResult_().status, SUCCESS); + assert.equal(getTestResult_().status, UPDATED); }); - it(`should correctly determine the status based on current result`, async () => { + it(`should add "${UPDATED}" status even if result has errors`, async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addUpdated(stubTest_({ + await reportBuilder.addUpdated(stubTest_({ + testPath: [], error: {name: 'some-error', message: 'some-message'}, imagesInfo: [{status: FAIL}], attempt: 4 })); - assert.equal(getTestResult_().status, ERROR); + assert.equal(getTestResult_().status, UPDATED); }); it('should update test image for current state name', async () => { @@ -244,12 +257,13 @@ describe('GuiReportBuilder', () => { imagesInfo: [ {stateName: 'plain1', status: UPDATED}, {stateName: 'plain2', status: FAIL} - ] + ], + testPath: [] }); - reportBuilder.addFail(failedTest); + await reportBuilder.addFail(failedTest); GuiTestsTreeBuilder.prototype.getImagesInfo.returns(failedTest.imagesInfo); - reportBuilder.addUpdated(updatedTest); + await reportBuilder.addUpdated(updatedTest); const updatedTestResult = GuiTestsTreeBuilder.prototype.addTestResult.secondCall.args[0]; @@ -270,12 +284,13 @@ describe('GuiReportBuilder', () => { id: 'result-2', imagesInfo: [ {status: UPDATED} - ] + ], + testPath: [] }); - reportBuilder.addFail(failedTest); + await reportBuilder.addFail(failedTest); GuiTestsTreeBuilder.prototype.getImagesInfo.returns(failedTest.imagesInfo); - reportBuilder.addUpdated(updatedTest, 'result-2'); + await reportBuilder.addUpdated(updatedTest, 'result-2'); const {imagesInfo} = GuiTestsTreeBuilder.prototype.addTestResult.secondCall.args[0]; @@ -297,7 +312,7 @@ describe('GuiReportBuilder', () => { [ { method: 'reuseTestsTree', - arg: 'some-tree' + arg: {results: {byId: {}}} }, { method: 'getTestBranch', @@ -412,7 +427,7 @@ describe('GuiReportBuilder', () => { assert.calledOnceWith(GuiTestsTreeBuilder.prototype.removeTestResult, 'result-id'); }); - it('should resolve removed result id', async () => { + it('should resolve removed result', async () => { const resultId = 'result-id'; const stateName = 's-name'; const formattedResult = mkFormattedResultStub_({id: resultId, stateName}); @@ -420,7 +435,7 @@ describe('GuiReportBuilder', () => { const {removedResult} = await reportBuilder.undoAcceptImage(formattedResult, stateName); - assert.deepEqual(removedResult, 'result-id'); + assert.deepEqual(removedResult, formattedResult); }); it('should update image info if "shouldRemoveResult" is false', async () => { @@ -507,7 +522,7 @@ describe('GuiReportBuilder', () => { it('"suiteUrl" field', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addSuccess(stubTest_({ + await reportBuilder.addSuccess(stubTest_({ url: 'some-url' })); @@ -517,7 +532,7 @@ describe('GuiReportBuilder', () => { it('"name" field as browser id', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addSuccess(stubTest_({browserId: 'yabro'})); + await reportBuilder.addSuccess(stubTest_({browserId: 'yabro'})); assert.equal(getTestResult_().name, 'yabro'); }); @@ -525,7 +540,7 @@ describe('GuiReportBuilder', () => { it('"metaInfo" field', async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addSuccess(stubTest_({ + await reportBuilder.addSuccess(stubTest_({ meta: {some: 'value', sessionId: '12345'}, file: '/path/file.js', url: '/test/url' @@ -545,7 +560,7 @@ describe('GuiReportBuilder', () => { it(`add "${name}" field`, async () => { const reportBuilder = await mkGuiReportBuilder_(); - reportBuilder.addSuccess(stubTest_({[name]: value})); + await reportBuilder.addSuccess(stubTest_({[name]: value})); assert.deepEqual(getTestResult_()[name], value); }); @@ -556,7 +571,7 @@ describe('GuiReportBuilder', () => { const reportBuilder = await mkGuiReportBuilder_({pluginConfig: {saveErrorDetails: false}}); const errorDetails = {title: 'some-title', filePath: 'some-path'}; - reportBuilder.addFail(stubTest_({errorDetails})); + await reportBuilder.addFail(stubTest_({errorDetails})); assert.isUndefined(getTestResult_().errorDetails); }); @@ -565,7 +580,7 @@ describe('GuiReportBuilder', () => { const reportBuilder = await mkGuiReportBuilder_({pluginConfig: {saveErrorDetails: true}}); const errorDetails = {title: 'some-title', filePath: 'some-path'}; - reportBuilder.addFail(stubTest_({errorDetails})); + await reportBuilder.addFail(stubTest_({errorDetails})); assert.deepEqual(getTestResult_().errorDetails, errorDetails); }); diff --git a/test/unit/lib/report-builder/static.js b/test/unit/lib/report-builder/static.js index ba80e2f5f..081dd5c10 100644 --- a/test/unit/lib/report-builder/static.js +++ b/test/unit/lib/report-builder/static.js @@ -8,6 +8,9 @@ const {HtmlReporter} = require('lib/plugin-api'); const {SUCCESS, FAIL, ERROR, SKIPPED} = require('lib/constants/test-statuses'); const {LOCAL_DATABASE_NAME} = require('lib/constants/database'); const {SqliteClient} = require('lib/sqlite-client'); +const {NoRefImageError, ImageDiffError} = require('../../utils'); +const sinon = require('sinon'); +const path = require('path'); const TEST_REPORT_PATH = 'test'; const TEST_DB_PATH = `${TEST_REPORT_PATH}/${LOCAL_DATABASE_NAME}`; @@ -15,6 +18,7 @@ const TEST_DB_PATH = `${TEST_REPORT_PATH}/${LOCAL_DATABASE_NAME}`; describe('StaticReportBuilder', () => { const sandbox = sinon.sandbox.create(); let StaticReportBuilder, htmlReporter, dbClient; + let cacheExpectedPaths = new Map(), cacheAllImages = new Map(), cacheDiffImages = new Map(); const fs = _.clone(fsOriginal); @@ -23,18 +27,34 @@ describe('StaticReportBuilder', () => { }); const utils = _.clone(originalUtils); - const mkStaticReportBuilder_ = async ({pluginConfig} = {}) => { + const {ImageHandler} = proxyquire('lib/image-handler', { + 'fs-extra': fs, + './image-cache': {cacheExpectedPaths, cacheAllImages, cacheDiffImages}, + './server-utils': utils + }); + + const {LocalImagesSaver} = proxyquire('lib/local-images-saver', { + './server-utils': utils + }); + + const mkStaticReportBuilder_ = async ({pluginConfig, workers} = {}) => { pluginConfig = _.defaults(pluginConfig, {baseHost: '', path: TEST_REPORT_PATH, baseTestPath: ''}); htmlReporter = _.extend(HtmlReporter.create({baseHost: ''}), { reportsSaver: { saveReportData: sandbox.stub() - } + }, + imagesSaver: LocalImagesSaver }); dbClient = await SqliteClient.create({htmlReporter, reportPath: TEST_REPORT_PATH}); - return StaticReportBuilder.create(htmlReporter, pluginConfig, {dbClient}); + const reportBuilder = StaticReportBuilder.create(htmlReporter, pluginConfig, {dbClient}); + workers = workers ?? {saveDiffTo: () => {}}; + + reportBuilder.registerWorkers(workers); + + return reportBuilder; }; const stubTest_ = (opts = {}) => { @@ -52,11 +72,16 @@ describe('StaticReportBuilder', () => { StaticReportBuilder = proxyquire('lib/report-builder/static', { 'fs-extra': fs, - '../server-utils': utils + '../server-utils': utils, + '../image-handler': {ImageHandler} }).StaticReportBuilder; }); afterEach(() => { + cacheAllImages.clear(); + cacheExpectedPaths.clear(); + cacheDiffImages.clear(); + fs.removeSync(TEST_DB_PATH); sandbox.restore(); }); @@ -129,6 +154,121 @@ describe('StaticReportBuilder', () => { }); }); + describe('saving images', () => { + let reportBuilder, saveDiffTo; + + beforeEach(async () => { + saveDiffTo = sinon.stub(); + + reportBuilder = await mkStaticReportBuilder_({workers: {saveDiffTo}}); + + sandbox.stub(utils, 'getReferencePath'); + sandbox.stub(utils, 'getCurrentPath'); + sandbox.stub(utils, 'getDiffPath').returns('diff'); + sandbox.stub(utils, 'copyFileAsync'); + sandbox.stub(utils, 'makeDirFor'); + + sandbox.stub(fs, 'readFile').resolves(Buffer.from('')); + }); + + it('should save image from passed test', async () => { + utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); + + await reportBuilder.addSuccess(stubTest_({assertViewResults: [{refImg: {path: 'ref/path'}, stateName: 'plain'}]})); + + assert.calledOnceWith(utils.copyFileAsync, 'ref/path', 'report/plain', {reportDir: 'test'}); + }); + + it('should save image from assert view error', async () => { + utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); + + const err = new NoRefImageError(); + err.stateName = 'plain'; + err.currImg = {path: 'current/path'}; + + await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); + + assert.calledOnceWith(utils.copyFileAsync, 'current/path', 'report/plain', {reportDir: 'test'}); + }); + + it('should save reference image from assert view fail', async () => { + utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); + + const err = new ImageDiffError(); + err.stateName = 'plain'; + err.refImg = {path: 'reference/path'}; + + await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); + + assert.calledWith(utils.copyFileAsync, 'reference/path', 'report/plain', {reportDir: 'test'}); + }); + + it('should save current image from assert view fail', async () => { + utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); + + const err = new ImageDiffError(); + err.stateName = 'plain'; + err.currImg = {path: 'current/path'}; + + await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); + + assert.calledWith(utils.copyFileAsync, 'current/path', 'report/plain', {reportDir: 'test'}); + }); + + it('should save current diff image from assert view fail', async () => { + fs.readFile.resolves(Buffer.from('some-buff')); + utils.getDiffPath.callsFake(({stateName}) => `report/${stateName}`); + + const err = new ImageDiffError(); + err.stateName = 'plain'; + + await reportBuilder.addFail(stubTest_({assertViewResults: [err]})); + + assert.calledWith( + saveDiffTo, sinon.match.instanceOf(ImageDiffError), sinon.match('/report/plain') + ); + }); + }); + + describe('saving error details', () => { + let reportBuilder; + + beforeEach(async () => { + reportBuilder = await mkStaticReportBuilder_({pluginConfig: {saveErrorDetails: true}}); + + sandbox.stub(utils, 'makeDirFor').resolves(); + sandbox.stub(utils, 'getDetailsFileName').returns('md5-bro-n-time'); + + sandbox.stub(fs, 'writeFile'); + sandbox.stub(fs, 'mkdirs'); + }); + + it('should do nothing if no error details are available', async () => { + await reportBuilder.addFail(stubTest_()); + + assert.notCalled(fs.writeFile); + }); + + it('should save error details to correct path', async () => { + await reportBuilder.addFail(stubTest_({errorDetails: {filePath: 'some-path'}})); + + assert.calledWithMatch(fs.writeFile, path.resolve(`${TEST_REPORT_PATH}/some-path`), sinon.match.any); + }); + + it('should create directory for error details', async () => { + await reportBuilder.addFail(stubTest_({errorDetails: {filePath: `some-dir/some-path`}})); + + assert.calledOnceWith(fs.mkdirs, path.resolve(TEST_REPORT_PATH, 'some-dir')); + }); + + it('should save error details', async () => { + const data = {foo: 'bar'}; + await reportBuilder.addFail(stubTest_({errorDetails: {filePath: 'some-path', data}})); + + assert.calledWith(fs.writeFile, sinon.match.any, JSON.stringify(data, null, 2)); + }); + }); + describe('finalization', () => { let reportBuilder; diff --git a/test/unit/lib/test-adapter/hermione.ts b/test/unit/lib/test-adapter/hermione.ts index 153044060..fad52c465 100644 --- a/test/unit/lib/test-adapter/hermione.ts +++ b/test/unit/lib/test-adapter/hermione.ts @@ -7,9 +7,10 @@ import tmpOriginal from 'tmp'; import {TestStatus} from 'lib/constants/test-statuses'; import {ERROR_DETAILS_PATH} from 'lib/constants/paths'; import {HermioneTestAdapter, HermioneTestAdapterOptions, ReporterTestResult} from 'lib/test-adapter'; -import {ErrorDetails, HermioneTestResult} from 'lib/types'; +import {HermioneTestResult} from 'lib/types'; import {ImagesInfoFormatter} from 'lib/image-handler'; import * as originalUtils from 'lib/server-utils'; +import * as originalTestAdapterUtils from 'lib/test-adapter/utils'; describe('HermioneTestAdapter', () => { const sandbox = sinon.sandbox.create(); @@ -21,6 +22,7 @@ describe('HermioneTestAdapter', () => { let fs: sinon.SinonStubbedInstance; let tmp: typeof tmpOriginal; let hermioneCache: typeof import('lib/test-adapter/cache/hermione'); + let testAdapterUtils: sinon.SinonStubbedInstance; const mkImagesInfoFormatter = (): sinon.SinonStubbedInstance => { return {} as sinon.SinonStubbedInstance; @@ -50,13 +52,19 @@ describe('HermioneTestAdapter', () => { }); utils = _.clone(originalUtils); + const originalTestAdapterUtils = proxyquire('lib/test-adapter/utils', { + '../../server-utils': utils + }); + testAdapterUtils = _.clone(originalTestAdapterUtils); + HermioneTestAdapter = proxyquire('lib/test-adapter/hermione', { tmp, 'fs-extra': fs, '../plugin-utils': {getSuitePath}, '../history-utils': {getCommandsHistory}, '../server-utils': utils, - './cache/hermione': hermioneCache + './cache/hermione': hermioneCache, + './utils': testAdapterUtils }).HermioneTestAdapter; sandbox.stub(utils, 'getCurrentPath').returns(''); sandbox.stub(utils, 'getDiffPath').returns(''); @@ -159,6 +167,7 @@ describe('HermioneTestAdapter', () => { }); it('should be memoized', () => { + const extractErrorDetails = sandbox.stub(testAdapterUtils, 'extractErrorDetails').returns({}); const testResult = mkTestResult_({ err: { details: {title: 'some-title', data: {foo: 'bar'}} @@ -169,7 +178,7 @@ describe('HermioneTestAdapter', () => { const firstErrDetails = hermioneTestAdapter.errorDetails; const secondErrDetails = hermioneTestAdapter.errorDetails; - assert.calledOnce(getDetailsFileName); + assert.calledOnce(extractErrorDetails); assert.deepEqual(firstErrDetails, secondErrDetails); }); @@ -198,52 +207,6 @@ describe('HermioneTestAdapter', () => { }); }); - describe('saveErrorDetails', () => { - beforeEach(() => { - sandbox.stub(utils, 'makeDirFor').resolves(); - sandbox.stub(utils, 'getDetailsFileName').returns('md5-bro-n-time'); - }); - - it('should do nothing if no error details are available', async () => { - const hermioneTestAdapter = mkHermioneTestResultAdapter(mkTestResult_({err: {} as any})); - - await hermioneTestAdapter.saveErrorDetails(''); - - assert.notCalled(fs.writeFile); - }); - - it('should save error details to correct path', async () => { - const testResult = mkTestResult_({err: { - details: {title: 'some-title', data: {}} - } as any}); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - const {filePath} = hermioneTestAdapter.errorDetails as ErrorDetails; - - await hermioneTestAdapter.saveErrorDetails('report-path'); - - assert.calledWithMatch(fs.writeFile, `report-path/${filePath}`, sinon.match.any); - }); - - it('should create directory for error details', async () => { - const testResult = mkTestResult_({err: {details: {data: {}}} as any}); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - await hermioneTestAdapter.saveErrorDetails('report-path'); - - assert.calledOnceWith(utils.makeDirFor, sinon.match(`report-path/${ERROR_DETAILS_PATH}`)); - }); - - it('should save error details', async () => { - const data = {foo: 'bar'}; - const testResult = mkTestResult_({err: {details: {data}} as any}); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - await hermioneTestAdapter.saveErrorDetails(''); - - assert.calledWith(fs.writeFile, sinon.match.any, JSON.stringify(data, null, 2)); - }); - }); - it('should return image dir', () => { const testResult = mkTestResult_({id: 'some-id'}); diff --git a/test/unit/lib/tests-tree-builder/base.js b/test/unit/lib/tests-tree-builder/base.js index cd0b3ab05..fb0ce8e36 100644 --- a/test/unit/lib/tests-tree-builder/base.js +++ b/test/unit/lib/tests-tree-builder/base.js @@ -8,7 +8,7 @@ const {ToolName} = require('lib/constants'); describe('ResultsTreeBuilder', () => { const sandbox = sinon.sandbox.create(); - let ResultsTreeBuilder, builder, determineStatus; + let ResultsTreeBuilder, builder, determineFinalStatus; const mkTestResult_ = (result) => { return _.defaults(result, {imagesInfo: [], metaInfo: {}}); @@ -23,9 +23,9 @@ describe('ResultsTreeBuilder', () => { }; beforeEach(() => { - determineStatus = sandbox.stub().returns(SUCCESS); + determineFinalStatus = sandbox.stub().returns(SUCCESS); ResultsTreeBuilder = proxyquire('lib/tests-tree-builder/base', { - '../common-utils': {determineStatus} + '../common-utils': {determineFinalStatus} }).BaseTestsTreeBuilder; builder = ResultsTreeBuilder.create({toolName: ToolName.Hermione}); @@ -253,7 +253,7 @@ describe('ResultsTreeBuilder', () => { mkFormattedResult_({testPath: ['s1']}) ); - assert.calledOnceWith(determineStatus, [SUCCESS]); + assert.calledOnceWith(determineFinalStatus, [SUCCESS]); }); it('should call "determineFinalStatus" with test result status from last attempt', () => { @@ -266,7 +266,7 @@ describe('ResultsTreeBuilder', () => { mkFormattedResult_({testPath: ['s1'], attempt: 1}) ); - assert.calledWith(determineStatus.lastCall, [SUCCESS]); + assert.calledWith(determineFinalStatus.lastCall, [SUCCESS]); }); it('should call "determineFinalStatus" with all test statuses from each browser', () => { @@ -279,12 +279,12 @@ describe('ResultsTreeBuilder', () => { mkFormattedResult_({testPath: ['s1'], browserId: 'b2'}) ); - assert.calledWith(determineStatus.secondCall, [FAIL, SUCCESS]); + assert.calledWith(determineFinalStatus.secondCall, [FAIL, SUCCESS]); }); it('should call "determineFinalStatus" with statuses from child suites', () => { - determineStatus.withArgs([FAIL]).returns('s1 s2 status'); - determineStatus.withArgs([ERROR]).returns('s1 s3 status'); + determineFinalStatus.withArgs([FAIL]).returns('s1 s2 status'); + determineFinalStatus.withArgs([ERROR]).returns('s1 s3 status'); builder.addTestResult( mkTestResult_({status: FAIL}), mkFormattedResult_({testPath: ['s1', 's2']}) @@ -294,7 +294,7 @@ describe('ResultsTreeBuilder', () => { mkFormattedResult_({testPath: ['s1', 's3']}) ); - assert.calledWith(determineStatus.getCall(3), ['s1 s2 status', 's1 s3 status']); + assert.calledWith(determineFinalStatus.getCall(3), ['s1 s2 status', 's1 s3 status']); }); }); }); diff --git a/test/unit/lib/tests-tree-builder/static.js b/test/unit/lib/tests-tree-builder/static.js index 7a2521c21..6270e7085 100644 --- a/test/unit/lib/tests-tree-builder/static.js +++ b/test/unit/lib/tests-tree-builder/static.js @@ -86,13 +86,13 @@ describe('StaticResultsTreeBuilder', () => { assert.calledWith( StaticTestsTreeBuilder.prototype.addTestResult.firstCall, - formatToTestResult(dataFromDb1, {attempt: 0}), - {browserId: 'yabro', testPath: ['s1'], attempt: 0} + sinon.match(formatToTestResult(dataFromDb1, {attempt: 0})), + sinon.match({browserId: 'yabro', testPath: ['s1'], attempt: 0}) ); assert.calledWith( StaticTestsTreeBuilder.prototype.addTestResult.secondCall, - formatToTestResult(dataFromDb2, {attempt: 0}), - {browserId: 'yabro', testPath: ['s2'], attempt: 0} + sinon.match(formatToTestResult(dataFromDb2, {attempt: 0})), + sinon.match({browserId: 'yabro', testPath: ['s2'], attempt: 0}) ); }); @@ -105,13 +105,13 @@ describe('StaticResultsTreeBuilder', () => { assert.calledWith( StaticTestsTreeBuilder.prototype.addTestResult.firstCall, - formatToTestResult(dataFromDb1, {attempt: 0}), - {browserId: 'yabro', testPath: ['s1'], attempt: 0} + sinon.match(formatToTestResult(dataFromDb1, {attempt: 0})), + sinon.match({browserId: 'yabro', testPath: ['s1'], attempt: 0}) ); assert.calledWith( StaticTestsTreeBuilder.prototype.addTestResult.secondCall, - formatToTestResult(dataFromDb2, {attempt: 1}), - {browserId: 'yabro', testPath: ['s1'], attempt: 1} + sinon.match(formatToTestResult(dataFromDb2, {attempt: 1})), + sinon.match({browserId: 'yabro', testPath: ['s1'], attempt: 1}) ); }); }); diff --git a/test/unit/utils.js b/test/unit/utils.js index 3ba25086d..1131994db 100644 --- a/test/unit/utils.js +++ b/test/unit/utils.js @@ -130,6 +130,24 @@ function mkFormattedTest(result) { }); } +class NoRefImageError extends Error { + name = 'NoRefImageError'; +} + +class ImageDiffError extends Error { + name = 'ImageDiffError'; + constructor() { + super(); + this.stateName = ''; + this.currImg = { + path: '' + }; + this.refImg = { + path: '' + }; + } +} + module.exports = { stubConfig, stubTestCollection, @@ -141,5 +159,7 @@ module.exports = { mkImagesInfo, mkSuiteTree, mkStorage, - mkFormattedTest + mkFormattedTest, + NoRefImageError, + ImageDiffError };