From 469387f97c69c970dae25ac5475b8546085c8e6a Mon Sep 17 00:00:00 2001 From: shadowusr <58862284+shadowusr@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:12:17 +0300 Subject: [PATCH] refactor: get rid of tight coupling with hermione and move image processing logic out of test-adapter (#491) * refactor: get rid of hermione in sqlite-adapter * refactor: get rid of hermione in test-adapter * refactor: get rid of hermione in static report builder * refactor: use correct types for workers in test-adapter * refactor: move image processing logic out of test-adapter --- hermione.js | 5 +- lib/common-utils.ts | 22 + lib/constants/plugin-events.ts | 3 +- lib/errors/index.ts | 28 + lib/gui/tool-runner/index.js | 4 +- lib/gui/tool-runner/report-subscriber.js | 13 +- lib/image-cache.ts | 3 + lib/image-handler.ts | 317 +++++++ lib/image-store.ts | 32 + lib/plugin-adapter.ts | 2 +- lib/plugin-api.ts | 1 + lib/report-builder/gui.js | 6 +- lib/report-builder/static.ts | 58 +- lib/server-utils.ts | 57 +- lib/sqlite-adapter.ts | 15 +- lib/test-adapter.ts | 365 +------- lib/types.ts | 16 +- lib/workers/create-workers.ts | 2 +- lib/workers/worker.ts | 2 +- package-lock.json | 102 ++- package.json | 6 +- test/tsconfig.json | 4 + test/types.ts | 5 +- test/unit/hermione.js | 38 +- test/unit/lib/gui/tool-runner/index.js | 5 +- .../lib/gui/tool-runner/report-subsciber.js | 9 +- test/unit/lib/image-handler.ts | 439 ++++++++++ test/unit/lib/report-builder/gui.js | 2 +- test/unit/lib/report-builder/static.js | 2 +- test/unit/lib/server-utils.js | 5 +- test/unit/lib/sqlite-adapter.js | 12 +- test/unit/lib/test-adapter.js | 780 ------------------ test/unit/lib/test-adapter.ts | 382 +++++++++ 33 files changed, 1499 insertions(+), 1243 deletions(-) create mode 100644 lib/errors/index.ts create mode 100644 lib/image-cache.ts create mode 100644 lib/image-handler.ts create mode 100644 lib/image-store.ts create mode 100644 test/tsconfig.json create mode 100644 test/unit/lib/image-handler.ts delete mode 100644 test/unit/lib/test-adapter.js create mode 100644 test/unit/lib/test-adapter.ts diff --git a/hermione.js b/hermione.js index 0d9b70d6a..612748646 100644 --- a/hermione.js +++ b/hermione.js @@ -29,10 +29,11 @@ module.exports = (hermione, opts) => { async function prepare(hermione, reportBuilder, pluginConfig) { const {path: reportPath} = pluginConfig; + const {imageHandler} = reportBuilder; const failHandler = async (testResult) => { const formattedResult = reportBuilder.format(testResult); - const actions = [formattedResult.saveTestImages(reportPath, workers)]; + const actions = [imageHandler.saveTestImages(testResult, formattedResult.attempt, workers)]; if (formattedResult.errorDetails) { actions.push(formattedResult.saveErrorDetails(reportPath)); @@ -56,7 +57,7 @@ async function prepare(hermione, reportBuilder, pluginConfig) { hermione.on(hermione.events.TEST_PASS, testResult => { promises.push(queue.add(async () => { const formattedResult = reportBuilder.format(testResult); - await formattedResult.saveTestImages(reportPath, workers); + await imageHandler.saveTestImages(testResult, formattedResult.attempt, workers); return reportBuilder.addSuccess(formattedResult); }).catch(reject)); diff --git a/lib/common-utils.ts b/lib/common-utils.ts index 37bce0fa3..62fff935e 100644 --- a/lib/common-utils.ts +++ b/lib/common-utils.ts @@ -5,6 +5,8 @@ import axios, {AxiosRequestConfig} from 'axios'; import {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED, TestStatus} from './constants'; import {UNCHECKED, INDETERMINATE, CHECKED} from './constants/checked-statuses'; +import {AssertViewResult, TestResult} from './types'; +import {ErrorName, ImageDiffError, NoRefImageError} from './errors'; export const getShortMD5 = (str: string): string => { return crypto.createHash('md5').update(str, 'ascii').digest('hex').substr(0, 7); }; @@ -44,6 +46,26 @@ export const determineStatus = (statuses: TestStatus[]): TestStatus | null => { return null; }; +export const mkTestId = (fullTitle: string, browserId: string): string => { + return fullTitle + '.' + browserId; +}; + +export const isImageDiffError = (assertResult: AssertViewResult): assertResult is ImageDiffError => { + return assertResult.name === ErrorName.IMAGE_DIFF; +}; + +export const isNoRefImageError = (assertResult: AssertViewResult): assertResult is NoRefImageError => { + return assertResult.name === ErrorName.NO_REF_IMAGE; +}; + +export const getError = (testResult: TestResult): undefined | {message?: string; stack?: string; stateName?: string} => { + if (!testResult.err) { + return undefined; + } + + return pick(testResult.err, ['message', 'stack', 'stateName']); +}; + export const isUrl = (str: string): boolean => { if (typeof str !== 'string') { return false; diff --git a/lib/constants/plugin-events.ts b/lib/constants/plugin-events.ts index 077f3c216..61f476739 100644 --- a/lib/constants/plugin-events.ts +++ b/lib/constants/plugin-events.ts @@ -1,5 +1,6 @@ export enum PluginEvents { DATABASE_CREATED = 'databaseCreated', TEST_SCREENSHOTS_SAVED = 'testScreenshotsSaved', - REPORT_SAVED = 'reportSaved' + REPORT_SAVED = 'reportSaved', + IMAGES_SAVER_UPDATED = 'imagesSaverUpdated' } diff --git a/lib/errors/index.ts b/lib/errors/index.ts new file mode 100644 index 000000000..5bb6b658e --- /dev/null +++ b/lib/errors/index.ts @@ -0,0 +1,28 @@ +import {CoordBounds} from 'looks-same'; +import {DiffOptions, ImageData} from '../types'; + +export enum ErrorName { + IMAGE_DIFF = 'ImageDiffError', + NO_REF_IMAGE = 'NoRefImageError' +} + +export interface ImageDiffError { + name: ErrorName.IMAGE_DIFF; + message: string; + stack: string; + stateName: string; + diffOpts: DiffOptions; + currImg: ImageData; + refImg: ImageData; + diffClusters: CoordBounds[]; + diffBuffer?: ArrayBuffer; +} + +export interface NoRefImageError { + name: ErrorName.NO_REF_IMAGE; + stateName: string; + message: string; + stack: string; + currImg: ImageData; + refImg: ImageData; +} diff --git a/lib/gui/tool-runner/index.js b/lib/gui/tool-runner/index.js index 8b97f8816..ba8ef45ee 100644 --- a/lib/gui/tool-runner/index.js +++ b/lib/gui/tool-runner/index.js @@ -52,7 +52,7 @@ module.exports = class ToolRunner { async initialize() { await mergeDatabasesForReuse(this._reportPath); - this._reportBuilder = GuiReportBuilder.create(this._hermione, this._pluginConfig, {reuse: true}); + this._reportBuilder = GuiReportBuilder.create(this._hermione.htmlReporter, this._pluginConfig, {reuse: true}); this._subscribeOnEvents(); this._collection = await this._readTests(); @@ -152,7 +152,7 @@ module.exports = class ToolRunner { } if (previousExpectedPath) { - formattedResult.updateCacheExpectedPath(stateName, previousExpectedPath); + this._reportBuilder.imageHandler.updateCacheExpectedPath(updateResult, stateName, previousExpectedPath); } }); }); diff --git a/lib/gui/tool-runner/report-subscriber.js b/lib/gui/tool-runner/report-subscriber.js index 696efee1b..18d0ac7f3 100644 --- a/lib/gui/tool-runner/report-subscriber.js +++ b/lib/gui/tool-runner/report-subscriber.js @@ -12,9 +12,10 @@ let workers; module.exports = (hermione, reportBuilder, client, reportPath) => { const queue = new PQueue({concurrency: os.cpus().length}); + const {imageHandler} = reportBuilder; - function failHandler(formattedResult) { - const actions = [formattedResult.saveTestImages(reportPath, workers)]; + function failHandler(testResult, formattedResult) { + const actions = [imageHandler.saveTestImages(testResult, formattedResult.attempt, workers)]; if (formattedResult.errorDetails) { actions.push(formattedResult.saveErrorDetails(reportPath)); @@ -53,7 +54,7 @@ module.exports = (hermione, reportBuilder, client, reportPath) => { const formattedResult = reportBuilder.format(testResult, hermione.events.TEST_PASS); formattedResult.attempt = reportBuilder.getCurrAttempt(formattedResult); - await formattedResult.saveTestImages(reportPath, workers); + await imageHandler.saveTestImages(testResult, formattedResult.attempt, workers); reportBuilder.addSuccess(formattedResult); const testBranch = reportBuilder.getTestBranch(formattedResult.id); @@ -66,7 +67,7 @@ module.exports = (hermione, reportBuilder, client, reportPath) => { const formattedResult = reportBuilder.format(testResult, hermione.events.RETRY); formattedResult.attempt = reportBuilder.getCurrAttempt(formattedResult); - await failHandler(formattedResult); + await failHandler(testResult, formattedResult); reportBuilder.addRetry(formattedResult); const testBranch = reportBuilder.getTestBranch(formattedResult.id); @@ -79,7 +80,7 @@ module.exports = (hermione, reportBuilder, client, reportPath) => { const formattedResult = reportBuilder.format(testResult, hermione.events.TEST_FAIL); formattedResult.attempt = reportBuilder.getCurrAttempt(formattedResult); - await failHandler(formattedResult); + await failHandler(testResult, formattedResult); formattedResult.hasDiff() ? reportBuilder.addFail(formattedResult) : reportBuilder.addError(formattedResult); @@ -94,7 +95,7 @@ module.exports = (hermione, reportBuilder, client, reportPath) => { const formattedResult = reportBuilder.format(testResult, hermione.events.TEST_PENDING); formattedResult.attempt = reportBuilder.getCurrAttempt(formattedResult); - await failHandler(formattedResult); + await failHandler(testResult, formattedResult); reportBuilder.addSkipped(formattedResult); const testBranch = reportBuilder.getTestBranch(formattedResult.id); diff --git a/lib/image-cache.ts b/lib/image-cache.ts new file mode 100644 index 000000000..58bf315da --- /dev/null +++ b/lib/image-cache.ts @@ -0,0 +1,3 @@ +export const cacheAllImages = new Map(); +export const cacheDiffImages = new Map(); +export const cacheExpectedPaths = new Map(); diff --git a/lib/image-handler.ts b/lib/image-handler.ts new file mode 100644 index 000000000..4e0d6e712 --- /dev/null +++ b/lib/image-handler.ts @@ -0,0 +1,317 @@ +import path from 'path'; +import EventEmitter2 from 'eventemitter2'; +import fs from 'fs-extra'; +import _ from 'lodash'; +import tmp from 'tmp'; + +import type {ImageStore} from './image-store'; +import {RegisterWorkers} from './workers/create-workers'; +import * as utils from './server-utils'; +import { + AssertViewResult, + ImageBase64, + ImageData, + ImageInfo, ImageInfoError, + ImageInfoFail, + ImageInfoFull, + ImagesSaver, + TestResult +} from './types'; +import {ERROR, FAIL, PluginEvents, SUCCESS, TestStatus, UPDATED} from './constants'; +import {getError, getShortMD5, isImageDiffError, isNoRefImageError, logger, mkTestId} from './common-utils'; +import {ImageDiffError} from './errors'; +import {cacheExpectedPaths, cacheAllImages, cacheDiffImages} from './image-cache'; + +export interface ImagesInfoFormatter { + getImagesInfo(testResult: TestResult, attempt: number): ImageInfoFull[]; + getCurrImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined; + getRefImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined; + getScreenshot(testResult: TestResult): ImageBase64 | undefined; +} + +export interface ImageHandlerOptions { + reportPath: string; +} + +export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter { + private _imageStore: ImageStore; + private _imagesSaver: ImagesSaver; + private _options: ImageHandlerOptions; + + constructor(imageStore: ImageStore, imagesSaver: ImagesSaver, options: ImageHandlerOptions) { + super(); + this._imageStore = imageStore; + this._imagesSaver = imagesSaver; + this._options = options; + } + + getCurrImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined { + return _.get(_.find(assertViewResults, {stateName}), 'currImg'); + } + + getImagesFor(testResult: TestResult, attempt: number, status: TestStatus, stateName?: string): ImageInfo | undefined { + const refImg = this.getRefImg(testResult.assertViewResults, stateName); + const currImg = this.getCurrImg(testResult.assertViewResults, stateName); + const errImg = this.getScreenshot(testResult); + + const {path: refPath} = this._getExpectedPath(testResult, attempt, stateName, status); + const currPath = utils.getCurrentPath({attempt, browserId: testResult.browserId, imageDir: testResult.id.toString(), stateName}); + const diffPath = utils.getDiffPath({attempt, browserId: testResult.browserId, imageDir: testResult.id.toString(), stateName}); + + if ((status === SUCCESS || status === UPDATED) && refImg) { + const result: ImageInfo = { + expectedImg: {path: this._getImgFromStorage(refPath), size: refImg.size} + }; + if (currImg) { + result.actualImg = {path: this._getImgFromStorage(currPath), size: currImg.size}; + } + + return result; + } + + if (status === FAIL && refImg && currImg) { + return { + expectedImg: { + path: this._getImgFromStorage(refPath), + size: refImg.size + }, + actualImg: { + path: this._getImgFromStorage(currPath), + size: currImg.size + }, + diffImg: { + path: this._getImgFromStorage(diffPath), + size: { + width: _.max([_.get(refImg, 'size.width'), _.get(currImg, 'size.width')]) as number, + height: _.max([_.get(refImg, 'size.height'), _.get(currImg, 'size.height')]) as number + } + } + }; + } + + if (status === ERROR && errImg) { + return { + actualImg: { + path: testResult.title ? this._getImgFromStorage(currPath) : '', + size: currImg?.size || errImg.size + } + }; + } + + return; + } + + getImagesInfo(testResult: TestResult, attempt: number): ImageInfoFull[] { + const imagesInfo: ImageInfoFull[] = testResult.assertViewResults?.map((assertResult): ImageInfoFull => { + let status: TestStatus | undefined, error: {message: string; stack: string;} | undefined; + + if (testResult.updated === true) { + status = UPDATED; + } else if (!(assertResult instanceof Error)) { + status = SUCCESS; + } else if (isImageDiffError(assertResult)) { + status = FAIL; + } else if (isNoRefImageError(assertResult)) { + status = ERROR; + error = _.pick(assertResult, ['message', 'stack']); + } + + const {stateName, refImg} = assertResult; + const diffClusters = (assertResult as ImageDiffError).diffClusters; + + return _.extend( + {stateName, refImg, status: status, error, diffClusters}, + this.getImagesFor(testResult, attempt, status as TestStatus, stateName) + ) as ImageInfoFull; + }) ?? []; + + // common screenshot on test fail + if (this.getScreenshot(testResult)) { + const errorImage = _.extend( + {status: ERROR, error: getError(testResult)}, + this.getImagesFor(testResult, attempt, ERROR) + ) as ImageInfoError; + + imagesInfo.push(errorImage); + } + + return imagesInfo; + } + + getRefImg(assertViewResults: AssertViewResult[], stateName?: string): ImageData | undefined { + return _.get(_.find(assertViewResults, {stateName}), 'refImg'); + } + + getScreenshot(testResult: TestResult): ImageBase64 | undefined { + return _.get(testResult, 'err.screenshot'); + } + + async saveTestImages(testResult: TestResult, attempt: number, worker: RegisterWorkers<['saveDiffTo']>): Promise { + const {assertViewResults = []} = testResult; + + const result = await Promise.all(assertViewResults.map(async (assertResult) => { + const {stateName} = assertResult; + const {path: destRefPath, reused: reusedReference} = this._getExpectedPath(testResult, attempt, stateName, undefined); + const srcRefPath = this.getRefImg(testResult.assertViewResults, stateName)?.path; + + const destCurrPath = utils.getCurrentPath({attempt, browserId: testResult.browserId, imageDir: testResult.id.toString(), stateName}); + const srcCurrPath = this.getCurrImg(testResult.assertViewResults, stateName)?.path; + + const dstCurrPath = utils.getDiffPath({attempt, browserId: testResult.browserId, imageDir: testResult.id.toString(), stateName}); + const srcDiffPath = path.resolve(tmp.tmpdir, dstCurrPath); + const actions: unknown[] = []; + + if (!(assertResult instanceof Error)) { + actions.push(this._saveImg(srcRefPath, destRefPath)); + } + + if (isImageDiffError(assertResult)) { + await this._saveDiffInWorker(assertResult, srcDiffPath, worker); + + actions.push( + this._saveImg(srcCurrPath, destCurrPath), + this._saveImg(srcDiffPath, dstCurrPath) + ); + + if (!reusedReference) { + actions.push(this._saveImg(srcRefPath, destRefPath)); + } + } + + if (isNoRefImageError(assertResult)) { + actions.push(this._saveImg(srcCurrPath, destCurrPath)); + } + + return Promise.all(actions); + })); + + if (this.getScreenshot(testResult)) { + await this._saveErrorScreenshot(testResult, attempt); + } + + await this.emitAsync(PluginEvents.TEST_SCREENSHOTS_SAVED, { + testId: mkTestId(testResult.fullTitle(), testResult.browserId), + attempt: attempt, + imagesInfo: this.getImagesInfo(testResult, attempt) + }); + + return result; + } + + setImagesSaver(newImagesSaver: ImagesSaver): void { + this._imagesSaver = newImagesSaver; + } + + updateCacheExpectedPath(testResult: TestResult, stateName: string, expectedPath: string): void { + const key = this._getExpectedKey(testResult, stateName); + + if (expectedPath) { + cacheExpectedPaths.set(key, expectedPath); + } else { + cacheExpectedPaths.delete(key); + } + } + + private _getExpectedKey(testResult: TestResult, stateName?: string): string { + const shortTestId = getShortMD5(mkTestId(testResult.fullTitle(), testResult.browserId)); + + return shortTestId + '#' + stateName; + } + + private _getExpectedPath(testResult: TestResult, attempt: number, stateName?: string, status?: TestStatus): {path: string, reused: boolean} { + const key = this._getExpectedKey(testResult, stateName); + + if (status === UPDATED) { + const expectedPath = utils.getReferencePath({attempt, browserId: testResult.browserId, imageDir: testResult.id.toString(), stateName}); + cacheExpectedPaths.set(key, expectedPath); + + return {path: expectedPath, reused: false}; + } + + if (cacheExpectedPaths.has(key)) { + return {path: cacheExpectedPaths.get(key) as string, reused: true}; + } + + const imageInfo = this._imageStore.getLastImageInfoFromDb(testResult, stateName); + + if (imageInfo && (imageInfo as ImageInfoFail).expectedImg) { + const expectedPath = (imageInfo as ImageInfoFail).expectedImg.path; + + cacheExpectedPaths.set(key, expectedPath); + + return {path: expectedPath, reused: true}; + } + + const expectedPath = utils.getReferencePath({attempt, browserId: testResult.browserId, imageDir: testResult.id.toString(), stateName}); + + cacheExpectedPaths.set(key, expectedPath); + + return {path: expectedPath, reused: false}; + } + + private _getImgFromStorage(imgPath: string): string { + // fallback for updating image in gui mode + return cacheAllImages.get(imgPath) || imgPath; + } + + private async _saveDiffInWorker(imageDiffError: ImageDiffError, destPath: string, worker: RegisterWorkers<['saveDiffTo']>): Promise { + await utils.makeDirFor(destPath); + + // new versions of hermione provide `diffBuffer` + if (imageDiffError.diffBuffer) { + const pngBuffer = Buffer.from(imageDiffError.diffBuffer); + + await fs.writeFile(destPath, pngBuffer); + + return; + } + + const currPath = imageDiffError.currImg.path; + const refPath = imageDiffError.refImg.path; + + const [currBuffer, refBuffer] = await Promise.all([ + fs.readFile(currPath), + fs.readFile(refPath) + ]); + + const hash = utils.createHash(currBuffer) + utils.createHash(refBuffer); + + if (cacheDiffImages.has(hash)) { + const cachedDiffPath = cacheDiffImages.get(hash) as string; + + await fs.copy(cachedDiffPath, destPath); + return; + } + + await worker.saveDiffTo(imageDiffError, destPath); + + cacheDiffImages.set(hash, destPath); + } + + private async _saveErrorScreenshot(testResult: TestResult, attempt: number): Promise { + const screenshot = this.getScreenshot(testResult); + if (!screenshot?.base64) { + logger.warn('Cannot save screenshot on reject'); + + return Promise.resolve(); + } + + const currPath = utils.getCurrentPath({attempt, browserId: testResult.browserId, imageDir: testResult.id.toString()}); + const localPath = path.resolve(tmp.tmpdir, currPath); + await utils.makeDirFor(localPath); + await fs.writeFile(localPath, new Buffer(screenshot.base64, 'base64'), 'base64'); + + await this._saveImg(localPath, currPath); + } + + private async _saveImg(localPath: string | undefined, destPath: string): Promise { + if (!localPath) { + return Promise.resolve(undefined); + } + + const res = await this._imagesSaver.saveImg(localPath, {destPath, reportDir: this._options.reportPath}); + + cacheAllImages.set(destPath, res || destPath); + return res; + } +} diff --git a/lib/image-store.ts b/lib/image-store.ts new file mode 100644 index 000000000..53040a82b --- /dev/null +++ b/lib/image-store.ts @@ -0,0 +1,32 @@ +import {DB_COLUMNS} from './constants'; +import {getSuitePath} from './plugin-utils'; +import {SqliteAdapter} from './sqlite-adapter'; +import {ImageInfo, ImageInfoFull, LabeledSuitesRow, TestResult} from './types'; + +export interface ImageStore { + getLastImageInfoFromDb(testResult: TestResult, stateName?: string): ImageInfo | undefined ; +} + +export class SqliteImageStore implements ImageStore { + private _sqliteAdapter: SqliteAdapter; + + constructor(sqliteAdapter: SqliteAdapter) { + this._sqliteAdapter = sqliteAdapter; + } + + getLastImageInfoFromDb(testResult: TestResult, stateName?: string): ImageInfo | undefined { + const browserName = testResult.browserId; + const suitePath = getSuitePath(testResult); + const suitePathString = JSON.stringify(suitePath); + + const imagesInfoResult = this._sqliteAdapter.query | undefined>({ + select: DB_COLUMNS.IMAGES_INFO, + where: `${DB_COLUMNS.SUITE_PATH} = ? AND ${DB_COLUMNS.NAME} = ?`, + orderBy: DB_COLUMNS.TIMESTAMP, + orderDescending: true + }, suitePathString, browserName); + + const imagesInfo: ImageInfoFull[] = imagesInfoResult && JSON.parse(imagesInfoResult[DB_COLUMNS.IMAGES_INFO as keyof Pick]) || []; + return imagesInfo.find(info => info.stateName === stateName); + } +} diff --git a/lib/plugin-adapter.ts b/lib/plugin-adapter.ts index dfe710872..401979a62 100644 --- a/lib/plugin-adapter.ts +++ b/lib/plugin-adapter.ts @@ -55,7 +55,7 @@ export class PluginAdapter { } protected async _createStaticReportBuilder(prepareData: PrepareFn): Promise { - const staticReportBuilder = StaticReportBuilder.create(this._hermione, this._config); + const staticReportBuilder = StaticReportBuilder.create(this._hermione.htmlReporter, this._config); await staticReportBuilder.init(); diff --git a/lib/plugin-api.ts b/lib/plugin-api.ts index 42a9d17e5..e99fc3faf 100644 --- a/lib/plugin-api.ts +++ b/lib/plugin-api.ts @@ -63,6 +63,7 @@ export class HtmlReporter extends EventsEmitter2 { } set imagesSaver(imagesSaver: ImagesSaver) { + this.emit(PluginEvents.IMAGES_SAVER_UPDATED, imagesSaver); this._values.imagesSaver = imagesSaver; } diff --git a/lib/report-builder/gui.js b/lib/report-builder/gui.js index b5f393fd7..9779411a7 100644 --- a/lib/report-builder/gui.js +++ b/lib/report-builder/gui.js @@ -48,10 +48,6 @@ module.exports = class GuiReportBuilder extends StaticReportBuilder { addUpdated(result, failResultId) { const formattedResult = this.format(result, UPDATED); - formattedResult.imagesInfo = [] - .concat(result.imagesInfo) - .map((imageInfo) => ({...imageInfo, ...formattedResult.getImagesFor(UPDATED, imageInfo.stateName)})); - return this._addTestResult(formattedResult, {status: UPDATED}, {failResultId}); } @@ -182,7 +178,7 @@ module.exports = class GuiReportBuilder extends StaticReportBuilder { } _extendTestWithImagePaths(test, formattedResult, opts = {}) { - const newImagesInfo = formattedResult.getImagesInfo(test.status); + const newImagesInfo = formattedResult.imagesInfo; if (test.status !== UPDATED) { return _.set(test, 'imagesInfo', newImagesInfo); diff --git a/lib/report-builder/static.ts b/lib/report-builder/static.ts index 388c4bbb9..5cf0402bf 100644 --- a/lib/report-builder/static.ts +++ b/lib/report-builder/static.ts @@ -1,14 +1,27 @@ -import _ from 'lodash'; import path from 'path'; +import {GeneralEventEmitter} from 'eventemitter2'; +import _ from 'lodash'; import fs from 'fs-extra'; -import type {default as Hermione} from 'hermione'; -import {IDLE, RUNNING, SKIPPED, FAIL, ERROR, SUCCESS, TestStatus, LOCAL_DATABASE_NAME} from '../constants'; +import { + IDLE, + RUNNING, + SKIPPED, + FAIL, + ERROR, + SUCCESS, + TestStatus, + LOCAL_DATABASE_NAME, + PluginEvents +} from '../constants'; import {PreparedTestResult, SqliteAdapter} from '../sqlite-adapter'; import {TestAdapter} from '../test-adapter'; import {hasNoRefImageErrors} from '../static/modules/utils'; import {hasImage, saveStaticFilesToReportDir, writeDatabaseUrlsFile} from '../server-utils'; -import {HtmlReporterApi, ReporterConfig, TestResult} from '../types'; +import {ReporterConfig, TestResult} from '../types'; +import {HtmlReporter} from '../plugin-api'; +import {ImageHandler} from '../image-handler'; +import {SqliteImageStore} from '../image-store'; const ignoredStatuses = [RUNNING, IDLE]; @@ -17,28 +30,42 @@ interface StaticReportBuilderOptions { } export class StaticReportBuilder { - protected _hermione: Hermione & HtmlReporterApi; + protected _htmlReporter: HtmlReporter; protected _pluginConfig: ReporterConfig; protected _sqliteAdapter: SqliteAdapter; + protected _imageHandler: ImageHandler; static create( - this: new (hermione: Hermione & HtmlReporterApi, pluginConfig: ReporterConfig, options?: Partial) => T, - hermione: Hermione & HtmlReporterApi, + this: new (htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, options?: Partial) => T, + htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, options?: Partial ): T { - return new this(hermione, pluginConfig, options); + return new this(htmlReporter, pluginConfig, options); } - constructor(hermione: Hermione & HtmlReporterApi, pluginConfig: ReporterConfig, {reuse = false}: Partial = {}) { - this._hermione = hermione; + constructor(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, {reuse = false}: Partial = {}) { + this._htmlReporter = htmlReporter; this._pluginConfig = pluginConfig; this._sqliteAdapter = SqliteAdapter.create({ - hermione: this._hermione, + htmlReporter: this._htmlReporter, reportPath: this._pluginConfig.path, reuse }); + + const imageStore = new SqliteImageStore(this._sqliteAdapter); + this._imageHandler = new ImageHandler(imageStore, htmlReporter.imagesSaver, {reportPath: pluginConfig.path}); + + this._htmlReporter.on(PluginEvents.IMAGES_SAVER_UPDATED, (newImagesSaver) => { + this._imageHandler.setImagesSaver(newImagesSaver); + }); + + this._htmlReporter.listenTo(this._imageHandler as unknown as GeneralEventEmitter, [PluginEvents.TEST_SCREENSHOTS_SAVED]); + } + + get imageHandler(): ImageHandler { + return this._imageHandler; } async init(): Promise { @@ -51,9 +78,8 @@ export class StaticReportBuilder { return result instanceof TestAdapter ? result : TestAdapter.create(result, { - hermione: this._hermione, - sqliteAdapter: this._sqliteAdapter, - status + status, + imagesInfoFormatter: this._imageHandler }); } @@ -61,7 +87,7 @@ export class StaticReportBuilder { const destPath = this._pluginConfig.path; await Promise.all([ - saveStaticFilesToReportDir(this._hermione, this._pluginConfig, destPath), + saveStaticFilesToReportDir(this._htmlReporter, this._pluginConfig, destPath), writeDatabaseUrlsFile(destPath, [LOCAL_DATABASE_NAME]) ]); } @@ -160,7 +186,7 @@ export class StaticReportBuilder { async finalize(): Promise { this._sqliteAdapter.close(); - const reportsSaver = this._hermione.htmlReporter.reportsSaver; + const reportsSaver = this._htmlReporter.reportsSaver; if (reportsSaver) { const reportDir = this._pluginConfig.path; diff --git a/lib/server-utils.ts b/lib/server-utils.ts index fccc4d9b3..c96ad198c 100644 --- a/lib/server-utils.ts +++ b/lib/server-utils.ts @@ -7,47 +7,60 @@ import {logger} from './common-utils'; import {UPDATED, RUNNING, IDLE, SKIPPED, IMAGES_PATH, TestStatus} from './constants'; import type {HtmlReporter} from './plugin-api'; import type {TestAdapter} from './test-adapter'; -import {CustomGuiItem, HtmlReporterApi, ReporterConfig} from './types'; +import {CustomGuiItem, ReporterConfig} from './types'; import type Hermione from 'hermione'; +import crypto from 'crypto'; const DATA_FILE_NAME = 'data.js'; -export const getReferencePath = (testResult: TestAdapter, stateName?: string): string => createPath('ref', testResult, stateName); -export const getCurrentPath = (testResult: TestAdapter, stateName?: string): string => createPath('current', testResult, stateName); -export const getDiffPath = (testResult: TestAdapter, stateName?: string): string => createPath('diff', testResult, stateName); +interface GetPathOptions { + stateName?: string; + imageDir: string; + attempt: number; + browserId: string; +} + +export const getReferencePath = (options: GetPathOptions): string => createPath({kind: 'ref', ...options}); +export const getCurrentPath = (options: GetPathOptions): string => createPath({kind: 'current', ...options}); +export const getDiffPath = (options: GetPathOptions): string => createPath({kind: 'diff', ...options}); export const getReferenceAbsolutePath = (testResult: TestAdapter, reportDir: string, stateName: string): string => { - const referenceImagePath = getReferencePath(testResult, stateName); + const referenceImagePath = getReferencePath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId, stateName}); return path.resolve(reportDir, referenceImagePath); }; export const getCurrentAbsolutePath = (testResult: TestAdapter, reportDir: string, stateName: string): string => { - const currentImagePath = getCurrentPath(testResult, stateName); + const currentImagePath = getCurrentPath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId, stateName}); return path.resolve(reportDir, currentImagePath); }; export const getDiffAbsolutePath = (testResult: TestAdapter, reportDir: string, stateName: string): string => { - const diffImagePath = getDiffPath(testResult, stateName); + const diffImagePath = getDiffPath({attempt: testResult.attempt, imageDir: testResult.imageDir, browserId: testResult.browserId, stateName}); return path.resolve(reportDir, diffImagePath); }; -/** - * @param {String} kind - одно из значений 'ref', 'current', 'diff' - * @param {TestAdapter} result - * @param {String} stateName - имя стэйта для теста - * @returns {String} - */ -export function createPath(kind: string, result: TestAdapter, stateName?: string): string { - const attempt: number = result.attempt || 0; - const imageDir = _.compact([IMAGES_PATH, result.imageDir, stateName]); - const components = imageDir.concat(`${result.browserId}~${kind}_${attempt}.png`); +interface CreatePathOptions extends GetPathOptions { + kind: string; +} + +export function createPath({attempt: attemptInput, imageDir: imageDirInput, browserId, kind, stateName}: CreatePathOptions): string { + const attempt: number = attemptInput || 0; + const imageDir = _.compact([IMAGES_PATH, imageDirInput, stateName]); + const components = imageDir.concat(`${browserId}~${kind}_${attempt}.png`); return path.join(...components); } +export function createHash(buffer: Buffer): string { + return crypto + .createHash('sha1') + .update(buffer) + .digest('base64'); +} + export interface CopyFileAsyncOptions { reportDir: string; overwrite: boolean @@ -82,7 +95,7 @@ export function logError(e: Error): void { } export function hasImage(formattedResult: TestAdapter): boolean { - return !!formattedResult.getImagesInfo().length || + return !!formattedResult.imagesInfo?.length || !!formattedResult.getCurrImg()?.path || !!formattedResult.screenshot; } @@ -106,13 +119,13 @@ export function getDetailsFileName(testId: string, browserId: string, attempt: n return `${testId}-${browserId}_${Number(attempt) + 1}_${Date.now()}.json`; } -export async function saveStaticFilesToReportDir(hermione: Hermione & HtmlReporterApi, pluginConfig: ReporterConfig, destPath: string): Promise { +export async function saveStaticFilesToReportDir(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig, destPath: string): Promise { const staticFolder = path.resolve(__dirname, './static'); await fs.ensureDir(destPath); await Promise.all([ fs.writeFile( path.resolve(destPath, DATA_FILE_NAME), - prepareCommonJSData(getDataForStaticFile(hermione, pluginConfig)), + prepareCommonJSData(getDataForStaticFile(htmlReporter, pluginConfig)), 'utf8' ), copyToReportDir(destPath, ['report.min.js', 'report.min.css'], staticFolder), @@ -206,9 +219,7 @@ export interface DataForStaticFile { date: string; } -export function getDataForStaticFile(hermione: Hermione & HtmlReporterApi, pluginConfig: ReporterConfig): DataForStaticFile { - const htmlReporter = hermione.htmlReporter; - +export function getDataForStaticFile(htmlReporter: HtmlReporter, pluginConfig: ReporterConfig): DataForStaticFile { return { skips: [], config: getConfigForStaticFile(pluginConfig), diff --git a/lib/sqlite-adapter.ts b/lib/sqlite-adapter.ts index 0d0502a2c..bb5c41857 100644 --- a/lib/sqlite-adapter.ts +++ b/lib/sqlite-adapter.ts @@ -1,9 +1,7 @@ import path from 'path'; import Database from 'better-sqlite3'; import makeDebug from 'debug'; -import type EventEmitter2 from 'eventemitter2'; import fs from 'fs-extra'; -import type Hermione from 'hermione'; import NestedError from 'nested-error-stacks'; import {getShortMD5} from './common-utils'; @@ -11,7 +9,8 @@ import {TestStatus} from './constants'; import {DB_SUITES_TABLE_NAME, SUITES_TABLE_COLUMNS, LOCAL_DATABASE_NAME, DATABASE_URLS_JSON_NAME} from './constants/database'; import {createTablesQuery} from './db-utils/common'; import {DbNotInitializedError} from './errors/db-not-initialized-error'; -import type {ErrorDetails, HtmlReporterApi, ImageInfoFull} from './types'; +import type {ErrorDetails, ImageInfoFull} from './types'; +import {HtmlReporter} from './plugin-api'; const debug = makeDebug('html-reporter:sqlite-adapter'); @@ -59,13 +58,13 @@ interface ParsedTestResult extends PreparedTestResult { } interface SqliteAdapterOptions { - hermione: Hermione & HtmlReporterApi; + htmlReporter: HtmlReporter; reportPath: string; reuse?: boolean; } export class SqliteAdapter { - private _hermione: Hermione & HtmlReporterApi; + private _htmlReporter: HtmlReporter; private _reportPath: string; private _reuse: boolean; private _db: null | Database.Database; @@ -75,8 +74,8 @@ export class SqliteAdapter { return new this(options); } - constructor({hermione, reportPath, reuse = false}: SqliteAdapterOptions) { - this._hermione = hermione; + constructor({htmlReporter, reportPath, reuse = false}: SqliteAdapterOptions) { + this._htmlReporter = htmlReporter; this._reportPath = reportPath; this._reuse = reuse; this._db = null; @@ -102,7 +101,7 @@ export class SqliteAdapter { createTablesQuery().forEach((query) => this._db?.prepare(query).run()); - (this._hermione.htmlReporter as unknown as EventEmitter2).emit(this._hermione.htmlReporter.events.DATABASE_CREATED, this._db); + this._htmlReporter.emit(this._htmlReporter.events.DATABASE_CREATED, this._db); } catch (err: any) { // eslint-disable-line @typescript-eslint/no-explicit-any throw new NestedError(`Error creating database at "${dbPath}"`, err); } diff --git a/lib/test-adapter.ts b/lib/test-adapter.ts index ccda49982..184a3e6a2 100644 --- a/lib/test-adapter.ts +++ b/lib/test-adapter.ts @@ -1,33 +1,24 @@ import _ from 'lodash'; import fs from 'fs-extra'; import path from 'path'; -import tmp from 'tmp'; -import crypto from 'crypto'; -import type {default as Hermione} from 'hermione'; import {SuiteAdapter} from './suite-adapter'; -import {DB_COLUMNS} from './constants/database'; import {getSuitePath} from './plugin-utils'; import {getCommandsHistory} from './history-utils'; -import {ERROR, ERROR_DETAILS_PATH, FAIL, SUCCESS, TestStatus, UPDATED} from './constants'; -import {getShortMD5, logger} from './common-utils'; +import {ERROR_DETAILS_PATH, TestStatus} from './constants'; +import {getError, isImageDiffError, mkTestId} from './common-utils'; import * as utils from './server-utils'; import { + AssertViewResult, ErrorDetails, - ImageInfoFail, - HtmlReporterApi, - ImageInfo, - ImagesSaver, - LabeledSuitesRow, - TestResult, + ImageBase64, ImageData, - ImageInfoFull, ImageDiffError, AssertViewResult, ImageInfoError, - ImageBase64 + ImageInfoFull, + TestResult } from './types'; -import type {SqliteAdapter} from './sqlite-adapter'; -import EventEmitter2 from 'eventemitter2'; -import type {HtmlReporter} from './plugin-api'; -import type * as Workers from './workers/worker'; +import {ImagesInfoFormatter} from './image-handler'; + +const testsAttempts: Map = new Map(); interface PrepareTestResultData { name: string; @@ -35,48 +26,29 @@ interface PrepareTestResultData { browserId: string; } -const globalCacheAllImages: Map = new Map(); -const globalCacheExpectedPaths: Map = new Map(); -const globalCacheDiffImages: Map = new Map(); -const testsAttempts: Map = new Map(); - -function createHash(buffer: Buffer): string { - return crypto - .createHash('sha1') - .update(buffer) - .digest('base64'); -} - export interface TestAdapterOptions { - hermione: Hermione & HtmlReporterApi; - sqliteAdapter: SqliteAdapter; status: TestStatus; + imagesInfoFormatter: ImagesInfoFormatter; } export class TestAdapter { + private _imagesInfoFormatter: ImagesInfoFormatter; private _testResult: TestResult; - private _hermione: Hermione & HtmlReporterApi; - private _sqliteAdapter: SqliteAdapter; - private _errors: Hermione['errors']; private _suite: SuiteAdapter; - private _imagesSaver: ImagesSaver; private _testId: string; private _errorDetails: ErrorDetails | null; private _timestamp: number; private _attempt: number; - static create(this: new (testResult: TestResult, options: TestAdapterOptions) => T, testResult: TestResult, {hermione, sqliteAdapter, status}: TestAdapterOptions): T { - return new this(testResult, {hermione, sqliteAdapter, status}); + static create(this: new (testResult: TestResult, options: TestAdapterOptions) => T, testResult: TestResult, options: TestAdapterOptions): T { + return new this(testResult, options); } - constructor(testResult: TestResult, {hermione, sqliteAdapter, status}: TestAdapterOptions) { + constructor(testResult: TestResult, {status, imagesInfoFormatter}: TestAdapterOptions) { + this._imagesInfoFormatter = imagesInfoFormatter; this._testResult = testResult; - this._hermione = hermione; - this._sqliteAdapter = sqliteAdapter; - this._errors = this._hermione.errors; this._suite = SuiteAdapter.create(this._testResult); - this._imagesSaver = this._hermione.htmlReporter.imagesSaver; - this._testId = this._mkTestId(); + this._testId = mkTestId(testResult.fullTitle(), testResult.browserId); this._errorDetails = null; this._timestamp = this._testResult.timestamp; @@ -106,134 +78,7 @@ export class TestAdapter { } get imagesInfo(): ImageInfoFull[] | undefined { - return this._testResult.imagesInfo; - } - - set imagesInfo(imagesInfo: ImageInfoFull[]) { - this._testResult.imagesInfo = imagesInfo; - } - - protected _getImgFromStorage(imgPath: string): string { - // fallback for updating image in gui mode - return globalCacheAllImages.get(imgPath) || imgPath; - } - - protected _getLastImageInfoFromDb(stateName?: string): ImageInfo | undefined { - const browserName = this._testResult.browserId; - const suitePath = getSuitePath(this._testResult); - const suitePathString = JSON.stringify(suitePath); - - const imagesInfoResult = this._sqliteAdapter.query | undefined>({ - select: DB_COLUMNS.IMAGES_INFO, - where: `${DB_COLUMNS.SUITE_PATH} = ? AND ${DB_COLUMNS.NAME} = ?`, - orderBy: DB_COLUMNS.TIMESTAMP, - orderDescending: true - }, suitePathString, browserName); - - const imagesInfo: ImageInfoFull[] = imagesInfoResult && JSON.parse(imagesInfoResult[DB_COLUMNS.IMAGES_INFO as keyof Pick]) || []; - return imagesInfo.find(info => info.stateName === stateName); - } - - protected _getExpectedPath(stateName: string | undefined, status: TestStatus | undefined, cacheExpectedPaths: Map): {path: string, reused: boolean} { - const key = this._getExpectedKey(stateName); - - if (status === UPDATED) { - const expectedPath = utils.getReferencePath(this, stateName); - cacheExpectedPaths.set(key, expectedPath); - - return {path: expectedPath, reused: false}; - } - - if (cacheExpectedPaths.has(key)) { - return {path: cacheExpectedPaths.get(key) as string, reused: true}; - } - - const imageInfo = this._getLastImageInfoFromDb(stateName); - - if (imageInfo && (imageInfo as ImageInfoFail).expectedImg) { - const expectedPath = (imageInfo as ImageInfoFail).expectedImg.path; - - cacheExpectedPaths.set(key, expectedPath); - - return {path: expectedPath, reused: true}; - } - - const expectedPath = utils.getReferencePath(this, stateName); - - cacheExpectedPaths.set(key, expectedPath); - - return {path: expectedPath, reused: false}; - } - - getImagesFor(status: TestStatus, stateName?: string): ImageInfo | undefined { - const refImg = this.getRefImg(stateName); - const currImg = this.getCurrImg(stateName); - const errImg = this.getErrImg(); - - const {path: refPath} = this._getExpectedPath(stateName, status, globalCacheExpectedPaths); - const currPath = utils.getCurrentPath(this, stateName); - const diffPath = utils.getDiffPath(this, stateName); - - if ((status === SUCCESS || status === UPDATED) && refImg) { - return {expectedImg: {path: this._getImgFromStorage(refPath), size: refImg.size}}; - } - - if (status === FAIL && refImg && currImg) { - return { - expectedImg: { - path: this._getImgFromStorage(refPath), - size: refImg.size - }, - actualImg: { - path: this._getImgFromStorage(currPath), - size: currImg.size - }, - diffImg: { - path: this._getImgFromStorage(diffPath), - size: { - width: _.max([_.get(refImg, 'size.width'), _.get(currImg, 'size.width')]) as number, - height: _.max([_.get(refImg, 'size.height'), _.get(currImg, 'size.height')]) as number - } - } - }; - } - - if (status === ERROR && currImg && errImg) { - return { - actualImg: { - path: this.state ? this._getImgFromStorage(currPath) : '', - size: currImg.size || errImg.size - } - }; - } - - return; - } - - protected async _saveErrorScreenshot(reportPath: string): Promise { - if (!this.screenshot?.base64) { - logger.warn('Cannot save screenshot on reject'); - - return Promise.resolve(); - } - - const currPath = utils.getCurrentPath(this); - const localPath = path.resolve(tmp.tmpdir, currPath); - await utils.makeDirFor(localPath); - await fs.writeFile(localPath, new Buffer(this.screenshot.base64, 'base64'), 'base64'); - - await this._saveImg(localPath, currPath, reportPath); - } - - protected async _saveImg(localPath: string | undefined, destPath: string, reportDir: string): Promise { - if (!localPath) { - return Promise.resolve(undefined); - } - - const res = await this._imagesSaver.saveImg(localPath, {destPath, reportDir}); - - globalCacheAllImages.set(destPath, res || destPath); - return res; + return this._imagesInfoFormatter.getImagesInfo(this._testResult, this.attempt); } get origAttempt(): number | undefined { @@ -250,70 +95,19 @@ export class TestAdapter { } hasDiff(): boolean { - return this.assertViewResults.some((result) => this.isImageDiffError(result)); + return this.assertViewResults.some((result) => isImageDiffError(result)); } get assertViewResults(): AssertViewResult[] { return this._testResult.assertViewResults || []; } - isImageDiffError(assertResult: AssertViewResult): boolean { - return assertResult instanceof this._errors.ImageDiffError; - } - - isNoRefImageError(assertResult: AssertViewResult): boolean { - return assertResult instanceof this._errors.NoRefImageError; - } - - getImagesInfo(): ImageInfoFull[] { - if (!_.isEmpty(this.imagesInfo)) { - return this.imagesInfo as ImageInfoFull[]; - } - - this.imagesInfo = this.assertViewResults.map((assertResult): ImageInfoFull => { - let status, error; - - if (!(assertResult instanceof Error)) { - status = SUCCESS; - } - - if (this.isImageDiffError(assertResult)) { - status = FAIL; - } - - if (this.isNoRefImageError(assertResult)) { - status = ERROR; - error = _.pick(assertResult, ['message', 'stack']); - } - - const {stateName, refImg, diffClusters} = assertResult; - - return _.extend( - {stateName, refImg, status: status, error, diffClusters}, - this.getImagesFor(status as TestStatus, stateName) - ) as ImageInfoFull; - }); - - // common screenshot on test fail - if (this.screenshot) { - const errorImage = _.extend( - {status: ERROR, error: this.error}, - this.getImagesFor(ERROR) - ) as ImageInfoError; - - (this.imagesInfo as ImageInfoFull[]).push(errorImage); - } - - return this.imagesInfo; - } - get history(): string[] { return getCommandsHistory(this._testResult.history) as string[]; } get error(): undefined | {message?: string; stack?: string; stateName?: string} { - // TODO: return undefined or null if there's no err - return _.pick(this._testResult.err, ['message', 'stack', 'stateName']); + return getError(this._testResult); } get imageDir(): string { @@ -367,16 +161,16 @@ export class TestAdapter { return this._errorDetails; } - getRefImg(stateName?: string): ImageData | Record { - return _.get(_.find(this.assertViewResults, {stateName}), 'refImg', {}); + getRefImg(stateName?: string): ImageData | undefined { + return this._imagesInfoFormatter.getRefImg(this._testResult.assertViewResults, stateName); } - getCurrImg(stateName?: string): ImageData | Record { - return _.get(_.find(this.assertViewResults, {stateName}), 'currImg', {}); + getCurrImg(stateName?: string): ImageData | undefined { + return this._imagesInfoFormatter.getCurrImg(this._testResult.assertViewResults, stateName); } - getErrImg(): ImageBase64 | Record { - return this.screenshot || {}; + getErrImg(): ImageBase64 | undefined { + return this._imagesInfoFormatter.getScreenshot(this._testResult); } prepareTestResult(): PrepareTestResultData { @@ -414,71 +208,8 @@ export class TestAdapter { await fs.writeFile(detailsFilePath, detailsData); } - async saveTestImages(reportPath: string, workers: typeof Workers, cacheExpectedPaths = globalCacheExpectedPaths): Promise { - const result = await Promise.all(this.assertViewResults.map(async (assertResult) => { - const {stateName} = assertResult; - const {path: destRefPath, reused: reusedReference} = this._getExpectedPath(stateName, undefined, cacheExpectedPaths); - const srcRefPath = this.getRefImg(stateName)?.path; - - const destCurrPath = utils.getCurrentPath(this, stateName); - // TODO: getErrImg returns base64, but here we mistakenly try to get its path - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const srcCurrPath = this.getCurrImg(stateName)?.path || (this.getErrImg() as any)?.path; - - const dstCurrPath = utils.getDiffPath(this, stateName); - const srcDiffPath = path.resolve(tmp.tmpdir, dstCurrPath); - const actions: unknown[] = []; - - if (!(assertResult instanceof Error)) { - actions.push(this._saveImg(srcRefPath, destRefPath, reportPath)); - } - - if (this.isImageDiffError(assertResult)) { - await this._saveDiffInWorker(assertResult, srcDiffPath, workers); - - actions.push( - this._saveImg(srcCurrPath, destCurrPath, reportPath), - this._saveImg(srcDiffPath, dstCurrPath, reportPath) - ); - - if (!reusedReference) { - actions.push(this._saveImg(srcRefPath, destRefPath, reportPath)); - } - } - - if (this.isNoRefImageError(assertResult)) { - actions.push(this._saveImg(srcCurrPath, destCurrPath, reportPath)); - } - - return Promise.all(actions); - })); - - if (this.screenshot) { - await this._saveErrorScreenshot(reportPath); - } - - const htmlReporter = this._hermione.htmlReporter as HtmlReporter & EventEmitter2; - await htmlReporter.emitAsync(htmlReporter.events.TEST_SCREENSHOTS_SAVED, { - testId: this._testId, - attempt: this.attempt, - imagesInfo: this.getImagesInfo() - }); - - return result; - } - - updateCacheExpectedPath(stateName: string, expectedPath: string): void { - const key = this._getExpectedKey(stateName); - - if (expectedPath) { - globalCacheExpectedPaths.set(key, expectedPath); - } else { - globalCacheExpectedPaths.delete(key); - } - } - decreaseAttemptNumber(): void { - const testId = this._mkTestId(); + const testId = mkTestId(this._testResult.fullTitle(), this.browserId); const currentTestAttempt = testsAttempts.get(testId) as number; const previousTestAttempt = currentTestAttempt - 1; @@ -488,48 +219,4 @@ export class TestAdapter { testsAttempts.delete(testId); } } - - protected _mkTestId(): string { - return this._testResult.fullTitle() + '.' + this._testResult.browserId; - } - - protected _getExpectedKey(stateName?: string): string { - const shortTestId = getShortMD5(this._mkTestId()); - - return shortTestId + '#' + stateName; - } - - //parallelize and cache of 'looks-same.createDiff' (because it is very slow) - protected async _saveDiffInWorker(imageDiffError: ImageDiffError, destPath: string, workers: typeof Workers, cacheDiffImages = globalCacheDiffImages): Promise { - await utils.makeDirFor(destPath); - - if (imageDiffError.diffBuffer) { // new versions of hermione provide `diffBuffer` - const pngBuffer = Buffer.from(imageDiffError.diffBuffer); - - await fs.writeFile(destPath, pngBuffer); - - return; - } - - const currPath = imageDiffError.currImg.path; - const refPath = imageDiffError.refImg.path; - - const [currBuffer, refBuffer] = await Promise.all([ - fs.readFile(currPath), - fs.readFile(refPath) - ]); - - const hash = createHash(currBuffer) + createHash(refBuffer); - - if (cacheDiffImages.has(hash)) { - const cachedDiffPath = cacheDiffImages.get(hash) as string; - - await fs.copy(cachedDiffPath, destPath); - return; - } - - await workers.saveDiffTo(imageDiffError, destPath); - - cacheDiffImages.set(hash, destPath); - } } diff --git a/lib/types.ts b/lib/types.ts index 53f3ce38a..66b107ea4 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -2,6 +2,7 @@ import type {LooksSameOptions, CoordBounds} from 'looks-same'; import type {default as Hermione} from 'hermione'; import {DiffModeId, SaveFormat, TestStatus, ViewMode} from './constants'; import type {HtmlReporter} from './plugin-api'; +import {ImageDiffError, NoRefImageError} from './errors'; declare module 'tmp' { export const tmpdir: string; @@ -72,6 +73,7 @@ export interface ImageInfoSuccess { refImg?: ImageData; diffClusters?: CoordBounds[]; expectedImg: ImageData; + actualImg?: ImageData; } export interface ImageInfoError { @@ -90,16 +92,7 @@ export type ImageInfo = | Omit | Omit; -export interface ImageDiffError { - stateName: string; - diffOpts: DiffOptions; - currImg: ImageData; - refImg: ImageData; - diffClusters: CoordBounds[]; - diffBuffer?: ArrayBuffer; -} - -export type AssertViewResult = ImageDiffError; +export type AssertViewResult = ImageDiffError | NoRefImageError; export interface TestResult extends ConfigurableTestObject { assertViewResults: AssertViewResult[]; @@ -109,16 +102,17 @@ export interface TestResult extends ConfigurableTestObject { stack: string; stateName?: string; details: ErrorDetails + screenshot: ImageBase64 }; fullTitle(): string; title: string; meta: Record sessionId: string; timestamp: number; - imagesInfo: ImageInfoFull[]; origAttempt?: number; history: unknown; parent: Suite; + updated?: boolean; } export interface LabeledSuitesRow { diff --git a/lib/workers/create-workers.ts b/lib/workers/create-workers.ts index 5407ee888..3bf7ed593 100644 --- a/lib/workers/create-workers.ts +++ b/lib/workers/create-workers.ts @@ -4,7 +4,7 @@ type MapOfMethods> = { [K in T[number]]: (...args: Array) => Promise | unknown; }; -type RegisterWorkers> = EventEmitter & MapOfMethods; +export type RegisterWorkers> = EventEmitter & MapOfMethods; export const createWorkers = ( runner: {registerWorkers: (workerFilePath: string, exportedMethods: string[]) => RegisterWorkers<['saveDiffTo']>} diff --git a/lib/workers/worker.ts b/lib/workers/worker.ts index 9c8276440..5476c4bb4 100644 --- a/lib/workers/worker.ts +++ b/lib/workers/worker.ts @@ -1,5 +1,5 @@ import looksSame from 'looks-same'; -import type {ImageDiffError} from '../types'; +import type {ImageDiffError} from '../errors'; export function saveDiffTo(imageDiffError: ImageDiffError, diffPath: string): Promise { const {diffColor: highlightColor, ...otherOpts} = imageDiffError.diffOpts; diff --git a/package-lock.json b/package-lock.json index fc9b6079a..8645b68de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "html-reporter", - "version": "9.10.3-hello", + "version": "9.11.0", "license": "MIT", "dependencies": { "@babel/runtime": "^7.22.5", @@ -35,7 +35,8 @@ "qs": "^6.9.1", "signal-exit": "^3.0.2", "tmp": "^0.1.0", - "urijs": "^1.18.12" + "urijs": "^1.18.12", + "worker-farm": "^1.7.0" }, "devDependencies": { "@babel/core": "^7.22.5", @@ -44,6 +45,7 @@ "@babel/preset-react": "^7.22.5", "@babel/preset-typescript": "^7.22.5", "@gemini-testing/commander": "^2.15.3", + "@playwright/test": "^1.37.1", "@swc/core": "^1.3.64", "@types/better-sqlite3": "^7.6.4", "@types/chai": "^4.3.5", @@ -53,6 +55,8 @@ "@types/lodash": "^4.14.195", "@types/nested-error-stacks": "^2.1.0", "@types/opener": "^1.4.0", + "@types/proxyquire": "^1.3.28", + "@types/sinon": "^4.3.3", "@types/tmp": "^0.1.0", "@types/urijs": "^1.19.19", "@typescript-eslint/eslint-plugin": "^5.60.0", @@ -3568,6 +3572,25 @@ "node": ">=14" } }, + "node_modules/@playwright/test": { + "version": "1.37.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.37.1.tgz", + "integrity": "sha512-bq9zTli3vWJo8S3LwB91U0qDNQDpEXnw7knhxLM0nwDvexQAwx9tO8iKDZSqqneVq+URd/WIoz+BALMqUTgdSg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "playwright-core": "1.37.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, "node_modules/@puppeteer/browsers": { "version": "1.4.6", "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.4.6.tgz", @@ -4532,6 +4555,12 @@ "integrity": "sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ==", "dev": true }, + "node_modules/@types/proxyquire": { + "version": "1.3.28", + "resolved": "https://registry.npmjs.org/@types/proxyquire/-/proxyquire-1.3.28.tgz", + "integrity": "sha512-SQaNzWQ2YZSr7FqAyPPiA3FYpux2Lqh3HWMZQk47x3xbMCqgC/w0dY3dw9rGqlweDDkrySQBcaScXWeR+Yb11Q==", + "dev": true + }, "node_modules/@types/q": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.5.tgz", @@ -4582,6 +4611,12 @@ "integrity": "sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==", "dev": true }, + "node_modules/@types/sinon": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-4.3.3.tgz", + "integrity": "sha512-Tt7w/ylBS/OEAlSCwzB0Db1KbxnkycP/1UkQpbvKFYoUuRn4uYsC3xh5TRPrOjTy0i8TIkSz1JdNL4GPVdf3KQ==", + "dev": true + }, "node_modules/@types/source-list-map": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz", @@ -4625,9 +4660,9 @@ } }, "node_modules/@types/urijs": { - "version": "1.19.19", - "resolved": "https://registry.npmjs.org/@types/urijs/-/urijs-1.19.19.tgz", - "integrity": "sha512-FDJNkyhmKLw7uEvTxx5tSXfPeQpO0iy73Ry+PmYZJvQy0QIWX8a7kJ4kLWRf+EbTPJEPDSgPXHaM7pzr5lmvCg==", + "version": "1.19.20", + "resolved": "https://registry.npmjs.org/@types/urijs/-/urijs-1.19.20.tgz", + "integrity": "sha512-77Mq/2BeHU894J364dUv9tSwxxyCLtcX228Pc8TwZpP5bvOoMns+gZoftp3LYl3FBH8vChpWbuagKGiMki2c1A==", "dev": true }, "node_modules/@types/webpack": { @@ -13555,7 +13590,6 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", - "dev": true, "dependencies": { "prr": "~1.0.1" }, @@ -22851,6 +22885,18 @@ "node": ">=8" } }, + "node_modules/playwright-core": { + "version": "1.37.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.37.1.tgz", + "integrity": "sha512-17EuQxlSIYCmEMwzMqusJ2ztDgJePjrbttaefgdsiqeLWidjYz9BxXaTaZWxH1J95SHGk6tjE+dwgWILJoUZfA==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/plugins-loader": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.2.0.tgz", @@ -24234,8 +24280,7 @@ "node_modules/prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", - "dev": true + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" }, "node_modules/pseudomap": { "version": "1.0.2", @@ -31535,7 +31580,6 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", - "dev": true, "dependencies": { "errno": "~0.1.7" } @@ -34552,6 +34596,17 @@ "dev": true, "optional": true }, + "@playwright/test": { + "version": "1.37.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.37.1.tgz", + "integrity": "sha512-bq9zTli3vWJo8S3LwB91U0qDNQDpEXnw7knhxLM0nwDvexQAwx9tO8iKDZSqqneVq+URd/WIoz+BALMqUTgdSg==", + "dev": true, + "requires": { + "@types/node": "*", + "fsevents": "2.3.2", + "playwright-core": "1.37.1" + } + }, "@puppeteer/browsers": { "version": "1.4.6", "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.4.6.tgz", @@ -35287,6 +35342,12 @@ "integrity": "sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ==", "dev": true }, + "@types/proxyquire": { + "version": "1.3.28", + "resolved": "https://registry.npmjs.org/@types/proxyquire/-/proxyquire-1.3.28.tgz", + "integrity": "sha512-SQaNzWQ2YZSr7FqAyPPiA3FYpux2Lqh3HWMZQk47x3xbMCqgC/w0dY3dw9rGqlweDDkrySQBcaScXWeR+Yb11Q==", + "dev": true + }, "@types/q": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.5.tgz", @@ -35337,6 +35398,12 @@ "integrity": "sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw==", "dev": true }, + "@types/sinon": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-4.3.3.tgz", + "integrity": "sha512-Tt7w/ylBS/OEAlSCwzB0Db1KbxnkycP/1UkQpbvKFYoUuRn4uYsC3xh5TRPrOjTy0i8TIkSz1JdNL4GPVdf3KQ==", + "dev": true + }, "@types/source-list-map": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz", @@ -35379,9 +35446,9 @@ } }, "@types/urijs": { - "version": "1.19.19", - "resolved": "https://registry.npmjs.org/@types/urijs/-/urijs-1.19.19.tgz", - "integrity": "sha512-FDJNkyhmKLw7uEvTxx5tSXfPeQpO0iy73Ry+PmYZJvQy0QIWX8a7kJ4kLWRf+EbTPJEPDSgPXHaM7pzr5lmvCg==", + "version": "1.19.20", + "resolved": "https://registry.npmjs.org/@types/urijs/-/urijs-1.19.20.tgz", + "integrity": "sha512-77Mq/2BeHU894J364dUv9tSwxxyCLtcX228Pc8TwZpP5bvOoMns+gZoftp3LYl3FBH8vChpWbuagKGiMki2c1A==", "dev": true }, "@types/webpack": { @@ -42404,7 +42471,6 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", - "dev": true, "requires": { "prr": "~1.0.1" } @@ -49746,6 +49812,12 @@ } } }, + "playwright-core": { + "version": "1.37.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.37.1.tgz", + "integrity": "sha512-17EuQxlSIYCmEMwzMqusJ2ztDgJePjrbttaefgdsiqeLWidjYz9BxXaTaZWxH1J95SHGk6tjE+dwgWILJoUZfA==", + "dev": true + }, "plugins-loader": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/plugins-loader/-/plugins-loader-1.2.0.tgz", @@ -50902,8 +50974,7 @@ "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", - "dev": true + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" }, "pseudomap": { "version": "1.0.2", @@ -56732,7 +56803,6 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", - "dev": true, "requires": { "errno": "~0.1.7" } diff --git a/package.json b/package.json index 8c359c879..13ad03e87 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,8 @@ "qs": "^6.9.1", "signal-exit": "^3.0.2", "tmp": "^0.1.0", - "urijs": "^1.18.12" + "urijs": "^1.18.12", + "worker-farm": "^1.7.0" }, "devDependencies": { "@babel/core": "^7.22.5", @@ -92,6 +93,7 @@ "@babel/preset-react": "^7.22.5", "@babel/preset-typescript": "^7.22.5", "@gemini-testing/commander": "^2.15.3", + "@playwright/test": "^1.37.1", "@swc/core": "^1.3.64", "@types/better-sqlite3": "^7.6.4", "@types/chai": "^4.3.5", @@ -101,6 +103,8 @@ "@types/lodash": "^4.14.195", "@types/nested-error-stacks": "^2.1.0", "@types/opener": "^1.4.0", + "@types/proxyquire": "^1.3.28", + "@types/sinon": "^4.3.3", "@types/tmp": "^0.1.0", "@types/urijs": "^1.19.19", "@typescript-eslint/eslint-plugin": "^5.60.0", diff --git a/test/tsconfig.json b/test/tsconfig.json new file mode 100644 index 000000000..f3407dccf --- /dev/null +++ b/test/tsconfig.json @@ -0,0 +1,4 @@ +// This tsconfig file helps IDEs to correctly resolve TS projects +{ + "extends": ["../tsconfig.spec.json"] +} diff --git a/test/types.ts b/test/types.ts index 561c05bfb..001511930 100644 --- a/test/types.ts +++ b/test/types.ts @@ -1,7 +1,10 @@ +import type sinon from 'sinon'; import {assert as chaiAssert} from 'chai'; import {mount as enzymeMount} from 'enzyme'; declare global { - const assert: typeof chaiAssert; + const assert: typeof chaiAssert & sinon.SinonAssert & { + calledOnceWith(spyOrSpyCall: sinon.SinonSpy | sinon.SinonSpyCall, ...args: TArgs): void; + }; const mount: typeof enzymeMount; } diff --git a/test/unit/hermione.js b/test/unit/hermione.js index 409013fc8..9c9883fa5 100644 --- a/test/unit/hermione.js +++ b/test/unit/hermione.js @@ -18,6 +18,7 @@ const mkSqliteDb = () => { describe('lib/hermione', () => { const sandbox = sinon.createSandbox(); let hermione; + let cacheExpectedPaths = new Map(), cacheAllImages = new Map(), cacheDiffImages = new Map(); const fs = _.clone(fsOriginal); const originalUtils = proxyquire('lib/server-utils', { @@ -30,6 +31,12 @@ describe('lib/hermione', () => { 'better-sqlite3': sinon.stub().returns(mkSqliteDb()) }); + const {ImageHandler} = proxyquire('lib/image-handler', { + 'fs-extra': fs, + './image-cache': {cacheExpectedPaths, cacheAllImages, cacheDiffImages}, + './server-utils': utils + }); + const {TestAdapter} = proxyquire('lib/test-adapter', { 'fs-extra': fs, './server-utils': utils @@ -39,7 +46,8 @@ describe('lib/hermione', () => { 'fs-extra': fs, '../server-utils': utils, '../sqlite-adapter': {SqliteAdapter}, - '../test-adapter': {TestAdapter} + '../test-adapter': {TestAdapter}, + '../image-handler': {ImageHandler} }); const {PluginAdapter} = proxyquire('lib/plugin-adapter', { @@ -68,6 +76,7 @@ describe('lib/hermione', () => { }; class ImageDiffError extends Error { + name = 'ImageDiffError'; constructor() { super(); this.stateName = ''; @@ -80,7 +89,9 @@ describe('lib/hermione', () => { } } - class NoRefImageError extends Error {} + class NoRefImageError extends Error { + name = 'NoRefImageError'; + } function mkHermione_() { return stubTool({ @@ -157,15 +168,16 @@ describe('lib/hermione', () => { sandbox.stub(SqliteAdapter.prototype, 'init').resolves({}); sandbox.stub(SqliteAdapter.prototype, 'query'); - const saveTestImagesImplementation = TestAdapter.prototype.saveTestImages; - sandbox.stub(TestAdapter.prototype, 'saveTestImages').callsFake(function(...args) { - return saveTestImagesImplementation.call(this, ...args, new Map()); // disable expectedPath cache - }); - sandbox.stub(fs, 'readFile').resolves(Buffer.from('')); }); - afterEach(() => sandbox.restore()); + afterEach(() => { + cacheAllImages.clear(); + cacheExpectedPaths.clear(); + cacheDiffImages.clear(); + + sandbox.restore(); + }); it('should do nothing if plugin is disabled', () => { return initReporter_({enabled: false}).then(() => { @@ -257,7 +269,7 @@ describe('lib/hermione', () => { }); it('should save image from passed test', async () => { - utils.getReferencePath.callsFake((test, stateName) => `report/${stateName}`); + utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); await initReporter_({path: '/absolute'}); const testData = mkStubResult_({assertViewResults: [{refImg: {path: 'ref/path'}, stateName: 'plain'}]}); @@ -268,7 +280,7 @@ describe('lib/hermione', () => { }); it('should save image from assert view error', async () => { - utils.getCurrentPath.callsFake((test, stateName) => `report/${stateName}`); + utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); await initReporter_({path: '/absolute'}); const err = new NoRefImageError(); err.stateName = 'plain'; @@ -281,7 +293,7 @@ describe('lib/hermione', () => { }); it('should save reference image from assert view fail', async () => { - utils.getReferencePath.callsFake((test, stateName) => `report/${stateName}`); + utils.getReferencePath.callsFake(({stateName}) => `report/${stateName}`); await initReporter_({path: '/absolute'}); await stubWorkers(); @@ -296,7 +308,7 @@ describe('lib/hermione', () => { }); it('should save current image from assert view fail', async () => { - utils.getCurrentPath.callsFake((test, stateName) => `report/${stateName}`); + utils.getCurrentPath.callsFake(({stateName}) => `report/${stateName}`); await initReporter_({path: '/absolute'}); await hermione.emitAsync(events.RUNNER_START, { registerWorkers: () => { @@ -315,7 +327,7 @@ describe('lib/hermione', () => { it('should save current diff image from assert view fail', async () => { fs.readFile.resolves(Buffer.from('some-buff')); - utils.getDiffPath.callsFake((test, stateName) => `report/${stateName}`); + utils.getDiffPath.callsFake(({stateName}) => `report/${stateName}`); const saveDiffTo = sinon.stub().resolves(); const err = new ImageDiffError(); err.stateName = 'plain'; diff --git a/test/unit/lib/gui/tool-runner/index.js b/test/unit/lib/gui/tool-runner/index.js index 1cf467520..906683776 100644 --- a/test/unit/lib/gui/tool-runner/index.js +++ b/test/unit/lib/gui/tool-runner/index.js @@ -87,6 +87,7 @@ describe('lib/gui/tool-runner/index', () => { } }); + sandbox.stub(reportBuilder, 'imageHandler').value({updateCacheExpectedPath: sinon.stub()}); sandbox.stub(logger, 'warn'); }); @@ -368,11 +369,11 @@ describe('lib/gui/tool-runner/index', () => { it('should update expected path', async () => { const stateName = 'plain'; const previousExpectedPath = 'previousExpectedPath'; - const {formattedResult, gui, tests} = await mkUndoTestData_({previousExpectedPath}, {stateName}); + const {gui, tests} = await mkUndoTestData_({previousExpectedPath}, {stateName}); await gui.undoAcceptImages(tests); - assert.calledOnceWith(formattedResult.updateCacheExpectedPath, stateName, previousExpectedPath); + assert.calledOnceWith(reportBuilder.imageHandler.updateCacheExpectedPath, sinon.match.any, stateName, previousExpectedPath); }); }); diff --git a/test/unit/lib/gui/tool-runner/report-subsciber.js b/test/unit/lib/gui/tool-runner/report-subsciber.js index 404094d52..98b1943d9 100644 --- a/test/unit/lib/gui/tool-runner/report-subsciber.js +++ b/test/unit/lib/gui/tool-runner/report-subsciber.js @@ -33,6 +33,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { reportBuilder = sinon.createStubInstance(GuiReportBuilder); sandbox.stub(GuiReportBuilder, 'create').returns(reportBuilder); reportBuilder.format.returns(mkTestAdapterStub_()); + sandbox.stub(reportBuilder, 'imageHandler').value({saveTestImages: sinon.stub()}); client = new EventEmitter(); sandbox.spy(client, 'emit'); @@ -54,10 +55,10 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { const hermione = mkHermione_(); const testResult = 'test-result'; const mediator = sinon.spy().named('mediator'); - const saveTestImages = sandbox.stub().callsFake(() => Promise.delay(100).then(mediator)); - const formattedResult = mkTestAdapterStub_({saveTestImages}); + const formattedResult = mkTestAdapterStub_(); reportBuilder.format.withArgs(testResult, hermione.events.TEST_FAIL).returns(formattedResult); + reportBuilder.imageHandler.saveTestImages.callsFake(() => Promise.delay(100).then(mediator)); reportSubscriber(hermione, reportBuilder, client); hermione.emit(hermione.events.TEST_FAIL, testResult); @@ -132,7 +133,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { it('should save images before fail adding', async () => { const hermione = mkHermione_(); const testData = 'test-data'; - const formattedResult = mkTestAdapterStub_({saveTestImages: sandbox.stub()}); + const formattedResult = mkTestAdapterStub_(); reportBuilder.format.withArgs(testData, hermione.events.TEST_FAIL).returns(formattedResult); @@ -140,7 +141,7 @@ describe('lib/gui/tool-runner/hermione/report-subscriber', () => { hermione.emit(hermione.events.TEST_FAIL, testData); await hermione.emitAsync(hermione.events.RUNNER_END); - assert.callOrder(formattedResult.saveTestImages, reportBuilder.addFail); + assert.callOrder(reportBuilder.imageHandler.saveTestImages, reportBuilder.addFail); }); it('should emit "TEST_RESULT" event for client with test data', async () => { diff --git a/test/unit/lib/image-handler.ts b/test/unit/lib/image-handler.ts new file mode 100644 index 000000000..eedce0b00 --- /dev/null +++ b/test/unit/lib/image-handler.ts @@ -0,0 +1,439 @@ +import * as fsOriginal from 'fs-extra'; +import _ from 'lodash'; +import proxyquire from 'proxyquire'; +import sinon, {SinonStubbedInstance} from 'sinon'; +import type tmpOriginal from 'tmp'; + +import type * as originalUtils from 'lib/server-utils'; +import {logger} from 'lib/common-utils'; +import {ImageHandler as ImageHandlerOriginal} from 'lib/image-handler'; +import {RegisterWorkers} from 'lib/workers/create-workers'; +import {AssertViewResult, ImageInfoFull, ImageInfoSuccess, ImagesSaver, TestResult} from 'lib/types'; +import {ErrorName, ImageDiffError} from 'lib/errors'; +import {ImageStore} from 'lib/image-store'; +import {FAIL, PluginEvents, SUCCESS, UPDATED} from 'lib/constants'; + +describe('image-handler', function() { + const sandbox = sinon.sandbox.create(); + let fs: sinon.SinonStubbedInstance; + let utils: sinon.SinonStubbedInstance; + let tmp: typeof tmpOriginal; + let err: AssertViewResult; + let ImageHandler: typeof ImageHandlerOriginal; + const cacheExpectedPaths = new Map(), + cacheAllImages = new Map(), + cacheDiffImages = new Map(); + + class ImageDiffErrorStub extends Error { + name = ErrorName.IMAGE_DIFF; + } + class NoRefImageErrorStub extends Error { + name = ErrorName.NO_REF_IMAGE; + } + + const mkImageStore = (): SinonStubbedInstance => ({getLastImageInfoFromDb: sinon.stub()} as SinonStubbedInstance); + + const mkImagesSaver = (): SinonStubbedInstance => ({saveImg: sinon.stub()} as SinonStubbedInstance); + + const mkTestResult = (result: Partial): TestResult => _.defaults(result, { + id: 'some-id', + fullTitle: () => 'default-title' + }) as TestResult; + + const mkErrStub = (ErrType = ImageDiffErrorStub, {stateName, currImg, refImg, diffBuffer}: Partial = {}): AssertViewResult => { + const err: AssertViewResult = new ErrType() as any; + + err.stateName = stateName || 'plain'; + err.currImg = currImg || {path: 'curr/path'} as any; + err.refImg = refImg || {path: 'ref/path'} as any; + (err as ImageDiffError).diffBuffer = diffBuffer; + + return err; + }; + + const mkWorker = (): sinon.SinonStubbedInstance> => { + return {saveDiffTo: sandbox.stub()} as any; + }; + + beforeEach(() => { + fs = sinon.stub(_.clone(fsOriginal)); + err = mkErrStub(); + tmp = {tmpdir: 'default/dir'} as any; + + const originalUtils = proxyquire('lib/server-utils', { + 'fs-extra': fs + }); + utils = _.clone(originalUtils); + + ImageHandler = proxyquire('lib/image-handler', { + tmp, + 'fs-extra': fs, + './server-utils': utils, + './image-cache': {cacheExpectedPaths, cacheAllImages, cacheDiffImages} + }).ImageHandler; + + sandbox.stub(utils, 'getCurrentPath').returns(''); + sandbox.stub(utils, 'getDiffPath').returns(''); + sandbox.stub(utils, 'getReferencePath').returns(''); + + fs.readFile.resolves(Buffer.from('')); + fs.writeFile.resolves(); + fs.copy.resolves(); + }); + + afterEach(() => { + sandbox.restore(); + + cacheExpectedPaths.clear(); + cacheAllImages.clear(); + cacheDiffImages.clear(); + }); + + describe('saveTestImages', () => { + it('should build diff to tmp dir', async () => { + (tmp as any).tmpdir = 'tmp/dir'; + const testResult = mkTestResult({ + assertViewResults: [err] + }); + utils.getDiffPath.returns('diff/report/path'); + + const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: 'some-dir'}); + const worker = mkWorker(); + await imageHandler.saveTestImages(testResult, 0, worker); + + assert.calledOnceWith(worker.saveDiffTo, err, sinon.match('tmp/dir/diff/report/path')); + }); + + it('should save diff in report from tmp dir using external storage', async () => { + (tmp as any).tmpdir = 'tmp/dir'; + const testResult = mkTestResult({ + assertViewResults: [err] + }); + utils.getDiffPath.returns('diff/report/path'); + const imagesSaver = mkImagesSaver(); + const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'html-report/path'}); + const worker = mkWorker(); + await imageHandler.saveTestImages(testResult, 0, worker); + + assert.calledWith( + imagesSaver.saveImg, + sinon.match('tmp/dir/diff/report/path'), + {destPath: 'diff/report/path', reportDir: 'html-report/path'} + ); + }); + + it('should emit TEST_SCREENSHOTS_SAVED event', async () => { + (tmp as any).tmpdir = 'tmp/dir'; + const testResult = mkTestResult({ + browserId: 'chrome', + assertViewResults: [err] + }); + utils.getDiffPath.returns('diff/report/path'); + + const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + sinon.stub(imageHandler, 'getImagesInfo').returns([{test: 123}]); + const worker = mkWorker(); + + const screenshotsSavedHandler = sinon.stub(); + imageHandler.on(PluginEvents.TEST_SCREENSHOTS_SAVED, screenshotsSavedHandler); + + await imageHandler.saveTestImages(testResult, 0, worker); + + assert.calledOnceWith(screenshotsSavedHandler, { + attempt: 0, + testId: 'default-title.chrome', + imagesInfo: [{test: 123}] + }); + }); + + describe('saving error screenshot', () => { + beforeEach(() => { + sandbox.stub(logger, 'warn'); + sandbox.stub(utils, 'makeDirFor').resolves(); + sandbox.stub(utils, 'copyFileAsync'); + }); + + describe('if screenshot on reject does not exist', () => { + it('should not save screenshot', () => { + const testResult = mkTestResult({ + err: {screenshot: {base64: null}} as any, + assertViewResults: [] + }); + const hermioneTestAdapter = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + + return hermioneTestAdapter.saveTestImages(testResult, 0, mkWorker()) + .then(() => assert.notCalled(fs.writeFile)); + }); + + it('should warn about it', () => { + const testResult = mkTestResult({ + err: {screenshot: {base64: null}} as any, + assertViewResults: [] + }); + const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + + return imageHandler.saveTestImages(testResult, 0, mkWorker()) + .then(() => assert.calledWith(logger.warn as sinon.SinonStub, 'Cannot save screenshot on reject')); + }); + }); + + it('should create directory for screenshot', () => { + const testResult = mkTestResult({ + err: {screenshot: {base64: 'base64-data'}} as any, + assertViewResults: [] + }); + utils.getCurrentPath.returns('dest/path'); + const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + + return imageHandler.saveTestImages(testResult, 0, mkWorker()) + .then(() => assert.calledOnceWith(utils.makeDirFor, sinon.match('dest/path'))); + }); + + it('should save screenshot from base64 format', async () => { + const testResult = mkTestResult({ + err: {screenshot: {base64: 'base64-data'}} as any, + assertViewResults: [] + }); + utils.getCurrentPath.returns('dest/path'); + const bufData = new Buffer('base64-data', 'base64'); + const imagesSaver = mkImagesSaver(); + const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'report/path'}); + + await imageHandler.saveTestImages(testResult, 0, mkWorker()); + + assert.calledOnceWith(fs.writeFile, sinon.match('dest/path'), bufData, 'base64'); + assert.calledWith(imagesSaver.saveImg, sinon.match('dest/path'), {destPath: 'dest/path', reportDir: 'report/path'}); + }); + }); + + describe('saving reference image', () => { + it('should save reference, if it is not reused', async () => { + (tmp as any).tmpdir = 'tmp/dir'; + const testResult = mkTestResult({assertViewResults: [err]}); + utils.getReferencePath.returns('ref/report/path'); + const imagesSaver = mkImagesSaver(); + const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'html-report/path'}); + + await imageHandler.saveTestImages(testResult, 0, mkWorker()); + + assert.calledWith( + imagesSaver.saveImg, 'ref/path', + {destPath: 'ref/report/path', reportDir: 'html-report/path'} + ); + }); + + it('should not save reference, if it is reused', async () => { + (tmp as any).tmpdir = 'tmp/dir'; + const error = mkErrStub(ImageDiffErrorStub, {stateName: 'plain'}); + const testResult = mkTestResult({assertViewResults: [error], browserId: 'browser-id'}); + utils.getReferencePath.returns('ref/report/path'); + const imagesSaver = mkImagesSaver(); + cacheExpectedPaths.set('da89771#plain', 'ref/report/path'); + const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'html-report/path'}); + + await imageHandler.saveTestImages(testResult, 0, mkWorker()); + + assert.neverCalledWith( + imagesSaver.saveImg, 'ref/path', + {destPath: 'ref/report/path', reportDir: 'html-report/path'} + ); + }); + + it('should save png buffer, if it is passed', async () => { + const error = mkErrStub(ImageDiffErrorStub, {stateName: 'plain', diffBuffer: 'foo' as any}); + const testResult = mkTestResult({assertViewResults: [error]}); + utils.getDiffPath.returns('diff/report/path'); + + const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + const workers = {saveDiffTo: sandbox.stub()}; + await imageHandler.saveTestImages(testResult, 0, mkWorker()); + + assert.calledOnceWith(fs.writeFile, sinon.match('diff/report/path'), Buffer.from('foo')); + assert.notCalled(workers.saveDiffTo); + }); + }); + }); + + ([ + {field: 'refImg', method: 'getRefImg'}, + {field: 'currImg', method: 'getCurrImg'} + ] as const).forEach(({field, method}) => { + describe(`${method}`, () => { + it(`should return ${field} from test result`, () => { + const testResult = mkTestResult({assertViewResults: [ + {[field]: 'some-value', stateName: 'plain'} as any]}); + + const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + + assert.equal((imageHandler[method])(testResult.assertViewResults, 'plain'), 'some-value' as any); + }); + }); + }); + + describe('getScreenshot', () => { + it('should return error screenshot from test result', () => { + const testResult = mkTestResult({err: {screenshot: 'some-value'} as any}); + + const hermioneTestAdapter = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + + assert.equal(hermioneTestAdapter.getScreenshot(testResult), 'some-value' as any); + }); + }); + + describe('getImagesInfo', () => { + beforeEach(() => { + sandbox.stub(utils, 'copyFileAsync'); + utils.getReferencePath.returns('some/ref.png'); + }); + + it('should return diffClusters', () => { + const testResult = mkTestResult({ + assertViewResults: [{diffClusters: [{left: 0, top: 0, right: 1, bottom: 1}]}] as any + }); + const imageHandler = new ImageHandler(mkImageStore(), mkImagesSaver(), {reportPath: ''}); + + const [{diffClusters}] = imageHandler.getImagesInfo(testResult, 0); + + assert.deepEqual(diffClusters, [{left: 0, top: 0, right: 1, bottom: 1}]); + }); + + it('should return saved images', async () => { + const testResult = mkTestResult({ + assertViewResults: [mkErrStub()] + }); + + const imagesSaver = mkImagesSaver(); + imagesSaver.saveImg.withArgs( + 'ref/path', + {destPath: 'some/ref.png', reportDir: 'some/rep'} + ).returns('saved/ref.png'); + const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'some/rep'}); + const workers = mkWorker(); + + await imageHandler.saveTestImages(testResult, 0, workers); + + const {expectedImg} = imageHandler.getImagesFor(testResult, 0, SUCCESS, 'plain') as ImageInfoSuccess; + assert.equal(expectedImg.path, 'saved/ref.png'); + }); + + it('should return dest image path by default', async () => { + const testResult = mkTestResult({ + assertViewResults: [mkErrStub()] + }); + + const imagesSaver = mkImagesSaver(); + const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'some/rep'}); + const workers = mkWorker(); + + await imageHandler.saveTestImages(testResult, 0, workers); + + const {expectedImg} = imageHandler.getImagesFor(testResult, 0, SUCCESS, 'plain') as ImageInfoSuccess; + assert.equal(expectedImg.path, 'some/ref.png'); + }); + + it('should return ref image path after update image for NoRefImageError', async () => { + const testResult = mkTestResult({ + assertViewResults: [mkErrStub(NoRefImageErrorStub)] + }); + + const imagesSaver = mkImagesSaver(); + const imageHandler = new ImageHandler(mkImageStore(), imagesSaver, {reportPath: 'some/rep'}); + const workers = mkWorker(); + + await imageHandler.saveTestImages(testResult, 0, workers); + + const {expectedImg} = imageHandler.getImagesFor(testResult, 0, UPDATED, 'plain') as ImageInfoSuccess; + assert.equal(expectedImg.path, 'some/ref.png'); + }); + + describe('expected path', () => { + const mkLastImageInfo_ = (opts = {}): ImageInfoFull => { + const {stateName, expectedImgPath} = _.defaults(opts, { + stateName: 'plain', + expectedImgPath: 'default/expected/img/path.png' + }); + + return { + stateName, + expectedImg: { + path: expectedImgPath + } + } as any; + }; + + it('should be pulled from the store if exists', async () => { + const testResult = mkTestResult({ + fullTitle: () => 'some-title', + assertViewResults: [mkErrStub()] + }); + const imageStore = mkImageStore(); + imageStore.getLastImageInfoFromDb.withArgs(testResult, 'plain').returns(mkLastImageInfo_()); + + const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); + + imageHandler.getImagesFor(testResult, 0, FAIL, 'plain'); + + assert.notCalled(utils.getReferencePath); + }); + + it('should be generated if does not exist in store', async () => { + const testResult = mkTestResult({ + fullTitle: () => 'some-title', + assertViewResults: [mkErrStub()] + }); + const imageStore = mkImageStore(); + imageStore.getLastImageInfoFromDb.withArgs(testResult, 'plain').returns(undefined); + + const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); + + imageHandler.getImagesFor(testResult, 0, FAIL, 'plain'); + + assert.calledOnce(utils.getReferencePath); + }); + + it('should be generated on update', async () => { + const testResult = mkTestResult({ + fullTitle: () => 'some-title', + assertViewResults: [mkErrStub()] + }); + const imageStore = mkImageStore(); + imageStore.getLastImageInfoFromDb.withArgs(testResult, 'plain').returns(mkLastImageInfo_()); + const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); + + imageHandler.getImagesFor(testResult, 0, UPDATED, 'plain'); + + assert.calledOnce(utils.getReferencePath); + }); + + it('should be queried from the database for each browser', async () => { + const chromeTestResult = mkTestResult({browserId: 'chrome'}); + const firefoxTestResult = mkTestResult({browserId: 'firefox'}); + + const imageStore = mkImageStore(); + const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); + + imageHandler.getImagesFor(chromeTestResult, 0, FAIL, 'plain'); + imageHandler.getImagesFor(firefoxTestResult, 0, FAIL, 'plain'); + + assert.calledTwice(imageStore.getLastImageInfoFromDb); + assert.calledWith(imageStore.getLastImageInfoFromDb.firstCall, chromeTestResult, 'plain'); + assert.calledWith(imageStore.getLastImageInfoFromDb.secondCall, firefoxTestResult, 'plain'); + }); + + it('should be queried from the database once per state', async () => { + const testResult = mkTestResult({ + fullTitle: () => 'some-title', + assertViewResults: [mkErrStub()] + }); + const imageStore = mkImageStore(); + imageStore.getLastImageInfoFromDb.returns(mkLastImageInfo_()); + const imageHandler = new ImageHandler(imageStore, mkImagesSaver(), {reportPath: ''}); + + imageHandler.getImagesFor(testResult, 0, FAIL, 'plain'); + imageHandler.getImagesFor(testResult, 0, FAIL, 'plain'); + + assert.calledOnce(imageStore.getLastImageInfoFromDb); + }); + }); + }); +}); diff --git a/test/unit/lib/report-builder/gui.js b/test/unit/lib/report-builder/gui.js index 58d6a4ad5..584175f90 100644 --- a/test/unit/lib/report-builder/gui.js +++ b/test/unit/lib/report-builder/gui.js @@ -31,7 +31,7 @@ describe('GuiReportBuilder', () => { TestAdapter.create = (obj) => obj; - const reportBuilder = GuiReportBuilder.create(hermione, pluginConfig); + const reportBuilder = GuiReportBuilder.create(hermione.htmlReporter, pluginConfig); await reportBuilder.init(); return reportBuilder; diff --git a/test/unit/lib/report-builder/static.js b/test/unit/lib/report-builder/static.js index 04e98f13d..e43964de4 100644 --- a/test/unit/lib/report-builder/static.js +++ b/test/unit/lib/report-builder/static.js @@ -38,7 +38,7 @@ describe('StaticReportBuilder', () => { }) }; - const reportBuilder = StaticReportBuilder.create(hermione, pluginConfig); + const reportBuilder = StaticReportBuilder.create(hermione.htmlReporter, pluginConfig); await reportBuilder.init(); return reportBuilder; diff --git a/test/unit/lib/server-utils.js b/test/unit/lib/server-utils.js index d61af60f8..50f680c50 100644 --- a/test/unit/lib/server-utils.js +++ b/test/unit/lib/server-utils.js @@ -44,10 +44,11 @@ describe('server-utils', () => { it('should add state name to the path if it was passed', () => { const test = { imageDir: 'some/dir', - browserId: 'bro' + browserId: 'bro', + stateName: 'plain' }; - const resultPath = utils[`get${testData.name}Path`](test, 'plain'); + const resultPath = utils[`get${testData.name}Path`](test); assert.equal(resultPath, path.join(IMAGES_PATH, 'some', 'dir', `plain/bro~${testData.prefix}_0.png`)); }); diff --git a/test/unit/lib/sqlite-adapter.js b/test/unit/lib/sqlite-adapter.js index 3bf15e344..90a2866ce 100644 --- a/test/unit/lib/sqlite-adapter.js +++ b/test/unit/lib/sqlite-adapter.js @@ -9,16 +9,16 @@ const {HtmlReporter} = require('lib/plugin-api'); describe('lib/sqlite-adapter', () => { const sandbox = sinon.createSandbox(); - let hermione; + let htmlReporter; const makeSqliteAdapter_ = async () => { - const sqliteAdapter = SqliteAdapter.create({hermione, reportPath: 'test'}); + const sqliteAdapter = SqliteAdapter.create({htmlReporter, reportPath: 'test'}); await sqliteAdapter.init(); return sqliteAdapter; }; beforeEach(() => { - hermione = {htmlReporter: HtmlReporter.create()}; + htmlReporter = HtmlReporter.create(); }); afterEach(() => { @@ -64,7 +64,7 @@ describe('lib/sqlite-adapter', () => { it('should emit "DATABASE_CREATED" event with new database connection', async () => { const onDatabaseCreated = sinon.spy(); - hermione.htmlReporter.on(hermione.htmlReporter.events.DATABASE_CREATED, onDatabaseCreated); + htmlReporter.on(htmlReporter.events.DATABASE_CREATED, onDatabaseCreated); await makeSqliteAdapter_(); @@ -79,7 +79,7 @@ describe('lib/sqlite-adapter', () => { prepareStub = sandbox.stub(Database.prototype, 'prepare').returns({get: getStub}); sqliteAdapter = proxyquire('lib/sqlite-adapter', { './db-utils/common': {createTablesQuery: () => []} - }).SqliteAdapter.create({hermione, reportPath: 'test'}); + }).SqliteAdapter.create({htmlReporter, reportPath: 'test'}); await sqliteAdapter.init(); }); @@ -147,7 +147,7 @@ describe('lib/sqlite-adapter', () => { prepareStub = sandbox.stub(Database.prototype, 'prepare').returns({run: runStub}); sqliteAdapter = proxyquire('lib/sqlite-adapter', { './db-utils/common': {createTablesQuery: () => []} - }).SqliteAdapter.create({hermione, reportPath: 'test'}); + }).SqliteAdapter.create({htmlReporter, reportPath: 'test'}); await sqliteAdapter.init(); }); diff --git a/test/unit/lib/test-adapter.js b/test/unit/lib/test-adapter.js deleted file mode 100644 index 891b62526..000000000 --- a/test/unit/lib/test-adapter.js +++ /dev/null @@ -1,780 +0,0 @@ -'use strict'; - -const _ = require('lodash'); -const {logger} = require('lib/common-utils'); -const {SUCCESS, UPDATED, SKIPPED, FAIL} = require('lib/constants/test-statuses'); -const {ERROR_DETAILS_PATH} = require('lib/constants/paths'); -const {stubTool, stubConfig} = require('../utils'); -const {SqliteAdapter} = require('lib/sqlite-adapter'); -const proxyquire = require('proxyquire'); -const fsOriginal = require('fs-extra'); - -describe('hermione test adapter', () => { - const sandbox = sinon.sandbox.create(); - let fs, tmp, HermioneTestResultAdapter, err, getSuitePath, getCommandsHistory, sqliteAdapter, utils; - - class ImageDiffError extends Error {} - class NoRefImageError extends Error {} - - const mkHermioneTestResultAdapter = (testResult, { - toolOpts = {}, htmlReporter = {}, status - } = {}) => { - const config = _.defaults(toolOpts.config, { - browsers: { - bro: {} - } - }); - - const hermione = stubTool( - stubConfig(config), - {}, - {ImageDiffError, NoRefImageError}, - Object.assign({ - imagesSaver: {saveImg: sandbox.stub()}, - events: {TEST_SCREENSHOTS_SAVED: 'testScreenshotsSaved'}, - emitAsync: sinon.stub() - }, htmlReporter) - ); - - return new HermioneTestResultAdapter(testResult, {hermione, sqliteAdapter, status}); - }; - - const mkTestResult_ = (result) => _.defaults(result, { - id: 'some-id', - fullTitle: () => 'default-title' - }); - - const mkErrStub = (ErrType = ImageDiffError, {stateName, currImg, refImg, diffBuffer} = {}) => { - const err = new ErrType(); - - err.stateName = stateName || 'plain'; - err.currImg = currImg || {path: 'curr/path'}; - err.refImg = refImg || {path: 'ref/path'}; - err.diffBuffer = diffBuffer; - - return err; - }; - - beforeEach(() => { - tmp = {tmpdir: 'default/dir'}; - fs = sinon.stub(_.clone(fsOriginal)); - getSuitePath = sandbox.stub(); - getCommandsHistory = sandbox.stub(); - sqliteAdapter = sandbox.createStubInstance(SqliteAdapter); - - const originalUtils = proxyquire('lib/server-utils', { - 'fs-extra': fs - }); - utils = _.clone(originalUtils); - - HermioneTestResultAdapter = proxyquire('lib/test-adapter', { - tmp, - 'fs-extra': fs, - './plugin-utils': {getSuitePath}, - './history-utils': {getCommandsHistory}, - './server-utils': utils - }).TestAdapter; - sandbox.stub(utils, 'getCurrentPath').returns(''); - sandbox.stub(utils, 'getDiffPath').returns(''); - sandbox.stub(utils, 'getReferencePath').returns(''); - - fs.readFile.resolves(Buffer.from('')); - fs.writeFile.resolves(); - fs.copy.resolves(); - - err = mkErrStub(); - }); - - afterEach(() => sandbox.restore()); - - it('should return suite attempt', () => { - const firstTestResult = mkTestResult_({fullTitle: () => 'some-title'}); - const secondTestResult = mkTestResult_({fullTitle: () => 'other-title'}); - - mkHermioneTestResultAdapter(firstTestResult); - - assert.equal(mkHermioneTestResultAdapter(firstTestResult).attempt, 1); - assert.equal(mkHermioneTestResultAdapter(secondTestResult).attempt, 0); - }); - - it('should not increment attempt for skipped tests', () => { - const testResult = mkTestResult_({fullTitle: () => 'some-title'}); - - mkHermioneTestResultAdapter(testResult, {status: SKIPPED}); - const result = mkHermioneTestResultAdapter(testResult, {status: SKIPPED}); - - assert.equal(result.attempt, 0); - }); - - it('should return test error with "message", "stack" and "stateName"', () => { - getCommandsHistory.withArgs([{name: 'foo'}], ['foo']).returns(['some-history']); - const testResult = mkTestResult_({ - file: 'bar', - history: [{name: 'foo'}], - err: { - message: 'some-message', - stack: 'some-stack', - stateName: 'some-test', - foo: 'bar' - } - }); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.error, { - message: 'some-message', - stack: 'some-stack', - stateName: 'some-test' - }); - }); - - it('should return test history', () => { - getCommandsHistory.withArgs([{name: 'foo'}]).returns(['some-history']); - const testResult = mkTestResult_({ - file: 'bar', - history: [{name: 'foo'}], - err: { - message: 'some-message', - stack: 'some-stack', - stateName: 'some-test', - foo: 'bar' - } - }); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.history, ['some-history']); - }); - - it('should return test state', () => { - const testResult = mkTestResult_({title: 'some-test'}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.state, {name: 'some-test'}); - }); - - it('should return assert view results', () => { - const testResult = mkTestResult_({assertViewResults: [1]}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.assertViewResults, [1]); - }); - - describe('error details', () => { - let getDetailsFileName; - - beforeEach(() => { - getDetailsFileName = sandbox.stub(utils, 'getDetailsFileName').returns(''); - }); - - it('should be returned for test if they are available', () => { - const testResult = mkTestResult_({ - err: { - details: {title: 'some-title', data: {foo: 'bar'}} - } - }); - getDetailsFileName.returns('md5-bro-n-time'); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.errorDetails, { - title: 'some-title', - data: {foo: 'bar'}, - filePath: `${ERROR_DETAILS_PATH}/md5-bro-n-time` - }); - }); - - it('should have "error details" title if no title is given', () => { - const testResult = mkTestResult_({err: {details: {}}}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.propertyVal(hermioneTestAdapter.errorDetails, 'title', 'error details'); - }); - - it('should be memoized', () => { - const testResult = mkTestResult_({ - err: { - details: {title: 'some-title', data: {foo: 'bar'}} - } - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - const firstErrDetails = hermioneTestAdapter.errorDetails; - const secondErrDetails = hermioneTestAdapter.errorDetails; - - assert.calledOnce(getDetailsFileName); - assert.deepEqual(firstErrDetails, secondErrDetails); - }); - - it('should be returned as null if absent', () => { - const testResult = mkTestResult_({err: {}}); - - const {errorDetails} = mkHermioneTestResultAdapter(testResult); - - assert.isNull(errorDetails); - }); - - it('should use test id, browser-id and attempt for filepath composing', () => { - const testResult = mkTestResult_({ - id: 'abcdef', - browserId: 'bro', - err: { - details: {data: {foo: 'bar'}} - } - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - // we need to get errorDetails to trigger getDetailsFileName to be called - hermioneTestAdapter.errorDetails; - - assert.calledWith(getDetailsFileName, 'abcdef', 'bro', hermioneTestAdapter.attempt); - }); - }); - - describe('saveErrorDetails', () => { - beforeEach(() => { - sandbox.stub(utils, 'makeDirFor').resolves(); - sandbox.stub(utils, 'getDetailsFileName').returns('md5-bro-n-time'); - }); - - it('should do nothing if no error details are available', async () => { - const hermioneTestAdapter = mkHermioneTestResultAdapter(mkTestResult_({err: {}})); - - await hermioneTestAdapter.saveErrorDetails(); - - assert.notCalled(fs.writeFile); - }); - - it('should save error details to correct path', async () => { - const testResult = mkTestResult_({err: { - details: {title: 'some-title', data: {}} - }}); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - const {filePath} = hermioneTestAdapter.errorDetails; - - await hermioneTestAdapter.saveErrorDetails('report-path'); - - assert.calledWithMatch(fs.writeFile, `report-path/${filePath}`, sinon.match.any); - }); - - it('should create directory for error details', async () => { - const testResult = mkTestResult_({err: {details: {data: {}}}}); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - await hermioneTestAdapter.saveErrorDetails('report-path'); - - assert.calledOnceWith(utils.makeDirFor, sinon.match(`report-path/${ERROR_DETAILS_PATH}`)); - }); - - it('should save error details', async () => { - const data = {foo: 'bar'}; - const testResult = mkTestResult_({err: {details: {data}}}); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - await hermioneTestAdapter.saveErrorDetails(''); - - assert.calledWith(fs.writeFile, sinon.match.any, JSON.stringify(data, null, 2)); - }); - }); - - describe('saveTestImages', () => { - it('should build diff to tmp dir', async () => { - tmp.tmpdir = 'tmp/dir'; - const testResult = mkTestResult_({ - assertViewResults: [err] - }); - utils.getDiffPath.returns('diff/report/path'); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - const workers = {saveDiffTo: sandbox.stub()}; - await hermioneTestAdapter.saveTestImages('', workers); - - assert.calledOnceWith(workers.saveDiffTo, err, sinon.match('tmp/dir/diff/report/path')); - }); - - it('should save diff in report from tmp dir using external storage', async () => { - tmp.tmpdir = 'tmp/dir'; - const testResult = mkTestResult_({ - assertViewResults: [err] - }); - utils.getDiffPath.returns('diff/report/path'); - const imagesSaver = {saveImg: sandbox.stub()}; - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - imagesSaver - } - }); - const workers = {saveDiffTo: sandbox.stub()}; - await hermioneTestAdapter.saveTestImages('html-report/path', workers); - - assert.calledWith( - imagesSaver.saveImg, - sinon.match('tmp/dir/diff/report/path'), - {destPath: 'diff/report/path', reportDir: 'html-report/path'} - ); - }); - - it('should emit TEST_SCREENSHOTS_SAVED event', async () => { - tmp.tmpdir = 'tmp/dir'; - const testResult = mkTestResult_({ - browserId: 'chrome', - assertViewResults: [err] - }); - utils.getDiffPath.returns('diff/report/path'); - - const htmlReporterEmitStub = sinon.stub(); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - emitAsync: htmlReporterEmitStub - } - }); - sinon.stub(hermioneTestAdapter, 'getImagesInfo').returns([{test: 123}]); - const workers = {saveDiffTo: sandbox.stub()}; - - await hermioneTestAdapter.saveTestImages('', workers); - - assert.calledOnceWith(htmlReporterEmitStub, 'testScreenshotsSaved', { - attempt: 0, - testId: 'default-title.chrome', - imagesInfo: [{test: 123}] - }); - }); - - describe('saving error screenshot', () => { - beforeEach(() => { - sandbox.stub(logger, 'warn'); - sandbox.stub(utils, 'makeDirFor').resolves(); - sandbox.stub(utils, 'copyFileAsync'); - }); - - describe('if screenshot on reject does not exist', () => { - it('should not save screenshot', () => { - const testResult = mkTestResult_({ - err: {screenshot: {base64: null}}, - assertViewResults: [] - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - return hermioneTestAdapter.saveTestImages() - .then(() => assert.notCalled(fs.writeFile)); - }); - - it('should warn about it', () => { - const testResult = mkTestResult_({ - err: {screenshot: {base64: null}}, - assertViewResults: [] - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - return hermioneTestAdapter.saveTestImages() - .then(() => assert.calledWith(logger.warn, 'Cannot save screenshot on reject')); - }); - }); - - it('should create directory for screenshot', () => { - const testResult = mkTestResult_({ - err: {screenshot: {base64: 'base64-data'}}, - assertViewResults: [] - }); - utils.getCurrentPath.returns('dest/path'); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - return hermioneTestAdapter.saveTestImages() - .then(() => assert.calledOnceWith(utils.makeDirFor, sinon.match('dest/path'))); - }); - - it('should save screenshot from base64 format', async () => { - const testResult = mkTestResult_({ - err: {screenshot: {base64: 'base64-data'}}, - assertViewResults: [] - }); - utils.getCurrentPath.returns('dest/path'); - const bufData = new Buffer('base64-data', 'base64'); - const imagesSaver = {saveImg: sandbox.stub()}; - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - imagesSaver - } - }); - - await hermioneTestAdapter.saveTestImages('report/path'); - - assert.calledOnceWith(fs.writeFile, sinon.match('dest/path'), bufData, 'base64'); - assert.calledWith(imagesSaver.saveImg, sinon.match('dest/path'), {destPath: 'dest/path', reportDir: 'report/path'}); - }); - }); - - describe('saving reference image', () => { - it('should save reference, if it is not reused', async () => { - tmp.tmpdir = 'tmp/dir'; - const testResult = mkTestResult_({assertViewResults: [err]}); - utils.getReferencePath.returns('ref/report/path'); - const imagesSaver = {saveImg: sandbox.stub()}; - const cacheExpectedPaths = new Map(); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - imagesSaver - } - }); - - await hermioneTestAdapter.saveTestImages( - 'html-report/path', - {saveDiffTo: sandbox.stub()}, - cacheExpectedPaths - ); - - assert.calledWith( - imagesSaver.saveImg, 'ref/path', - {destPath: 'ref/report/path', reportDir: 'html-report/path'} - ); - }); - - it('should not save reference, if it is reused', async () => { - tmp.tmpdir = 'tmp/dir'; - const error = mkErrStub(ImageDiffError, {stateName: 'plain'}); - const testResult = mkTestResult_({assertViewResults: [error], browserId: 'browser-id'}); - utils.getReferencePath.returns('ref/report/path'); - const imagesSaver = {saveImg: sandbox.stub()}; - const cacheExpectedPaths = new Map([['da89771#plain', 'ref/report/path']]); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - imagesSaver - } - }); - - await hermioneTestAdapter.saveTestImages( - 'html-report/path', - {saveDiffTo: sandbox.stub()}, - cacheExpectedPaths - ); - - assert.neverCalledWith( - imagesSaver.saveImg, 'ref/path', - {destPath: 'ref/report/path', reportDir: 'html-report/path'} - ); - }); - - it('should save png buffer, if it is passed', async () => { - const error = mkErrStub(ImageDiffError, {stateName: 'plain', diffBuffer: 'foo'}); - const testResult = mkTestResult_({assertViewResults: [error]}); - utils.getDiffPath.returns('diff/report/path'); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - const workers = {saveDiffTo: sandbox.stub()}; - await hermioneTestAdapter.saveTestImages('', workers); - - assert.calledOnceWith(fs.writeFile, sinon.match('diff/report/path'), Buffer.from('foo')); - assert.notCalled(workers.saveDiffTo); - }); - }); - }); - - describe('hasDiff()', () => { - it('should return true if test has image diff errors', () => { - const testResult = mkTestResult_({assertViewResults: [new ImageDiffError()]}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - toolOpts: { - errors: {ImageDiffError} - } - }); - - assert.isTrue(hermioneTestAdapter.hasDiff()); - }); - - it('should return false if test has not image diff errors', () => { - const testResult = mkTestResult_({assertViewResults: [new Error()]}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - toolOpts: { - errors: {ImageDiffError} - } - }); - - assert.isFalse(hermioneTestAdapter.hasDiff()); - }); - }); - - it('should return image dir', () => { - const testResult = mkTestResult_({id: 'some-id'}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.imageDir, 'some-id'); - }); - - it('should return description', () => { - const testResult = mkTestResult_({description: 'some-description'}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.deepEqual(hermioneTestAdapter.description, 'some-description'); - }); - - [ - {field: 'refImg', method: 'getRefImg'}, - {field: 'currImg', method: 'getCurrImg'} - ].forEach(({field, method}) => { - describe(`${method}`, () => { - it(`should return ${field} from test result`, () => { - const testResult = mkTestResult_({assertViewResults: [ - {[field]: 'some-value', stateName: 'plain'} - ]}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.equal(hermioneTestAdapter[method]('plain'), 'some-value'); - }); - }); - }); - - describe('getErrImg', () => { - it('should return error screenshot from test result', () => { - const testResult = mkTestResult_({err: {screenshot: 'some-value'}}); - - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.equal(hermioneTestAdapter.getErrImg(), 'some-value'); - }); - }); - - describe('prepareTestResult()', () => { - it('should return correct "name" field', () => { - const testResult = mkTestResult_({ - root: true, - title: 'some-title' - }); - - const result = mkHermioneTestResultAdapter(testResult).prepareTestResult(); - - assert.propertyVal(result, 'name', 'some-title'); - }); - - it('should return correct "suitePath" field', () => { - const parentSuite = {parent: {root: true}, title: 'root-title'}; - const testResult = mkTestResult_({ - parent: parentSuite, - title: 'some-title' - }); - getSuitePath.returns(['root-title', 'some-title']); - - const result = mkHermioneTestResultAdapter(testResult).prepareTestResult(); - - assert.deepEqual(result.suitePath, ['root-title', 'some-title']); - }); - - it('should return "browserId" field as is', () => { - const testResult = mkTestResult_({ - root: true, - browserId: 'bro' - }); - - const result = mkHermioneTestResultAdapter(testResult).prepareTestResult(); - - assert.propertyVal(result, 'browserId', 'bro'); - }); - }); - - describe('getImagesInfo()', () => { - beforeEach(() => { - sandbox.stub(utils, 'copyFileAsync'); - utils.getReferencePath.returns('some/ref.png'); - }); - - it('should not reinit "imagesInfo"', () => { - const testResult = mkTestResult_({imagesInfo: [1, 2]}); - - mkHermioneTestResultAdapter(testResult).getImagesInfo(); - - assert.deepEqual(testResult.imagesInfo, [1, 2]); - }); - - it('should reinit "imagesInfo" if it was empty', () => { - const testResult = mkTestResult_({assertViewResults: [1], imagesInfo: []}); - - mkHermioneTestResultAdapter(testResult).getImagesInfo(); - - assert.lengthOf(testResult.imagesInfo, 1); - }); - - it('should return diffClusters', () => { - const testResult = mkTestResult_({ - assertViewResults: [{diffClusters: [{left: 0, top: 0, right: 1, bottom: 1}]}], - imagesInfo: [] - }); - - const [{diffClusters}] = mkHermioneTestResultAdapter(testResult).getImagesInfo(); - - assert.deepEqual(diffClusters, [{left: 0, top: 0, right: 1, bottom: 1}]); - }); - - it('should return saved images', async () => { - const testResult = mkTestResult_({ - assertViewResults: [mkErrStub()], - imagesInfo: [] - }); - - const imagesSaver = {saveImg: sandbox.stub()}; - imagesSaver.saveImg.withArgs( - 'ref/path', - {destPath: 'some/ref.png', reportDir: 'some/rep'} - ).returns('saved/ref.png'); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - imagesSaver - } - }); - const workers = {saveDiffTo: sandbox.stub()}; - - await hermioneTestAdapter.saveTestImages('some/rep', workers); - - const {expectedImg} = hermioneTestAdapter.getImagesFor(SUCCESS, 'plain'); - assert.equal(expectedImg.path, 'saved/ref.png'); - }); - - it('should return dest image path by default', async () => { - const testResult = mkTestResult_({ - assertViewResults: [mkErrStub()], - imagesInfo: [] - }); - - const imagesSaver = {saveImg: sandbox.stub()}; - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - imagesSaver - } - }); - const workers = {saveDiffTo: sandbox.stub()}; - - await hermioneTestAdapter.saveTestImages('some/rep', workers); - - const {expectedImg} = hermioneTestAdapter.getImagesFor(SUCCESS, 'plain'); - assert.equal(expectedImg.path, 'some/ref.png'); - }); - - it('should return ref image path after update image for NoRefImageError', async () => { - const testResult = mkTestResult_({ - assertViewResults: [mkErrStub(NoRefImageError)], - imagesInfo: [] - }); - - const imagesSaver = {saveImg: sandbox.stub()}; - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, { - htmlReporter: { - imagesSaver - } - }); - const workers = {saveDiffTo: sandbox.stub()}; - - await hermioneTestAdapter.saveTestImages('some/rep', workers); - - const {expectedImg} = hermioneTestAdapter.getImagesFor(UPDATED, 'plain'); - assert.equal(expectedImg.path, 'some/ref.png'); - }); - - describe('expected path', () => { - const mkLastImageInfo_ = (opts = {}) => { - const {stateName, expectedImgPath} = _.defaults(opts, { - stateName: 'plain', - expectedImgPath: 'default/expected/img/path.png' - }); - - return [{ - stateName, - expectedImg: { - path: expectedImgPath - } - }]; - }; - - it('should be pulled from the database if exists', async () => { - sqliteAdapter.query.withArgs({ - select: 'imagesInfo', - where: 'suitePath = ? AND name = ?', - orderBy: 'timestamp', - orderDescending: true - }).returns({imagesInfo: JSON.stringify(mkLastImageInfo_())}); - - const testResult = mkTestResult_({ - fullTitle: () => 'some-title', - assertViewResults: [mkErrStub()], - imagesInfo: [] - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, {status: FAIL}); - - hermioneTestAdapter.getImagesFor(FAIL, 'plain'); - - assert.notCalled(utils.getReferencePath); - }); - - it('should be generated if does not exist in sqlite', async () => { - sqliteAdapter.query.returns(null); - const testResult = mkTestResult_({ - fullTitle: () => 'some-title', - assertViewResults: [mkErrStub()], - imagesInfo: [] - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, {status: FAIL}); - - hermioneTestAdapter.getImagesFor(FAIL, 'plain'); - - assert.calledOnce(utils.getReferencePath); - }); - - it('should be generated on update', async () => { - sqliteAdapter.query.returns({imagesInfo: JSON.stringify(mkLastImageInfo_())}); - const testResult = mkTestResult_({ - fullTitle: () => 'some-title', - assertViewResults: [mkErrStub()], - imagesInfo: [] - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, {status: FAIL}); - - hermioneTestAdapter.getImagesFor(UPDATED, 'plain'); - - assert.calledOnce(utils.getReferencePath); - }); - - it('should be queried from the database for each browser', async () => { - const chromeTestResult = mkTestResult_({browserId: 'chrome'}); - const firefoxTestResult = mkTestResult_({browserId: 'firefox'}); - - mkHermioneTestResultAdapter(chromeTestResult, {status: FAIL}).getImagesFor(FAIL, 'plain'); - mkHermioneTestResultAdapter(firefoxTestResult, {status: FAIL}).getImagesFor(FAIL, 'plain'); - - assert.calledTwice(sqliteAdapter.query); - assert.calledWith(sqliteAdapter.query.firstCall, sinon.match.any, sinon.match.any, 'chrome'); - assert.calledWith(sqliteAdapter.query.secondCall, sinon.match.any, sinon.match.any, 'firefox'); - }); - - it('should be queried from the database once per state', async () => { - sqliteAdapter.query.returns({imagesInfo: JSON.stringify(mkLastImageInfo_())}); - const testResult = mkTestResult_({ - fullTitle: () => 'some-title', - assertViewResults: [mkErrStub()], - imagesInfo: [] - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, {status: FAIL}); - - hermioneTestAdapter.getImagesFor(FAIL, 'plain'); - hermioneTestAdapter.getImagesFor(FAIL, 'plain'); - - assert.calledOnce(sqliteAdapter.query); - }); - }); - }); - - describe('timestamp', () => { - it('should return corresponding timestamp of the test result', () => { - const testResult = mkTestResult_({ - timestamp: 100500 - }); - const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); - - assert.strictEqual(hermioneTestAdapter.timestamp, 100500); - }); - }); -}); diff --git a/test/unit/lib/test-adapter.ts b/test/unit/lib/test-adapter.ts new file mode 100644 index 000000000..ba2d5c3b0 --- /dev/null +++ b/test/unit/lib/test-adapter.ts @@ -0,0 +1,382 @@ +import _ from 'lodash'; +import * as fsOriginal from 'fs-extra'; +import proxyquire from 'proxyquire'; +import sinon from 'sinon'; +import tmpOriginal from 'tmp'; + +import {SKIPPED, TestStatus} from 'lib/constants/test-statuses'; +import {ERROR_DETAILS_PATH} from 'lib/constants/paths'; +import {TestAdapter} from 'lib/test-adapter'; +import {ErrorDetails, Suite, TestResult} from 'lib/types'; +import {ImagesInfoFormatter} from 'lib/image-handler'; +import * as originalUtils from 'lib/server-utils'; +import {ErrorName} from 'lib/errors'; + +describe('hermione test adapter', () => { + const sandbox = sinon.sandbox.create(); + + let HermioneTestResultAdapter: typeof TestAdapter; + let getCommandsHistory: sinon.SinonStub; + let getSuitePath: sinon.SinonStub; + let utils: sinon.SinonStubbedInstance; + let fs: sinon.SinonStubbedInstance; + let tmp: typeof tmpOriginal; + + class ImageDiffError extends Error { + name = ErrorName.IMAGE_DIFF; + } + + const mkImagesInfoFormatter = (): sinon.SinonStubbedInstance => { + return { + getRefImg: sinon.stub(), + getCurrImg: sinon.stub(), + getScreenshot: sinon.stub() + } as sinon.SinonStubbedInstance; + }; + + const mkHermioneTestResultAdapter = ( + testResult: TestResult, + {status = TestStatus.SUCCESS, imagesInfoFormatter = mkImagesInfoFormatter()}: {status?: TestStatus, imagesInfoFormatter?: ImagesInfoFormatter} = {} + ): TestAdapter => { + return new HermioneTestResultAdapter(testResult, {status, imagesInfoFormatter}); + }; + + const mkTestResult_ = (result: Partial): TestResult => _.defaults(result, { + id: 'some-id', + fullTitle: () => 'default-title' + }) as TestResult; + + beforeEach(() => { + tmp = {tmpdir: 'default/dir'} as typeof tmpOriginal; + fs = sinon.stub(_.clone(fsOriginal)); + getSuitePath = sandbox.stub(); + getCommandsHistory = sandbox.stub(); + + const originalUtils = proxyquire('lib/server-utils', { + 'fs-extra': fs + }); + utils = _.clone(originalUtils); + + HermioneTestResultAdapter = proxyquire('lib/test-adapter', { + tmp, + 'fs-extra': fs, + './plugin-utils': {getSuitePath}, + './history-utils': {getCommandsHistory}, + './server-utils': utils + }).TestAdapter; + sandbox.stub(utils, 'getCurrentPath').returns(''); + sandbox.stub(utils, 'getDiffPath').returns(''); + sandbox.stub(utils, 'getReferencePath').returns(''); + + fs.readFile.resolves(Buffer.from('')); + fs.writeFile.resolves(); + fs.copy.resolves(); + }); + + afterEach(() => sandbox.restore()); + + it('should return suite attempt', () => { + const firstTestResult = mkTestResult_({fullTitle: () => 'some-title'}); + const secondTestResult = mkTestResult_({fullTitle: () => 'other-title'}); + + mkHermioneTestResultAdapter(firstTestResult); + + assert.equal(mkHermioneTestResultAdapter(firstTestResult).attempt, 1); + assert.equal(mkHermioneTestResultAdapter(secondTestResult).attempt, 0); + }); + + it('should not increment attempt for skipped tests', () => { + const testResult = mkTestResult_({fullTitle: () => 'some-title'}); + + mkHermioneTestResultAdapter(testResult, {status: SKIPPED}); + const result = mkHermioneTestResultAdapter(testResult, {status: SKIPPED}); + + assert.equal(result.attempt, 0); + }); + + it('should return test error with "message", "stack" and "stateName"', () => { + getCommandsHistory.withArgs([{name: 'foo'}], ['foo']).returns(['some-history']); + const testResult = mkTestResult_({ + file: 'bar', + history: [{name: 'foo'}], + err: { + message: 'some-message', + stack: 'some-stack', + stateName: 'some-test', + foo: 'bar' + } as any + }); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.error, { + message: 'some-message', + stack: 'some-stack', + stateName: 'some-test' + }); + }); + + it('should return test history', () => { + getCommandsHistory.withArgs([{name: 'foo'}]).returns(['some-history']); + const testResult = mkTestResult_({ + file: 'bar', + history: [{name: 'foo'}], + err: { + message: 'some-message', + stack: 'some-stack', + stateName: 'some-test', + foo: 'bar' + } as any + }); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.history, ['some-history']); + }); + + it('should return test state', () => { + const testResult = mkTestResult_({title: 'some-test'}); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.state, {name: 'some-test'}); + }); + + it('should return assert view results', () => { + const testResult = mkTestResult_({assertViewResults: [1 as any]}); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.assertViewResults, [1 as any]); + }); + + describe('error details', () => { + let getDetailsFileName: sinon.SinonStub; + + beforeEach(() => { + getDetailsFileName = sandbox.stub(utils, 'getDetailsFileName').returns(''); + }); + + it('should be returned for test if they are available', () => { + const testResult = mkTestResult_({ + err: { + details: {title: 'some-title', data: {foo: 'bar'}} + } as any + }); + getDetailsFileName.returns('md5-bro-n-time'); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.errorDetails, { + title: 'some-title', + data: {foo: 'bar'}, + filePath: `${ERROR_DETAILS_PATH}/md5-bro-n-time` + }); + }); + + it('should have "error details" title if no title is given', () => { + const testResult = mkTestResult_({err: {details: {}} as any}); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.propertyVal(hermioneTestAdapter.errorDetails, 'title', 'error details'); + }); + + it('should be memoized', () => { + const testResult = mkTestResult_({ + err: { + details: {title: 'some-title', data: {foo: 'bar'}} + } as any + }); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + const firstErrDetails = hermioneTestAdapter.errorDetails; + const secondErrDetails = hermioneTestAdapter.errorDetails; + + assert.calledOnce(getDetailsFileName); + assert.deepEqual(firstErrDetails, secondErrDetails); + }); + + it('should be returned as null if absent', () => { + const testResult = mkTestResult_({err: {}} as any); + + const {errorDetails} = mkHermioneTestResultAdapter(testResult); + + assert.isNull(errorDetails); + }); + + it('should use test id, browser-id and attempt for filepath composing', () => { + const testResult = mkTestResult_({ + id: 'abcdef', + browserId: 'bro', + err: { + details: {data: {foo: 'bar'}} + } as any + }); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + // we need to get errorDetails to trigger getDetailsFileName to be called + hermioneTestAdapter.errorDetails; + + assert.calledWith(getDetailsFileName, 'abcdef', 'bro', hermioneTestAdapter.attempt); + }); + }); + + describe('saveErrorDetails', () => { + beforeEach(() => { + sandbox.stub(utils, 'makeDirFor').resolves(); + sandbox.stub(utils, 'getDetailsFileName').returns('md5-bro-n-time'); + }); + + it('should do nothing if no error details are available', async () => { + const hermioneTestAdapter = mkHermioneTestResultAdapter(mkTestResult_({err: {} as any})); + + await hermioneTestAdapter.saveErrorDetails(''); + + assert.notCalled(fs.writeFile); + }); + + it('should save error details to correct path', async () => { + const testResult = mkTestResult_({err: { + details: {title: 'some-title', data: {}} + } as any}); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + const {filePath} = hermioneTestAdapter.errorDetails as ErrorDetails; + + await hermioneTestAdapter.saveErrorDetails('report-path'); + + assert.calledWithMatch(fs.writeFile, `report-path/${filePath}`, sinon.match.any); + }); + + it('should create directory for error details', async () => { + const testResult = mkTestResult_({err: {details: {data: {}}} as any}); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + await hermioneTestAdapter.saveErrorDetails('report-path'); + + assert.calledOnceWith(utils.makeDirFor, sinon.match(`report-path/${ERROR_DETAILS_PATH}`)); + }); + + it('should save error details', async () => { + const data = {foo: 'bar'}; + const testResult = mkTestResult_({err: {details: {data}} as any}); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + await hermioneTestAdapter.saveErrorDetails(''); + + assert.calledWith(fs.writeFile, sinon.match.any, JSON.stringify(data, null, 2)); + }); + }); + + describe('hasDiff', () => { + it('should return true if test has image diff errors', () => { + const testResult = mkTestResult_({assertViewResults: [new ImageDiffError() as any]}); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.isTrue(hermioneTestAdapter.hasDiff()); + }); + + it('should return false if test has not image diff errors', () => { + const testResult = mkTestResult_({assertViewResults: [new Error() as any]}); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.isFalse(hermioneTestAdapter.hasDiff()); + }); + }); + + it('should return image dir', () => { + const testResult = mkTestResult_({id: 'some-id'}); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.imageDir, 'some-id'); + }); + + it('should return description', () => { + const testResult = mkTestResult_({description: 'some-description'}); + + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.deepEqual(hermioneTestAdapter.description, 'some-description'); + }); + + ([ + {field: 'refImg', method: 'getRefImg'}, + {field: 'currImg', method: 'getCurrImg'} + ] as const).forEach(({field, method}) => { + describe(`${method}`, () => { + it(`should use imagesInfoFormatter to get ${field} from test result`, () => { + const testResult = mkTestResult_({assertViewResults: [ + {[field]: 'some-value', stateName: 'plain'} as any + ]}); + const imagesInfoFormatter = mkImagesInfoFormatter(); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, {imagesInfoFormatter}); + + hermioneTestAdapter[method]('plain'); + + assert.calledOnceWith(imagesInfoFormatter[method], testResult.assertViewResults, 'plain'); + }); + }); + }); + + describe('getErrImg', () => { + it('should return error screenshot from test result', () => { + const testResult = mkTestResult_({err: {screenshot: 'some-value'} as any}); + + const imagesInfoFormatter = mkImagesInfoFormatter(); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult, {imagesInfoFormatter}); + + hermioneTestAdapter.getErrImg(); + + assert.calledOnceWith(imagesInfoFormatter.getScreenshot, testResult); + }); + }); + + describe('prepareTestResult', () => { + it('should return correct "name" field', () => { + const testResult = mkTestResult_({ + title: 'some-title' + }); + + const result = mkHermioneTestResultAdapter(testResult).prepareTestResult(); + + assert.propertyVal(result, 'name', 'some-title'); + }); + + it('should return correct "suitePath" field', () => { + const parentSuite = {parent: {root: true}, title: 'root-title'} as Suite; + const testResult = mkTestResult_({ + parent: parentSuite, + title: 'some-title' + }); + getSuitePath.returns(['root-title', 'some-title']); + + const result = mkHermioneTestResultAdapter(testResult).prepareTestResult(); + + assert.deepEqual(result.suitePath, ['root-title', 'some-title']); + }); + + it('should return "browserId" field as is', () => { + const testResult = mkTestResult_({ + browserId: 'bro' + }); + + const result = mkHermioneTestResultAdapter(testResult).prepareTestResult(); + + assert.propertyVal(result, 'browserId', 'bro'); + }); + }); + + describe('timestamp', () => { + it('should return corresponding timestamp of the test result', () => { + const testResult = mkTestResult_({ + timestamp: 100500 + }); + const hermioneTestAdapter = mkHermioneTestResultAdapter(testResult); + + assert.strictEqual(hermioneTestAdapter.timestamp, 100500); + }); + }); +});