diff --git a/lib/common-utils.ts b/lib/common-utils.ts index fdb56a771..b1dd92896 100644 --- a/lib/common-utils.ts +++ b/lib/common-utils.ts @@ -5,7 +5,7 @@ import axios, {AxiosRequestConfig} from 'axios'; import {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED, TestStatus} from './constants'; import {UNCHECKED, INDETERMINATE, CHECKED} from './constants/checked-statuses'; -import {AssertViewResult, TestError} from './types'; +import {AssertViewResult, ImageInfoFull, TestError, ImageInfoError} from './types'; import {ErrorName, ImageDiffError, NoRefImageError} from './errors'; export const getShortMD5 = (str: string): string => { return crypto.createHash('md5').update(str, 'ascii').digest('hex').substr(0, 7); @@ -88,20 +88,38 @@ export const mkTestId = (fullTitle: string, browserId: string): string => { return fullTitle + '.' + browserId; }; -export const isImageDiffError = (assertResult: AssertViewResult): assertResult is ImageDiffError => { - return (assertResult as ImageDiffError).name === ErrorName.IMAGE_DIFF; +export const isAssertViewError = (error?: unknown): boolean => { + return (error as {name?: string})?.name === ErrorName.ASSERT_VIEW; +}; + +export const isImageDiffError = (error?: unknown): error is ImageDiffError => { + return (error as {name?: string})?.name === ErrorName.IMAGE_DIFF; }; export const isNoRefImageError = (assertResult: AssertViewResult): assertResult is NoRefImageError => { return (assertResult as NoRefImageError).name === ErrorName.NO_REF_IMAGE; }; -export const getError = (error?: TestError): undefined | Pick => { +export const hasNoRefImageErrors = ({assertViewResults = []}: {assertViewResults?: AssertViewResult[]}): boolean => { + return Boolean(assertViewResults.filter((assertViewResult: AssertViewResult) => isNoRefImageError(assertViewResult)).length); +}; + +const hasFailedImages = (result: {imagesInfo?: ImageInfoFull[]}): boolean => { + const {imagesInfo = []} = result; + + return imagesInfo.some((imageInfo: ImageInfoFull) => !isAssertViewError((imageInfo as ImageInfoError).error) && (isErroredStatus(imageInfo.status) || isFailStatus(imageInfo.status))); +}; + +export const hasResultFails = (testResult: {status: TestStatus, imagesInfo?: ImageInfoFull[]}): boolean => { + return hasFailedImages(testResult) || isErroredStatus(testResult.status) || isFailStatus(testResult.status); +}; + +export const getError = (error?: TestError): undefined | Pick => { if (!error) { return undefined; } - return pick(error, ['message', 'stack', 'stateName']); + return pick(error, ['name', 'message', 'stack', 'stateName']); }; export const hasDiff = (assertViewResults: AssertViewResult[]): boolean => { diff --git a/lib/errors/index.ts b/lib/errors/index.ts index bca3450b5..6e51602aa 100644 --- a/lib/errors/index.ts +++ b/lib/errors/index.ts @@ -4,7 +4,8 @@ import {ValueOf} from 'type-fest'; export const ErrorName = { IMAGE_DIFF: 'ImageDiffError', - NO_REF_IMAGE: 'NoRefImageError' + NO_REF_IMAGE: 'NoRefImageError', + ASSERT_VIEW: 'AssertViewError' } as const; export type ErrorName = ValueOf; export type ErrorNames = typeof ErrorName; diff --git a/lib/image-handler.ts b/lib/image-handler.ts index f6a011ac4..f146c1fd5 100644 --- a/lib/image-handler.ts +++ b/lib/image-handler.ts @@ -120,7 +120,7 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter { status = FAIL; } else if (isNoRefImageError(assertResult)) { status = ERROR; - error = _.pick(assertResult, ['message', 'stack']); + error = _.pick(assertResult, ['message', 'name', 'stack']); } else { status = SUCCESS; } diff --git a/lib/static/components/retry-switcher/item.jsx b/lib/static/components/retry-switcher/item.jsx index 080808b97..3c790e176 100644 --- a/lib/static/components/retry-switcher/item.jsx +++ b/lib/static/components/retry-switcher/item.jsx @@ -3,9 +3,8 @@ import PropTypes from 'prop-types'; import classNames from 'classnames'; import {connect} from 'react-redux'; import {get} from 'lodash'; -import {isNoRefImageError, isAssertViewError} from '../../modules/utils'; -import {ERROR} from '../../../constants/test-statuses'; -import {isFailStatus} from '../../../common-utils'; +import {ERROR} from '../../../constants'; +import {isAssertViewError, isFailStatus, isImageDiffError, isNoRefImageError} from '../../../common-utils'; class RetrySwitcherItem extends Component { static propTypes = { @@ -51,5 +50,5 @@ export default connect( )(RetrySwitcherItem); function hasScreenAndAssertErrors(status, error) { - return isFailStatus(status) && error && !isNoRefImageError(error) && !isAssertViewError(error); + return isFailStatus(status) && error && !isNoRefImageError(error) && !isImageDiffError(error) && !isAssertViewError(error); } diff --git a/lib/static/components/section/body/tabs.jsx b/lib/static/components/section/body/tabs.jsx index 0c62b740e..b3a8e2593 100644 --- a/lib/static/components/section/body/tabs.jsx +++ b/lib/static/components/section/body/tabs.jsx @@ -42,6 +42,7 @@ export default class Tabs extends Component { ? null : this._drawTab({key: errorTabId}); } + console.log(result.imageIds); const tabs = result.imageIds.map((imageId) => this._drawTab({key: imageId, imageId})); diff --git a/lib/static/components/state/state-error.jsx b/lib/static/components/state/state-error.jsx index a04a977cb..226541695 100644 --- a/lib/static/components/state/state-error.jsx +++ b/lib/static/components/state/state-error.jsx @@ -8,10 +8,10 @@ import {isEmpty, map, isFunction} from 'lodash'; import ReactHtmlParser from 'react-html-parser'; import * as actions from '../../modules/actions'; import ResizedScreenshot from './screenshot/resized'; -import {isNoRefImageError, isAssertViewError} from '../../modules/utils'; import ErrorDetails from './error-details'; import Details from '../details'; import {ERROR_TITLE_TEXT_LENGTH} from '../../../constants/errors'; +import {isAssertViewError, isImageDiffError, isNoRefImageError} from '../../../common-utils'; class StateError extends Component { static propTypes = { @@ -95,7 +95,7 @@ class StateError extends Component { } _shouldDrawErrorInfo(error) { - return !isAssertViewError(error); + return !isImageDiffError(error) && !isAssertViewError(error); } render() { diff --git a/lib/test-adapter/playwright.ts b/lib/test-adapter/playwright.ts new file mode 100644 index 000000000..2f097f898 --- /dev/null +++ b/lib/test-adapter/playwright.ts @@ -0,0 +1,240 @@ +import {TestCase as PlaywrightTestCase, TestResult as PlaywrightTestResult} from '@playwright/test/reporter'; +import sizeOf from 'image-size'; +import {ReporterTestResult} from './index'; +import {TestStatus} from '../constants'; +import { + AssertViewResult, + ErrorDetails, + ImageBase64, + ImageData, + ImageInfoFull, + ImageSize, + TestError +} from '../types'; +import path from 'path'; +import * as utils from '../server-utils'; +import {testsAttempts} from './cache/playwright'; +import _ from 'lodash'; +import {getShortMD5, mkTestId} from '../common-utils'; +import {ImagesInfoFormatter} from '../image-handler'; +import stripAnsi from 'strip-ansi'; +import {ErrorName} from '../errors'; + +export type PlaywrightAttachment = PlaywrightTestResult['attachments'][number]; + +export enum PwtTestStatus { + PASSED = 'passed', + FAILED = 'failed', + TIMED_OUT = 'timedOut', + INTERRUPTED = 'interrupted', + SKIPPED = 'skipped', +} + +export enum ImageTitleEnding { + Expected = '-expected.png', + Actual = '-actual.png', + Diff = '-diff.png', + Previous = '-previous.png' +} + +const ANY_IMAGE_ENDING_REGEXP = new RegExp(Object.values(ImageTitleEnding).join('|')); + +const getStatus = (result: PlaywrightTestResult): TestStatus => { + if (result.status === PwtTestStatus.PASSED) { + return TestStatus.SUCCESS; + } + + if ( + [PwtTestStatus.FAILED, PwtTestStatus.TIMED_OUT, PwtTestStatus.INTERRUPTED].includes( + result.status as PwtTestStatus + ) + ) { + return TestStatus.FAIL; + } + + return TestStatus.SKIPPED; +}; + +const extractErrorMessage = (result: PlaywrightTestResult): string => { + if (_.isEmpty(result.errors)) { + return ''; + } + + if (result.errors.length === 1) { + return stripAnsi(result.errors[0].message || '').split('\n')[0]; + } + + return JSON.stringify(result.errors.map(err => stripAnsi(err.message || '').split('\n')[0])); +}; + +const extractErrorStack = (result: PlaywrightTestResult): string => { + if (_.isEmpty(result.errors)) { + return ''; + } + + if (result.errors.length === 1) { + return stripAnsi(result.errors[0].stack || ''); + } + + return JSON.stringify(result.errors.map(err => stripAnsi(err.stack || ''))); +}; + +const getImageData = (attachment: PlaywrightAttachment | undefined): ImageData | null => { + if (!attachment) { + return null; + } + + return { + path: attachment.path as string, + size: _.pick(sizeOf(attachment.path as string), ['height', 'width']) as ImageSize + }; +}; + +export interface PlaywrightTestAdapterOptions { + imagesInfoFormatter: ImagesInfoFormatter; +} + +export class PlaywrightTestAdapter implements ReporterTestResult { + private readonly _testCase: PlaywrightTestCase; + private readonly _testResult: PlaywrightTestResult; + private _attempt: number; + private _imagesInfoFormatter: ImagesInfoFormatter; + + constructor(testCase: PlaywrightTestCase, testResult: PlaywrightTestResult, {imagesInfoFormatter}: PlaywrightTestAdapterOptions) { + this._testCase = testCase; + this._testResult = testResult; + this._imagesInfoFormatter = imagesInfoFormatter; + + const testId = mkTestId(this.fullName, this.browserId); + if (utils.shouldUpdateAttempt(this.status)) { + testsAttempts.set(testId, _.isUndefined(testsAttempts.get(testId)) ? 0 : testsAttempts.get(testId) as number + 1); + } + + this._attempt = testsAttempts.get(testId) || 0; + } + + get assertViewResults(): AssertViewResult[] { + return Object.entries(this._attachmentsByState).map(([state, attachments]): AssertViewResult | null => { + const refImg = getImageData(attachments.find(a => a.path?.endsWith(ImageTitleEnding.Expected))); + const diffImg = getImageData(attachments.find(a => a.path?.endsWith(ImageTitleEnding.Diff))); + const currImg = getImageData(attachments.find(a => a.path?.endsWith(ImageTitleEnding.Actual))); + + if (this.error?.name === ErrorName.IMAGE_DIFF && refImg && diffImg && currImg) { + return { + name: ErrorName.IMAGE_DIFF, + stateName: state, + refImg, + diffImg, + currImg + }; + } else if (this.error?.name === ErrorName.NO_REF_IMAGE && currImg) { + return { + name: ErrorName.NO_REF_IMAGE, + message: this.error.message, + stack: this.error.stack, + stateName: state, + currImg + }; + } + + return null; + }).filter(Boolean) as AssertViewResult[]; + } + + get attempt(): number { + return this._attempt; + } + get browserId(): string { + return this._testCase.parent.project()?.name as string; + } + get description(): string | undefined { + return undefined; + } + get error(): TestError | undefined { + const message = extractErrorMessage(this._testResult); + if (message) { + const result: TestError = {message}; + + const stack = extractErrorStack(this._testResult); + if (stack) { + result.stack = stack; + } + + if (message.includes('snapshot doesn\'t exist') && message.includes('.png')) { + result.name = ErrorName.NO_REF_IMAGE; + } else if (message.includes('Screenshot comparison failed')) { + result.name = ErrorName.IMAGE_DIFF; + } + + return result; + } + return undefined; + } + get errorDetails(): ErrorDetails | null { + return null; + } + get file(): string { + return path.relative(process.cwd(), this._testCase.location.file); + } + get fullName(): string { + return this.testPath.join(' '); + } + get history(): string[] { + return this._testResult.steps.map(step => `${step.title} <- ${step.duration}ms\n`); + } + get id(): string { + return this.testPath.concat(this.browserId, this.attempt.toString()).join(' '); + } + get imageDir(): string { + return getShortMD5(this.fullName); + } + get imagesInfo(): ImageInfoFull[] | undefined { + return this._imagesInfoFormatter.getImagesInfo(this); + } + get isUpdated(): boolean { + return false; + } + get meta(): Record { + return Object.fromEntries(this._testCase.annotations.map(a => [a.type, a.description ?? ''])); + } + + get multipleTabs(): boolean { + return true; + } + + get screenshot(): ImageBase64 | undefined { + return undefined; + } + get sessionId(): string { + return this._testCase.annotations.find(a => a.type === 'surfwax.sessionId')?.description || ''; + } + + get skipReason(): string { + return this._testCase.annotations.find(a => a.type === 'skip')?.description || ''; + } + + get state(): { name: string } { + return {name: this._testCase.title}; + } + + get status(): TestStatus { + return getStatus(this._testResult); + } + + get testPath(): string[] { + // slicing because first entries are not actually test-name, but a file, etc. + return this._testCase.titlePath().slice(3); + } + get timestamp(): number | undefined { + return this._testResult.startTime.getTime(); + } + get url(): string { + return this._testCase.annotations.find(a => a.type === 'annotations.lastOpenedUrl')?.description || ''; + } + + private get _attachmentsByState(): Record { + const imageAttachments = this._testResult.attachments.filter(a => a.contentType === 'image/png'); + + return _.groupBy(imageAttachments, a => a.name.replace(ANY_IMAGE_ENDING_REGEXP, '')); + } +} diff --git a/lib/types.ts b/lib/types.ts index 5a00e04b2..e5bcf0d69 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -94,6 +94,7 @@ export type ImageInfo = export type AssertViewResult = AssertViewSuccess | ImageDiffError | NoRefImageError; export interface TestError { + name: string; message: string; stack?: string; stateName?: string; diff --git a/test/func/fixtures/hermione/failed-describe.hermione.js b/test/func/fixtures/hermione/failed-describe.hermione.js index 1dcd2aa1b..50f609c02 100644 --- a/test/func/fixtures/hermione/failed-describe.hermione.js +++ b/test/func/fixtures/hermione/failed-describe.hermione.js @@ -11,6 +11,12 @@ describe('failed describe', function() { await browser.assertView('header', 'header'); }); + it('test with diff', async ({browser}) => { + await browser.url(browser.options.baseUrl); + + await browser.assertView('header', 'header'); + }); + it('test with long error message', async () => { throw new Error(`long_error_message ${'0123456789'.repeat(20)}\n message content`); }); diff --git a/test/func/fixtures/hermione/screens/7357338/chrome/header.png b/test/func/fixtures/hermione/screens/7357338/chrome/header.png new file mode 100644 index 000000000..5bc4221ca Binary files /dev/null and b/test/func/fixtures/hermione/screens/7357338/chrome/header.png differ diff --git a/test/func/fixtures/playwright/tests/failed-describe.spec.ts-snapshots/header-chromium-darwin.png b/test/func/fixtures/playwright/tests/failed-describe.spec.ts-snapshots/header-chromium-darwin.png new file mode 100644 index 000000000..67940ac7d Binary files /dev/null and b/test/func/fixtures/playwright/tests/failed-describe.spec.ts-snapshots/header-chromium-darwin.png differ