From 9c9f8cbfa8271182e97451715ff345ecbadd74d7 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Wed, 20 Sep 2023 12:57:10 +0300 Subject: [PATCH] fix: fix incorrect behavior for diff and no ref errors --- lib/common-utils.ts | 28 +- lib/errors/index.ts | 3 +- lib/image-handler.ts | 2 +- lib/static/components/retry-switcher/item.jsx | 7 +- lib/static/components/section/body/tabs.jsx | 1 + lib/static/components/state/state-error.jsx | 4 +- lib/test-adapter/playwright.ts | 240 ++++++++++++++++++ lib/types.ts | 1 + .../hermione/failed-describe.hermione.js | 6 + .../screens/7357338/chrome/header.png | Bin 0 -> 176 bytes .../header-chromium-darwin.png | Bin 0 -> 3775 bytes 11 files changed, 279 insertions(+), 13 deletions(-) create mode 100644 lib/test-adapter/playwright.ts create mode 100644 test/func/fixtures/hermione/screens/7357338/chrome/header.png create mode 100644 test/func/fixtures/playwright/tests/failed-describe.spec.ts-snapshots/header-chromium-darwin.png diff --git a/lib/common-utils.ts b/lib/common-utils.ts index fdb56a771..b1dd92896 100644 --- a/lib/common-utils.ts +++ b/lib/common-utils.ts @@ -5,7 +5,7 @@ import axios, {AxiosRequestConfig} from 'axios'; import {SUCCESS, FAIL, ERROR, SKIPPED, UPDATED, IDLE, RUNNING, QUEUED, TestStatus} from './constants'; import {UNCHECKED, INDETERMINATE, CHECKED} from './constants/checked-statuses'; -import {AssertViewResult, TestError} from './types'; +import {AssertViewResult, ImageInfoFull, TestError, ImageInfoError} from './types'; import {ErrorName, ImageDiffError, NoRefImageError} from './errors'; export const getShortMD5 = (str: string): string => { return crypto.createHash('md5').update(str, 'ascii').digest('hex').substr(0, 7); @@ -88,20 +88,38 @@ export const mkTestId = (fullTitle: string, browserId: string): string => { return fullTitle + '.' + browserId; }; -export const isImageDiffError = (assertResult: AssertViewResult): assertResult is ImageDiffError => { - return (assertResult as ImageDiffError).name === ErrorName.IMAGE_DIFF; +export const isAssertViewError = (error?: unknown): boolean => { + return (error as {name?: string})?.name === ErrorName.ASSERT_VIEW; +}; + +export const isImageDiffError = (error?: unknown): error is ImageDiffError => { + return (error as {name?: string})?.name === ErrorName.IMAGE_DIFF; }; export const isNoRefImageError = (assertResult: AssertViewResult): assertResult is NoRefImageError => { return (assertResult as NoRefImageError).name === ErrorName.NO_REF_IMAGE; }; -export const getError = (error?: TestError): undefined | Pick => { +export const hasNoRefImageErrors = ({assertViewResults = []}: {assertViewResults?: AssertViewResult[]}): boolean => { + return Boolean(assertViewResults.filter((assertViewResult: AssertViewResult) => isNoRefImageError(assertViewResult)).length); +}; + +const hasFailedImages = (result: {imagesInfo?: ImageInfoFull[]}): boolean => { + const {imagesInfo = []} = result; + + return imagesInfo.some((imageInfo: ImageInfoFull) => !isAssertViewError((imageInfo as ImageInfoError).error) && (isErroredStatus(imageInfo.status) || isFailStatus(imageInfo.status))); +}; + +export const hasResultFails = (testResult: {status: TestStatus, imagesInfo?: ImageInfoFull[]}): boolean => { + return hasFailedImages(testResult) || isErroredStatus(testResult.status) || isFailStatus(testResult.status); +}; + +export const getError = (error?: TestError): undefined | Pick => { if (!error) { return undefined; } - return pick(error, ['message', 'stack', 'stateName']); + return pick(error, ['name', 'message', 'stack', 'stateName']); }; export const hasDiff = (assertViewResults: AssertViewResult[]): boolean => { diff --git a/lib/errors/index.ts b/lib/errors/index.ts index bca3450b5..6e51602aa 100644 --- a/lib/errors/index.ts +++ b/lib/errors/index.ts @@ -4,7 +4,8 @@ import {ValueOf} from 'type-fest'; export const ErrorName = { IMAGE_DIFF: 'ImageDiffError', - NO_REF_IMAGE: 'NoRefImageError' + NO_REF_IMAGE: 'NoRefImageError', + ASSERT_VIEW: 'AssertViewError' } as const; export type ErrorName = ValueOf; export type ErrorNames = typeof ErrorName; diff --git a/lib/image-handler.ts b/lib/image-handler.ts index f6a011ac4..f146c1fd5 100644 --- a/lib/image-handler.ts +++ b/lib/image-handler.ts @@ -120,7 +120,7 @@ export class ImageHandler extends EventEmitter2 implements ImagesInfoFormatter { status = FAIL; } else if (isNoRefImageError(assertResult)) { status = ERROR; - error = _.pick(assertResult, ['message', 'stack']); + error = _.pick(assertResult, ['message', 'name', 'stack']); } else { status = SUCCESS; } diff --git a/lib/static/components/retry-switcher/item.jsx b/lib/static/components/retry-switcher/item.jsx index 080808b97..3c790e176 100644 --- a/lib/static/components/retry-switcher/item.jsx +++ b/lib/static/components/retry-switcher/item.jsx @@ -3,9 +3,8 @@ import PropTypes from 'prop-types'; import classNames from 'classnames'; import {connect} from 'react-redux'; import {get} from 'lodash'; -import {isNoRefImageError, isAssertViewError} from '../../modules/utils'; -import {ERROR} from '../../../constants/test-statuses'; -import {isFailStatus} from '../../../common-utils'; +import {ERROR} from '../../../constants'; +import {isAssertViewError, isFailStatus, isImageDiffError, isNoRefImageError} from '../../../common-utils'; class RetrySwitcherItem extends Component { static propTypes = { @@ -51,5 +50,5 @@ export default connect( )(RetrySwitcherItem); function hasScreenAndAssertErrors(status, error) { - return isFailStatus(status) && error && !isNoRefImageError(error) && !isAssertViewError(error); + return isFailStatus(status) && error && !isNoRefImageError(error) && !isImageDiffError(error) && !isAssertViewError(error); } diff --git a/lib/static/components/section/body/tabs.jsx b/lib/static/components/section/body/tabs.jsx index 0c62b740e..b3a8e2593 100644 --- a/lib/static/components/section/body/tabs.jsx +++ b/lib/static/components/section/body/tabs.jsx @@ -42,6 +42,7 @@ export default class Tabs extends Component { ? null : this._drawTab({key: errorTabId}); } + console.log(result.imageIds); const tabs = result.imageIds.map((imageId) => this._drawTab({key: imageId, imageId})); diff --git a/lib/static/components/state/state-error.jsx b/lib/static/components/state/state-error.jsx index a04a977cb..226541695 100644 --- a/lib/static/components/state/state-error.jsx +++ b/lib/static/components/state/state-error.jsx @@ -8,10 +8,10 @@ import {isEmpty, map, isFunction} from 'lodash'; import ReactHtmlParser from 'react-html-parser'; import * as actions from '../../modules/actions'; import ResizedScreenshot from './screenshot/resized'; -import {isNoRefImageError, isAssertViewError} from '../../modules/utils'; import ErrorDetails from './error-details'; import Details from '../details'; import {ERROR_TITLE_TEXT_LENGTH} from '../../../constants/errors'; +import {isAssertViewError, isImageDiffError, isNoRefImageError} from '../../../common-utils'; class StateError extends Component { static propTypes = { @@ -95,7 +95,7 @@ class StateError extends Component { } _shouldDrawErrorInfo(error) { - return !isAssertViewError(error); + return !isImageDiffError(error) && !isAssertViewError(error); } render() { diff --git a/lib/test-adapter/playwright.ts b/lib/test-adapter/playwright.ts new file mode 100644 index 000000000..2f097f898 --- /dev/null +++ b/lib/test-adapter/playwright.ts @@ -0,0 +1,240 @@ +import {TestCase as PlaywrightTestCase, TestResult as PlaywrightTestResult} from '@playwright/test/reporter'; +import sizeOf from 'image-size'; +import {ReporterTestResult} from './index'; +import {TestStatus} from '../constants'; +import { + AssertViewResult, + ErrorDetails, + ImageBase64, + ImageData, + ImageInfoFull, + ImageSize, + TestError +} from '../types'; +import path from 'path'; +import * as utils from '../server-utils'; +import {testsAttempts} from './cache/playwright'; +import _ from 'lodash'; +import {getShortMD5, mkTestId} from '../common-utils'; +import {ImagesInfoFormatter} from '../image-handler'; +import stripAnsi from 'strip-ansi'; +import {ErrorName} from '../errors'; + +export type PlaywrightAttachment = PlaywrightTestResult['attachments'][number]; + +export enum PwtTestStatus { + PASSED = 'passed', + FAILED = 'failed', + TIMED_OUT = 'timedOut', + INTERRUPTED = 'interrupted', + SKIPPED = 'skipped', +} + +export enum ImageTitleEnding { + Expected = '-expected.png', + Actual = '-actual.png', + Diff = '-diff.png', + Previous = '-previous.png' +} + +const ANY_IMAGE_ENDING_REGEXP = new RegExp(Object.values(ImageTitleEnding).join('|')); + +const getStatus = (result: PlaywrightTestResult): TestStatus => { + if (result.status === PwtTestStatus.PASSED) { + return TestStatus.SUCCESS; + } + + if ( + [PwtTestStatus.FAILED, PwtTestStatus.TIMED_OUT, PwtTestStatus.INTERRUPTED].includes( + result.status as PwtTestStatus + ) + ) { + return TestStatus.FAIL; + } + + return TestStatus.SKIPPED; +}; + +const extractErrorMessage = (result: PlaywrightTestResult): string => { + if (_.isEmpty(result.errors)) { + return ''; + } + + if (result.errors.length === 1) { + return stripAnsi(result.errors[0].message || '').split('\n')[0]; + } + + return JSON.stringify(result.errors.map(err => stripAnsi(err.message || '').split('\n')[0])); +}; + +const extractErrorStack = (result: PlaywrightTestResult): string => { + if (_.isEmpty(result.errors)) { + return ''; + } + + if (result.errors.length === 1) { + return stripAnsi(result.errors[0].stack || ''); + } + + return JSON.stringify(result.errors.map(err => stripAnsi(err.stack || ''))); +}; + +const getImageData = (attachment: PlaywrightAttachment | undefined): ImageData | null => { + if (!attachment) { + return null; + } + + return { + path: attachment.path as string, + size: _.pick(sizeOf(attachment.path as string), ['height', 'width']) as ImageSize + }; +}; + +export interface PlaywrightTestAdapterOptions { + imagesInfoFormatter: ImagesInfoFormatter; +} + +export class PlaywrightTestAdapter implements ReporterTestResult { + private readonly _testCase: PlaywrightTestCase; + private readonly _testResult: PlaywrightTestResult; + private _attempt: number; + private _imagesInfoFormatter: ImagesInfoFormatter; + + constructor(testCase: PlaywrightTestCase, testResult: PlaywrightTestResult, {imagesInfoFormatter}: PlaywrightTestAdapterOptions) { + this._testCase = testCase; + this._testResult = testResult; + this._imagesInfoFormatter = imagesInfoFormatter; + + const testId = mkTestId(this.fullName, this.browserId); + if (utils.shouldUpdateAttempt(this.status)) { + testsAttempts.set(testId, _.isUndefined(testsAttempts.get(testId)) ? 0 : testsAttempts.get(testId) as number + 1); + } + + this._attempt = testsAttempts.get(testId) || 0; + } + + get assertViewResults(): AssertViewResult[] { + return Object.entries(this._attachmentsByState).map(([state, attachments]): AssertViewResult | null => { + const refImg = getImageData(attachments.find(a => a.path?.endsWith(ImageTitleEnding.Expected))); + const diffImg = getImageData(attachments.find(a => a.path?.endsWith(ImageTitleEnding.Diff))); + const currImg = getImageData(attachments.find(a => a.path?.endsWith(ImageTitleEnding.Actual))); + + if (this.error?.name === ErrorName.IMAGE_DIFF && refImg && diffImg && currImg) { + return { + name: ErrorName.IMAGE_DIFF, + stateName: state, + refImg, + diffImg, + currImg + }; + } else if (this.error?.name === ErrorName.NO_REF_IMAGE && currImg) { + return { + name: ErrorName.NO_REF_IMAGE, + message: this.error.message, + stack: this.error.stack, + stateName: state, + currImg + }; + } + + return null; + }).filter(Boolean) as AssertViewResult[]; + } + + get attempt(): number { + return this._attempt; + } + get browserId(): string { + return this._testCase.parent.project()?.name as string; + } + get description(): string | undefined { + return undefined; + } + get error(): TestError | undefined { + const message = extractErrorMessage(this._testResult); + if (message) { + const result: TestError = {message}; + + const stack = extractErrorStack(this._testResult); + if (stack) { + result.stack = stack; + } + + if (message.includes('snapshot doesn\'t exist') && message.includes('.png')) { + result.name = ErrorName.NO_REF_IMAGE; + } else if (message.includes('Screenshot comparison failed')) { + result.name = ErrorName.IMAGE_DIFF; + } + + return result; + } + return undefined; + } + get errorDetails(): ErrorDetails | null { + return null; + } + get file(): string { + return path.relative(process.cwd(), this._testCase.location.file); + } + get fullName(): string { + return this.testPath.join(' '); + } + get history(): string[] { + return this._testResult.steps.map(step => `${step.title} <- ${step.duration}ms\n`); + } + get id(): string { + return this.testPath.concat(this.browserId, this.attempt.toString()).join(' '); + } + get imageDir(): string { + return getShortMD5(this.fullName); + } + get imagesInfo(): ImageInfoFull[] | undefined { + return this._imagesInfoFormatter.getImagesInfo(this); + } + get isUpdated(): boolean { + return false; + } + get meta(): Record { + return Object.fromEntries(this._testCase.annotations.map(a => [a.type, a.description ?? ''])); + } + + get multipleTabs(): boolean { + return true; + } + + get screenshot(): ImageBase64 | undefined { + return undefined; + } + get sessionId(): string { + return this._testCase.annotations.find(a => a.type === 'surfwax.sessionId')?.description || ''; + } + + get skipReason(): string { + return this._testCase.annotations.find(a => a.type === 'skip')?.description || ''; + } + + get state(): { name: string } { + return {name: this._testCase.title}; + } + + get status(): TestStatus { + return getStatus(this._testResult); + } + + get testPath(): string[] { + // slicing because first entries are not actually test-name, but a file, etc. + return this._testCase.titlePath().slice(3); + } + get timestamp(): number | undefined { + return this._testResult.startTime.getTime(); + } + get url(): string { + return this._testCase.annotations.find(a => a.type === 'annotations.lastOpenedUrl')?.description || ''; + } + + private get _attachmentsByState(): Record { + const imageAttachments = this._testResult.attachments.filter(a => a.contentType === 'image/png'); + + return _.groupBy(imageAttachments, a => a.name.replace(ANY_IMAGE_ENDING_REGEXP, '')); + } +} diff --git a/lib/types.ts b/lib/types.ts index 5a00e04b2..e5bcf0d69 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -94,6 +94,7 @@ export type ImageInfo = export type AssertViewResult = AssertViewSuccess | ImageDiffError | NoRefImageError; export interface TestError { + name: string; message: string; stack?: string; stateName?: string; diff --git a/test/func/fixtures/hermione/failed-describe.hermione.js b/test/func/fixtures/hermione/failed-describe.hermione.js index 1dcd2aa1b..50f609c02 100644 --- a/test/func/fixtures/hermione/failed-describe.hermione.js +++ b/test/func/fixtures/hermione/failed-describe.hermione.js @@ -11,6 +11,12 @@ describe('failed describe', function() { await browser.assertView('header', 'header'); }); + it('test with diff', async ({browser}) => { + await browser.url(browser.options.baseUrl); + + await browser.assertView('header', 'header'); + }); + it('test with long error message', async () => { throw new Error(`long_error_message ${'0123456789'.repeat(20)}\n message content`); }); diff --git a/test/func/fixtures/hermione/screens/7357338/chrome/header.png b/test/func/fixtures/hermione/screens/7357338/chrome/header.png new file mode 100644 index 0000000000000000000000000000000000000000..5bc4221ca688962862c7b63a15bc321f05979a62 GIT binary patch literal 176 zcmeAS@N?(olHy`uVBq!ia0vp^5EamT(V6_?>+9$BQAurq+;zOG6gF+x@MHao6&`|dn^Fw(xHU{;0s}wR=jZF^aW7i5 z=%2NdK8+7{*}rN{c}XamvO1e W&P`V`b9{idF?hQAxvXNEDK_LiUj@`;uJ@62{Vy@?a9OCF`iHW6w}_MF}DM zI>r(j(^#@^1??+0NN(ZO}=w6W1s0`vlh1E@YKNaq3PaHl1+bV_BwLN`$Vu zY+gRTsOQhwevVafL~>rVr7uZ@ow=2lVHD}m{$es-SCJeRQH?~QE{3u4FO=047M`5` z990wZXE|%oHW31sn8kHx=e)Su(YnB^7Ghj@JpQ(U!IQec4SGmthC*bz9NJfxpt#T& z!cb~floF8C-p|tQUG(7z%0OfJ-UUk?1R1Jch5-_QBOMV;ZwzEz(d27!^ zPfr`pyy`Y2E^0)Kq5Q%HG`E?2VQ= zY`!jxACi@mBXH%4LH@I-sD7&a{V5e*g@9F>%7F4(*r0kily8K1n(bf#Yw#e*^Vrx| z_`-sM6`4lcvQyWtZH?bjJuIgJq3KM_%*_>CAO-E9B&DSTcoPy5ygl9=o|u3DWIy93 z#aq@Wk%Zac6Xj%H|2h2R%NO;IOuaX^?slX|aZA{S4z`Mji0GM^L`+TDK0_$L@7}%p zQr`dVFH%5}WkZl92P5Rj(W9ru#o2UpbiTB}$ zp~1AhMhQP}kaJ32zP2OWb7vj3`>XxZIZ@Hj4>s35J;if$anRu4ed!ioLx2BkZ{NP1 z@Ro(Kv(p#!kIc%Vd91-rF^l()0IWbWw#^YRZiN6zMBq<}RqE)w4ig?8p8K?2|1_mw z#B@uHnzy&)v6I~2UhPoTcdxpSebeD(g-aga+u2}*SX6l(C6md&@tEN^g$?}0cX*~- zf5Bp#n^ovCcQn-1pM#Fc5pi+s*ocU4>0Qo}ZIeymQg+%x@$UOkJXbY+eJex5!tRZH ze)i89RaI31fw=Vai;m@{y~1K*$YSe;VxHXgnYay7ur=DUd|0%qs*11L0~Y)EF@1-H zhK7dM@{}qQGjlHioY3=1EV#+L)$rR!X!E68U0t z(@zKE;^LA!%?gDMP-|>Ioyf=I<6gex?kGRx;dU@1ES7;0>ZiRE z)0XHp{h6h;vvc5V1)PT`@jh7bWv5Qwj_R_qGOOtm%*^owf@BG1whLE+T^T4cXXoQ9 zS}ijX)i*X4PP$~x`KCMXY_%DCC^b8&@k(k(TUr1POzh(Bz7V%*rsVXd``po^N2k21 zvIqpj*RNk2QaaP+j4i~HW)X4m@jmfE;~8av^o#CO??$P=J~ChWL0eyq!$Ho9h!iN3 zSP%AJPnQ7}o0B61rbP)YL%xWV)C#srRLt8STt=q#8{( z?9&2?QWrTR?VX(3u^htQ=wUjxg8mCF(R}d<34amodn;TA4PA?@>dg8xUcAU56q&z6 z`mh?8SfMr;`n4wLo=;5JT=SfG^nG@AenDFF{CP6yW>NnNJWAixG@n4G;^q8*W!0Py9;R~k@)lT^N&NJ zx6BpyRt*4FrpBrs-~(VE-a51!`YbOmA3u50uuV@-55Jx$Vm#HBD5B}#m#3Gqc2ik7 zHaQtf?#(>|ptJ#r8X7W5QwZdIFxUC%e*O}&fteZJf{Tl5`U$Jz?FuE@=5*Uy zU%o*i1?c6&m5!?^Te1Q>hRsEXx~;X{N~cpHf1LA0-K)tGc46~2&q_<;CcsFy`I)e_3v6&Nx-vR(kZaxQWruxb`6bh(6-V6AEBxasM(X+NL3D8-$a@~_fF#}4~D6?%k4G!*Uj?-CkScCl-4_s83 zsc;0Y4+vqqGTVU|tP5Df=M;O8>iL-X9xWLvHBO2DHwr2H8nO<^p(XL-|+xwA?; z)mlzduFlTaZ{DP@-&;4&b|0_xA8H6z;)R9}9?3w|Ef!v^r#<`w3-%edNX*Xq)`ssQ&zxQQy%Xu$fAK^4W@a)5eO*6Z?#)Z zYR_Zr{i_S4T7QFHrxAh+HIl%rlObr znwF0w)-uqvK3Dnz@cKrfItFo<4i8rMwdRDNpddLzWn_duc=wfsxhSV#2tWvoho@kk zLc{qt?5(Fd)(=_*7TLkjA*SVyF7trlWp*u4qx7c9#mR5Up3l?^)fFiCUgNyKZ z{bF+~^twG*VhZ1jTwUB9L0cX;+?y3~Uxqh5njeyqnkr>yj!V!w=(ulmv&y)+r6q98 z$7jCp{E(Y;cCAwvqJMu3MXI;#-~SSO#@X5VcH6aU*YN8=#cfxAbZeR}CiHd?;V1}G{aA(68F_U+~GGc&J>i=n_%o5NUm45@2t zYeS`X89;(`C)Gy*Y7v{XdwP0K38X*ehqSe~``Dd5bb$(x0>mJaI5Y`k?B(T^Rb6fN zH1c<5y{MVm?VO+-4#j4a)NgA_bv^xCB&)LW4hTJlYv2xECLQSj%wi6dx_R{lNC1%_ zN~ERnmVspI8{c|X7@5Wfi#5naVu#%ZR(xEQ(s>jn$w%wK`Yfk-mqw0

BLo0WRA@3Eul;t;_hSbefc9G5U|$Y;gsm!0>CHX(Dv8M4#&+c zm+>*`9cVZM$WQOR$?V5i5zm&EJo^jkh(&Lf6O)oeN~Sw2zOAqS zxUuoN%B@>d9~k-j(sO6fw~dVc%{Ryukd*uj*pk1$zq6ZLGr^_z+dB`ESPbsKfg)&j zY+D;K?T3FOzRYb7+>!Z{-}UeQ|1W0ZLQ8+jJsjGHnAs>s3-eJ$!5b+=Llvn~eADLf Fe*mcfIg