test: implement unit tests for pwt test-adapter and e2e pwt fixture

gemini-testing · Sep 26, 2023 · b092b80 · b092b80
1 parent efd396f
commit b092b80
Show file tree

Hide file tree

Showing 10 changed files with 351 additions and 5 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -11,3 +11,4 @@ hot
 /test/func/packages/*/plugin.js
 /hermione-report
 tmp
+**/playwright-report
diff --git a/.gitignore b/.gitignore
@@ -22,6 +22,7 @@ sqlite.db
 .nyc_output
 tmp
 
+**/playwright-report
 hermione-report
 test/func/**/report
 test/func/**/report-backup

diff --git a/test/func/fixtures/playwright/index.html b/test/func/fixtures/playwright/index.html
@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="en" dir="ltr">
+<head>
+    <meta charset="utf-8">
+    <title></title>
+</head>
+<style media="screen">
+.text {
+    width: 400px;
+    height: 200px;
+    border: 1px solid #000;
+    background: green;
+    margin: 20px;
+    color: beige;
+}
+
+.tag {
+    display: inline-block;
+    padding: 5px;
+    background: grey;
+    border-radius: 3px;
+    margin-right: 10px;
+    margin-bottom: 10px;
+}
+
+.tag:hover {
+    background: yellow;
+}
+
+.diff {
+    width: 200px;
+    height: 200px;
+    border: 1px solid red;
+}
+
+footer {
+    margin: 10px 0;
+}
+</style>
+<body>
+    <header>
+        <h1>Some header</h1>
+    </header>
+    <div class="content">
+        <div class="text">
+            Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </div>
+        <div class="nav">
+            <span class="tag">tag1</span>
+            <span class="tag">tag2</span>
+            <span class="tag">tag3</span>
+        </div>
+    </div>
+    <div class="diff"></div>
+    <footer>Copyright ©</footer>
+    <script type="text/javascript">
+        document.querySelector('.diff').innerHTML = Date.now();
+    </script>
+</body>
+</html>
diff --git a/test/func/fixtures/playwright/package.json b/test/func/fixtures/playwright/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "playwright-fixture-report",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "generate": "npx playwright test"
+  }
+}
diff --git a/test/func/fixtures/playwright/tests/success-describe.spec.ts b/test/func/fixtures/playwright/tests/success-describe.spec.ts
@@ -3,8 +3,6 @@ import {test, expect} from '@playwright/test';
 test.describe('success describe', () => {
     test('successfully passed test', async ({page, baseURL}) => {
         await page.goto(baseURL as string);
-
-        expect(true).toBe(true);
     });
 
     test('test with screenshot', async ({page, baseURL}) => {

diff --git a/test/func/packages/html-reporter-tester/package.json b/test/func/packages/html-reporter-tester/package.json
@@ -2,7 +2,10 @@
   "name": "html-reporter-tester",
   "version": "1.0.0",
   "description": "",
-  "main": "index.js",
+  "exports": {
+    ".": "./index.js",
+    "./playwright": "./playwright.js"
+  },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },

diff --git a/test/func/packages/html-reporter-tester/playwright.js b/test/func/packages/html-reporter-tester/playwright.js
@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('../../../../build/playwright');
diff --git a/test/func/tests/common/error-group.hermione.js b/test/func/tests/common/error-group.hermione.js
@@ -9,7 +9,7 @@ describe('Error grouping', function() {
         const groupedTestsContainer = await browser.$('.grouped-tests');
 
         const errorGroups = await browser.$$('.grouped-tests > div');
-        assert.equal(errorGroups.length, 3);
+        assert(errorGroups.length >= 2 && errorGroups.length <= 3);
 
         const longErrorMessageGroup = await groupedTestsContainer.$('span*=long_error_message').$('..');
 

diff --git a/test/func/tests/common/tests-details.hermione.js b/test/func/tests/common/tests-details.hermione.js
@@ -8,7 +8,7 @@ describe('Test details', function() {
 
         await erroredTestSection.$('.details__summary').click();
 
-        const fileMetaInfo = await erroredTestSection.$('div*=failed-describe.hermione.js').$('..');
+        const fileMetaInfo = await erroredTestSection.$('div*=failed-describe').$('..');
 
         await expect(fileMetaInfo).toBeDisplayed();
         await expect(await fileMetaInfo.$('span*=file')).toBeDisplayed();

diff --git a/test/unit/lib/test-adapter/playwright.ts b/test/unit/lib/test-adapter/playwright.ts
@@ -0,0 +1,272 @@
+import sinon from 'sinon';
+import _ from 'lodash';
+import proxyquire from 'proxyquire';
+import {TestCase, TestResult} from '@playwright/test/reporter';
+import {ImageTitleEnding, PlaywrightAttachment, PlaywrightTestAdapterOptions, PwtTestStatus} from 'lib/test-adapter/playwright';
+import {ErrorName, ImageDiffError, NoRefImageError} from 'lib/errors';
+import {TestStatus} from 'lib/constants';
+
+describe('PlaywrightTestAdapter', () => {
+    let sandbox: sinon.SinonSandbox;
+    let PlaywrightTestAdapter: typeof import('lib/test-adapter/playwright').PlaywrightTestAdapter;
+    let imageSizeStub: sinon.SinonStub;
+    let playwrightCache: typeof import('lib/test-adapter/cache/playwright');
+
+    const createAttachment = (path: string): PlaywrightAttachment => ({
+        contentType: 'image/png',
+        name: path,
+        path,
+        body: Buffer.from('dummy-data')
+    });
+
+    const mkTestCase = (overrides: Partial<TestCase> = {}): TestCase => _.defaults(overrides, {
+        parent: {project: sinon.stub().returns({name: 'some-browser'})},
+        titlePath: sinon.stub().returns(['root', 'suite', 'subsuite', 'describe', 'test']),
+        annotations: [],
+        location: {file: 'test-file-path', column: 0, line: 0}
+    } as any);
+    const mkTestResult = (overrides: Partial<TestResult> = {}): TestResult => _.defaults(overrides, {
+        status: 'failed',
+        attachments: [
+            createAttachment('state1' + ImageTitleEnding.Expected),
+            createAttachment('state1' + ImageTitleEnding.Diff),
+            createAttachment('state1' + ImageTitleEnding.Actual)
+        ],
+        errors: [{name: ErrorName.IMAGE_DIFF, message: 'Screenshot comparison failed', stack: ''}],
+        steps: []
+    } as any);
+
+    const mkAdapterOptions = (overrides: Partial<PlaywrightTestAdapterOptions> = {}): PlaywrightTestAdapterOptions => _.defaults(overrides, {
+        imagesInfoFormatter: sinon.stub()
+    } as any);
+
+    beforeEach(() => {
+        sandbox = sinon.createSandbox();
+
+        playwrightCache = {testsAttempts: new Map()};
+
+        imageSizeStub = sinon.stub().returns({height: 100, width: 200});
+
+        PlaywrightTestAdapter = proxyquire('lib/test-adapter/playwright', {
+            'image-size': imageSizeStub,
+            './cache/playwright': playwrightCache
+        }).PlaywrightTestAdapter;
+    });
+
+    afterEach(() => {
+        sandbox.restore();
+    });
+
+    describe('assertViewResults', () => {
+        it('should return an IMAGE_DIFF result when error is IMAGE_DIFF and all images are present', () => {
+            const testCaseStub = mkTestCase();
+            const testResultStub = mkTestResult();
+            const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions());
+
+            const results = adapter.assertViewResults as ImageDiffError[];
+
+            assert.lengthOf(results, 1);
+            assert.strictEqual(results[0].name, ErrorName.IMAGE_DIFF);
+            assert.strictEqual(results[0].stateName, 'state1');
+            assert.strictEqual(results[0].refImg?.path, 'state1' + ImageTitleEnding.Expected);
+            assert.strictEqual(results[0].diffImg?.path, 'state1' + ImageTitleEnding.Diff);
+            assert.strictEqual(results[0].currImg?.path, 'state1' + ImageTitleEnding.Actual);
+        });
+
+        it('should return a NO_REF_IMAGE result when error is NO_REF_IMAGE and only actual image is present', () => {
+            const testCaseStub = mkTestCase();
+            const testResultStub = mkTestResult({
+                attachments: [createAttachment('state1' + ImageTitleEnding.Actual)],
+                errors: [{name: ErrorName.NO_REF_IMAGE, message: 'snapshot doesn\'t exist: some.png.', stack: 'error-stack'}] as any
+            });
+            const adapter = new PlaywrightTestAdapter(testCaseStub, testResultStub, mkAdapterOptions());
+
+            const results = adapter.assertViewResults as NoRefImageError[];
+
+            assert.lengthOf(results, 1);
+            assert.strictEqual(results[0].name, ErrorName.NO_REF_IMAGE);
+            assert.strictEqual(results[0].stateName, 'state1');
+            assert.strictEqual(results[0].currImg?.path, 'state1' + ImageTitleEnding.Actual);
+        });
+    });
+
+    describe('attempt', () => {
+        it('should return suite attempt', () => {
+            // eslint-disable-next-line no-new
+            new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+            const adapter2 = new PlaywrightTestAdapter(mkTestCase({titlePath: sinon.stub().returns(['another-title'])}), mkTestResult(), mkAdapterOptions());
+            const adapter3 = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+
+            assert.equal(adapter3.attempt, 1);
+            assert.equal(adapter2.attempt, 0);
+        });
+
+        it('should not increment attempt for skipped tests', () => {
+            const testResult = mkTestResult({status: 'skipped'});
+
+            // eslint-disable-next-line no-new
+            new PlaywrightTestAdapter(mkTestCase(), testResult, mkAdapterOptions());
+            const adapter2 = new PlaywrightTestAdapter(mkTestCase(), testResult, mkAdapterOptions());
+
+            assert.equal(adapter2.attempt, 0);
+        });
+    });
+
+    describe('browserId', () => {
+        it('should return browserId', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+
+            assert.equal(adapter.browserId, 'some-browser');
+        });
+    });
+
+    describe('error', () => {
+        it('should return undefined if there are no errors', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors: []}), mkAdapterOptions());
+
+            const {error} = adapter;
+
+            assert.isUndefined(error);
+        });
+
+        it('should return an error with name NO_REF_IMAGE for snapshot missing errors', () => {
+            const errorMessage = 'A snapshot doesn\'t exist: image-name.png.';
+            const errors = [{message: errorMessage}];
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions());
+
+            const {error} = adapter;
+
+            assert.strictEqual(error?.name, ErrorName.NO_REF_IMAGE);
+            assert.strictEqual(error?.message, errorMessage);
+        });
+
+        it('should return an error with name IMAGE_DIFF for screenshot comparison failures', () => {
+            const errorMessage = 'Screenshot comparison failed';
+            const errors = [{message: errorMessage}];
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions());
+
+            const {error} = adapter;
+
+            assert.strictEqual(error?.name, ErrorName.IMAGE_DIFF);
+            assert.strictEqual(error?.message, errorMessage);
+        });
+
+        it('should include the error stack if present', () => {
+            const errorMessage = 'Some error occurred';
+            const errorStack = 'Error: Some error occurred at some-file.ts:10:15';
+            const errors = [{message: errorMessage, stack: errorStack}];
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions());
+
+            const {error} = adapter;
+
+            assert.strictEqual(error?.stack, errorStack);
+        });
+
+        it('should convert multiple errors to a single JSON string', () => {
+            const errors = [
+                {message: 'First error', stack: 'Error: First error at some-file.ts:5:10'},
+                {message: 'Second error', stack: 'Error: Second error at another-file.ts:15:20'}
+            ];
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({errors}), mkAdapterOptions());
+            const expectedMessage = JSON.stringify(errors.map(err => err.message));
+            const expectedStack = JSON.stringify(errors.map(err => err.stack));
+
+            const {error} = adapter;
+
+            assert.strictEqual(error?.message, expectedMessage);
+            assert.strictEqual(error?.stack, expectedStack);
+        });
+    });
+
+    describe('file', () => {
+        it('should return file path', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+
+            assert.strictEqual(adapter.file, 'test-file-path');
+        });
+    });
+
+    describe('fullName', () => {
+        it('should return fullName', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+
+            assert.strictEqual(adapter.fullName, 'describe test');
+        });
+    });
+
+    describe('history', () => {
+        it('should return an array of formatted step titles and durations', () => {
+            const steps = [
+                {title: 'Step1', duration: 100},
+                {title: 'Step2', duration: 200}
+            ];
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({steps} as any), mkAdapterOptions());
+            const expectedHistory = ['Step1 <- 100ms\n', 'Step2 <- 200ms\n'];
+
+            assert.deepEqual(adapter.history, expectedHistory);
+        });
+    });
+
+    describe('id', () => {
+        it('should return id', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+
+            assert.strictEqual(adapter.id, 'describe test some-browser 0');
+        });
+    });
+
+    describe('imageDir', () => {
+        it('should return imageDir', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+
+            assert.strictEqual(adapter.imageDir, '75bcb6c');
+        });
+    });
+
+    describe('imagesInfo', () => {
+        it('should call formatter', () => {
+            const getImagesInfoStub = sinon.stub();
+            const options = mkAdapterOptions({imagesInfoFormatter: {getImagesInfo: getImagesInfoStub}});
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), options);
+
+            adapter.imagesInfo;
+
+            assert.calledOnceWith(getImagesInfoStub, adapter);
+        });
+    });
+
+    describe('status', () => {
+        it('should return SUCCESS for PASSED PwtTestStatus', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.PASSED}), mkAdapterOptions());
+            assert.equal(adapter.status, TestStatus.SUCCESS);
+        });
+
+        it('should return FAIL for FAILED PwtTestStatus', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.FAILED}), mkAdapterOptions());
+            assert.equal(adapter.status, TestStatus.FAIL);
+        });
+
+        it('should return FAIL for TIMED_OUT PwtTestStatus', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.TIMED_OUT}), mkAdapterOptions());
+            assert.equal(adapter.status, TestStatus.FAIL);
+        });
+
+        it('should return FAIL for INTERRUPTED PwtTestStatus', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.INTERRUPTED}), mkAdapterOptions());
+            assert.equal(adapter.status, TestStatus.FAIL);
+        });
+
+        it('should return SKIPPED for any other PwtTestStatus', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult({status: PwtTestStatus.SKIPPED}), mkAdapterOptions());
+            assert.equal(adapter.status, TestStatus.SKIPPED);
+        });
+    });
+
+    describe('testPath', () => {
+        it('should return testPath', () => {
+            const adapter = new PlaywrightTestAdapter(mkTestCase(), mkTestResult(), mkAdapterOptions());
+
+            assert.deepEqual(adapter.testPath, ['describe', 'test']);
+        });
+    });
+});
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		'use strict';

		module.exports = require('../../../../build/playwright');