From 719da9f2fe2c3994e40c0a983419dc80618cb7d4 Mon Sep 17 00:00:00 2001 From: shadowusr Date: Thu, 21 Sep 2023 22:41:22 +0300 Subject: [PATCH] fix: add gui e2e tests --- package-lock.json | 25 +++ package.json | 3 +- test/func/tests/.hermione.conf.js | 9 +- test/func/tests/common-gui/index.hermione.js | 187 +++++++++++++++--- .../test-results-appearance.hermione.js | 1 + test/func/tests/package.json | 4 +- .../screens/197b2c2/chrome/retry-switcher.png | Bin 0 -> 491 bytes .../screens/87f8b7e/chrome/retry-switcher.png | Bin 0 -> 517 bytes .../screens/9b544b9/chrome/retry-switcher.png | Bin 0 -> 491 bytes .../screens/9e6270b/chrome/retry-switcher.png | Bin 0 -> 517 bytes .../screens/aec277d/chrome/retry-switcher.png | Bin 0 -> 517 bytes .../screens/c9299b2/chrome/retry-switcher.png | Bin 0 -> 491 bytes 12 files changed, 198 insertions(+), 31 deletions(-) create mode 100644 test/func/tests/screens/197b2c2/chrome/retry-switcher.png create mode 100644 test/func/tests/screens/87f8b7e/chrome/retry-switcher.png create mode 100644 test/func/tests/screens/9b544b9/chrome/retry-switcher.png create mode 100644 test/func/tests/screens/9e6270b/chrome/retry-switcher.png create mode 100644 test/func/tests/screens/aec277d/chrome/retry-switcher.png create mode 100644 test/func/tests/screens/c9299b2/chrome/retry-switcher.png diff --git a/package-lock.json b/package-lock.json index 8877ae8f0..c8552c69a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -134,6 +134,7 @@ "styled-components": "^3.4.10", "stylus": "^0.57.0", "stylus-loader": "^3.0.2", + "tree-kill": "^1.2.2", "ts-node": "^10.9.1", "type-fest": "^3.13.1", "typescript": "^5.0.4", @@ -16571,6 +16572,10 @@ } } }, + "node_modules/hermione-eye": { + "resolved": "test/func/fixtures/hermione-eye", + "link": true + }, "node_modules/hermione-fixture-report": { "resolved": "test/func/fixtures/hermione", "link": true @@ -16584,6 +16589,10 @@ "gemini-configparser": "^1.0.0" } }, + "node_modules/hermione-gui": { + "resolved": "test/func/fixtures/hermione-gui", + "link": true + }, "node_modules/hermione-test-repeater": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/hermione-test-repeater/-/hermione-test-repeater-0.0.8.tgz", @@ -32644,12 +32653,21 @@ } }, "test/func/fixtures/hermione": { + "name": "hermione-fixture-report", + "version": "0.0.0" + }, + "test/func/fixtures/hermione-eye": { + "version": "0.0.0" + }, + "test/func/fixtures/hermione-gui": { "version": "0.0.0" }, "test/func/fixtures/playwright": { + "name": "playwright-fixture-report", "version": "0.0.0" }, "test/func/fixtures/plugins": { + "name": "plugins-fixture-report", "version": "0.0.0" }, "test/func/packages/basic": { @@ -32677,6 +32695,7 @@ "version": "0.1.0" }, "test/func/tests": { + "name": "html-reporter-e2e-hermione-tests", "version": "0.0.0" } }, @@ -46452,6 +46471,9 @@ } } }, + "hermione-eye": { + "version": "file:test/func/fixtures/hermione-eye" + }, "hermione-fixture-report": { "version": "file:test/func/fixtures/hermione" }, @@ -46464,6 +46486,9 @@ "gemini-configparser": "^1.0.0" } }, + "hermione-gui": { + "version": "file:test/func/fixtures/hermione-gui" + }, "hermione-test-repeater": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/hermione-test-repeater/-/hermione-test-repeater-0.0.8.tgz", diff --git a/package.json b/package.json index 60f2a1cef..b090d7bb2 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "e2e:build-browsers": "docker build -f test/func/docker/Dockerfile -t html-reporter-browsers:0.0.1 --network host test/func/docker", "e2e:build-packages": "npm run --workspace=test/func/packages --if-present build", "e2e:generate-fixtures": "npm run --workspace=test/func/fixtures generate", - "e2e:launch-browsers": "docker run -it --rm --network=host --add-host=host.docker.internal:host-gateway html-reporter-browsers:0.0.1", + "e2e:launch-browsers": "docker run -it --rm --network=host --add-host=host.docker.internal:127.0.0.1 html-reporter-browsers:0.0.1", "e2e:test": "npm run --workspace=test/func/tests test", "e2e": "npm run e2e:build-packages && npm run e2e:generate-fixtures ; npm run e2e:test", "lint": "eslint .", @@ -181,6 +181,7 @@ "styled-components": "^3.4.10", "stylus": "^0.57.0", "stylus-loader": "^3.0.2", + "tree-kill": "^1.2.2", "ts-node": "^10.9.1", "type-fest": "^3.13.1", "typescript": "^5.0.4", diff --git a/test/func/tests/.hermione.conf.js b/test/func/tests/.hermione.conf.js index 472b17ced..6ec741103 100644 --- a/test/func/tests/.hermione.conf.js +++ b/test/func/tests/.hermione.conf.js @@ -13,6 +13,8 @@ const serverHost = process.env.SERVER_HOST ?? 'host.docker.internal'; const serverPort = process.env.SERVER_PORT ?? 8083; const projectUnderTest = process.env.PROJECT_UNDER_TEST; +const isRunningGuiTests = projectUnderTest.includes('gui'); + if (!projectUnderTest) { throw 'Project under test was not specified'; } @@ -27,6 +29,9 @@ module.exports = { common: { files: 'common/**/*.hermione.js' }, + 'common-gui': { + files: 'common-gui/**/*.hermione.js' + }, eye: { files: 'eye/**/*.hermione.js', }, @@ -51,7 +56,7 @@ module.exports = { plugins: { 'html-reporter-test-server': { - enabled: true, + enabled: !isRunningGuiTests, port: serverPort }, 'html-reporter-tester': { @@ -61,7 +66,7 @@ module.exports = { }, 'hermione-global-hook': { - beforeEach: async function() { + beforeEach: isRunningGuiTests ? undefined : async function() { await this.browser.url(this.browser.options.baseUrl); await this.browser.execute(() => { document.querySelectorAll('.section').forEach((section) => { diff --git a/test/func/tests/common-gui/index.hermione.js b/test/func/tests/common-gui/index.hermione.js index 8b9da2e9a..616bf5a69 100644 --- a/test/func/tests/common-gui/index.hermione.js +++ b/test/func/tests/common-gui/index.hermione.js @@ -1,63 +1,196 @@ const childProcess = require('child_process'); const fs = require('fs/promises'); const path = require('path'); +const {promisify} = require('util'); + +const treeKill = promisify(require('tree-kill')); const {PORTS} = require('../../utils/constants'); +const {getTestSectionByName} = require('../utils'); const projectName = process.env.PROJECT_UNDER_TEST; const projectDir = path.resolve(__dirname, '../../fixtures', projectName); const guiUrl = `http://host.docker.internal:${PORTS[projectName].gui}`; const reportDir = path.join(projectDir, 'report'); -const reportBackupDir = path.join(projectDir, 'report'); +const reportBackupDir = path.join(projectDir, 'report-backup'); + +const runGui = async () => { + return new Promise((resolve, reject) => { + const child = childProcess.spawn('npm', ['run', 'gui'], {cwd: projectDir}); + + let timeoutId = setTimeout(() => { + treeKill(child.pid).then(() => { + reject(new Error('Couldn\'t start GUI: timed out')); + }); + }, 3000); + + child.stdout.on('data', (data) => { + if (data.toString().includes('GUI is running at')) { + clearTimeout(timeoutId); + resolve(child); + } + }); + + child.stderr.on('data', (data) => { + console.error(`stderr: ${data}`); + }); + + child.on('close', (code) => { + if (code !== 0) { + reject(new Error(`GUI process exited with code ${code}`)); + } + }); + }); +}; // These tests should not be launched in parallel describe('GUI mode', () => { let guiProcess; beforeEach(async ({browser}) => { - await fs.cp(reportDir, reportBackupDir); + await fs.cp(reportDir, reportBackupDir, {recursive: true}); - guiProcess = childProcess.spawn('npm run gui', {cwd: projectDir}); + guiProcess = await runGui(); await browser.url(guiUrl); + await browser.$('button*=Expand all').click(); }); - afterEach(async () => { - guiProcess.kill('SIGINT'); + afterEach(async ({browser}) => { + await browser.execute(() => { + window.localStorage.clear(); + }); + + await treeKill(guiProcess.pid); - await fs.rmdir(reportDir); + await fs.rm(reportDir, {recursive: true, force: true, maxRetries: 3}); await fs.rename(reportBackupDir, reportDir); - // TODO: restore project directory via git + + childProcess.execSync('git restore .', {cwd: path.join(projectDir, 'screens')}); + childProcess.execSync('git clean -dfx .', {cwd: path.join(projectDir, 'screens')}); }); describe('running tests', () => { - it('should run a single test'); - }); + it('should run a single test', async ({browser}) => { + const retryButton = await browser.$([ + getTestSectionByName('successful test'), + '//button[contains(text(), "Retry")]' + ].join('')); - describe('accepting diff', () => { - it('should create a successful retry'); - it('should make the test pass on next run'); - }); + await retryButton.click(); + await retryButton.waitForClickable({timeout: 10000}); - describe('undo accepting diff', () => { - beforeEach(() => { - // TODO: accept diff - }); + // Should be passed + const testSection = await browser.$(getTestSectionByName('successful test')); + const testSectionClassNames = (await testSection.getAttribute('class')).split(' '); - it('should leave project files intact'); - }); + expect(testSectionClassNames).toContain('section_status_success'); + + // History should appear + const historySelector = [getTestSectionByName('successful test'), '//details[.//summary[contains(text(), "History")]]'].join(''); + + await browser.$(historySelector).click(); + const historyText = await browser.$(historySelector + '/div').getText(); - describe('accepting new screenshot', () => { - it('should create a successful retry'); - it('should make the test pass on next run'); + expect(historyText.includes('<-')).toBeTruthy(); + }); }); - describe('undo accepting new screenshot', () => { - beforeEach(() => { - // TODO: accept new screenshot + for (const testName of ['diff', 'no ref']) { + const fullTestName = `test with ${testName}`; + describe(`accepting ${fullTestName}`, () => { + beforeEach(async ({browser}) => { + await browser.$(getTestSectionByName(fullTestName)).scrollIntoView(); + + const acceptButton = await browser.$([ + getTestSectionByName(fullTestName), + '//button[contains(text(), "Accept")]' + ].join('')); + + await acceptButton.click(); + + // Waiting a bit for the GUI to save images + await new Promise(resolve => setTimeout(resolve, 500)); + }); + + it('should create a successful retry', async ({browser}) => { + const allRetryButtonsSelector = [ + getTestSectionByName(fullTestName), + '//button[@data-test-id="retry-switcher"]' + ].join(''); + const retrySwitcher = browser.$(`(${allRetryButtonsSelector})[last()]`); + + await retrySwitcher.assertView('retry-switcher'); + + const testSection = await browser.$(getTestSectionByName(fullTestName)); + const testSectionClassNames = (await testSection.getAttribute('class')).split(' '); + + expect(testSectionClassNames).toContain('section_status_success'); + }); + + it('should make the test pass on next run', async ({browser}) => { + const retryButton = await browser.$([ + getTestSectionByName(fullTestName), + '//button[contains(text(), "Retry")]' + ].join('')); + + await retryButton.click(); + await retryButton.waitForClickable({timeout: 10000}); + + // Verify green retry button + const allRetryButtonsSelector = [ + getTestSectionByName(fullTestName), + '//button[@data-test-id="retry-switcher"]' + ].join(''); + const retrySwitcher = browser.$(`(${allRetryButtonsSelector})[last()]`); + + await retrySwitcher.assertView('retry-switcher'); + + // Verify green test section + const testSection = await browser.$(getTestSectionByName(fullTestName)); + const testSectionClassNames = (await testSection.getAttribute('class')).split(' '); + + expect(testSectionClassNames).toContain('section_status_success'); + }); }); + } - it('should leave project files intact'); - }); + for (const testName of ['diff', 'no ref']) { + const fullTestName = `test with ${testName}`; + describe(`undo accepting ${fullTestName}`, () => { + beforeEach(async ({browser}) => { + await browser.$(getTestSectionByName(fullTestName)).scrollIntoView(); + + const acceptButton = await browser.$([ + getTestSectionByName(fullTestName), + '//button[contains(text(), "Accept")]' + ].join('')); + + await acceptButton.click(); + + // Waiting a bit for the GUI to save images + await new Promise(resolve => setTimeout(resolve, 500)); + }); + + it('should leave project files intact', async ({browser}) => { + const gitDiffBeforeUndo = childProcess.execSync('git status . --porcelain=v2', {cwd: path.join(projectDir, 'screens')}); + + const undoButton = await browser.$([ + getTestSectionByName(fullTestName), + '//button[contains(text(), "Undo")]' + ].join('')); + + await undoButton.click(); + + // Waiting a bit for the GUI to save images + await new Promise(resolve => setTimeout(resolve, 500)); + + const gitDiffAfterUndo = childProcess.execSync('git status . --porcelain=v2', {cwd: path.join(projectDir, 'screens')}); + + expect(gitDiffBeforeUndo.length > 0).toBeTruthy(); + expect(gitDiffAfterUndo.length === 0).toBeTruthy(); + }); + }); + } }); diff --git a/test/func/tests/common/test-results-appearance.hermione.js b/test/func/tests/common/test-results-appearance.hermione.js index d870fae3c..7d7ae66dd 100644 --- a/test/func/tests/common/test-results-appearance.hermione.js +++ b/test/func/tests/common/test-results-appearance.hermione.js @@ -66,6 +66,7 @@ describe('Test results appearance', () => { describe('Test with no ref image', function() { it('should have pink retry selector', async ({browser}) => { + // TODO const retrySelectorButton = await browser.$('//div[contains(text(),\'test without screenshot\')]/..//button[@data-test-id="retry-switcher"]'); await hideHeader(browser); diff --git a/test/func/tests/package.json b/test/func/tests/package.json index 74282b095..e0c7deedf 100644 --- a/test/func/tests/package.json +++ b/test/func/tests/package.json @@ -5,11 +5,13 @@ "scripts": { "gui:hermione-common": "PROJECT_UNDER_TEST=hermione npx hermione --set common gui", "gui:hermione-eye": "PROJECT_UNDER_TEST=hermione-eye npx hermione --no --set eye gui", + "gui:hermione-gui": "PROJECT_UNDER_TEST=hermione-gui npx hermione --no --set common-gui gui", "gui:playwright": "PROJECT_UNDER_TEST=playwright npx hermione --set common gui", "gui:plugins": "PROJECT_UNDER_TEST=plugins SERVER_PORT=8084 npx hermione --set plugins gui", - "hermione:hermione": "conc npm:hermione:*", + "hermione:hermione": "conc npm:hermione:hermione*", "hermione:hermione-common": "PROJECT_UNDER_TEST=hermione npx hermione --set common", "hermione:hermione-eye": "PROJECT_UNDER_TEST=hermione-eye npx hermione --set common", + "hermione:hermione-gui": "PROJECT_UNDER_TEST=hermione-gui npx hermione --no --set common-gui", "hermione:playwright": "PROJECT_UNDER_TEST=playwright npx hermione --set common", "hermione:plugins": "PROJECT_UNDER_TEST=plugins SERVER_PORT=8084 npx hermione --set plugins", "test": "conc npm:hermione:*" diff --git a/test/func/tests/screens/197b2c2/chrome/retry-switcher.png b/test/func/tests/screens/197b2c2/chrome/retry-switcher.png new file mode 100644 index 0000000000000000000000000000000000000000..3864eb30499d9b1697007f77fb18180f3aebbc5c GIT binary patch literal 491 zcmVJNR7i>KmQ70nK^(?Mlsfvd?zWX08d=v6tYcR_D55UtvVkEhvM`4q*oc1wYCYv+6-e5S=~Mzyrg;%lTAxlY+1SHiY27|#Yp#9Z`=o6vcXec#Hpr3Rpm6|1J*Bad0ZgjpoV?e*( zhvbu%ew5CU=n*j-4nb1clo2^^fO@4u(G(C?w^hnQ=SXkraBsL-ZwU9t=|q#}{}n*| zJPxPV3ER32@_`KHLg5F{A9cw4GV*tMR9jU1oK7eYgipoo6=FGL@LZlHf zuLn+#(>y@#HV2E(F* h*xj_2HX@%|#~%&u_^(+G->3ip002ovPDHLkV1gN5*lYj* literal 0 HcmV?d00001 diff --git a/test/func/tests/screens/87f8b7e/chrome/retry-switcher.png b/test/func/tests/screens/87f8b7e/chrome/retry-switcher.png new file mode 100644 index 0000000000000000000000000000000000000000..d788a5e780f9ba001dc1a8ce5aab43f699361990 GIT binary patch literal 517 zcmV+g0{Z=lP)Kmdi^5K^(^i@lk{*u8-}3T?9#wo`O7d3Bi+SE9n>m zB{2CR!nClHpzS zqW)R$1KRC2{6YW3i|S+KTk@gX?Sf=-8KZL60O?YKAPXS0Z_~6zA0wrzz#H&VZwT|n z^??TF{{?{dt|7h?hbQQPZN>&Z%tJgA*#XM;Wkfb2DBYD%d8r_In1pTChP|gfHh_cr z!MCx+YC--c&)x&=O+!LUz_sAQ{_{Rt0Jq%#PC4cr$eriRIYpJ%3WO5@_BlHWHwE*K zX?Q)1m=H6^WWAvyv=+kZ)+!r7`Z$gFZXC*^Vi=n`O2O)|qIg?m15h3mL=#as1J3?M zTj7yC&6;Df{sa93@H<+>nf=QYWWk6UtO3%61aVsgp&^Mz4AuZ{ojy6ylH3b! zBWlpCP16=yxr=w9*=mkPM;ekcQf3%6CK8>yDQ_B;52NE38}RuQ=xP7q00000NkvXX Hu0mjf2Gi)F literal 0 HcmV?d00001 diff --git a/test/func/tests/screens/9b544b9/chrome/retry-switcher.png b/test/func/tests/screens/9b544b9/chrome/retry-switcher.png new file mode 100644 index 0000000000000000000000000000000000000000..3864eb30499d9b1697007f77fb18180f3aebbc5c GIT binary patch literal 491 zcmVJNR7i>KmQ70nK^(?Mlsfvd?zWX08d=v6tYcR_D55UtvVkEhvM`4q*oc1wYCYv+6-e5S=~Mzyrg;%lTAxlY+1SHiY27|#Yp#9Z`=o6vcXec#Hpr3Rpm6|1J*Bad0ZgjpoV?e*( zhvbu%ew5CU=n*j-4nb1clo2^^fO@4u(G(C?w^hnQ=SXkraBsL-ZwU9t=|q#}{}n*| zJPxPV3ER32@_`KHLg5F{A9cw4GV*tMR9jU1oK7eYgipoo6=FGL@LZlHf zuLn+#(>y@#HV2E(F* h*xj_2HX@%|#~%&u_^(+G->3ip002ovPDHLkV1gN5*lYj* literal 0 HcmV?d00001 diff --git a/test/func/tests/screens/9e6270b/chrome/retry-switcher.png b/test/func/tests/screens/9e6270b/chrome/retry-switcher.png new file mode 100644 index 0000000000000000000000000000000000000000..d788a5e780f9ba001dc1a8ce5aab43f699361990 GIT binary patch literal 517 zcmV+g0{Z=lP)Kmdi^5K^(^i@lk{*u8-}3T?9#wo`O7d3Bi+SE9n>m zB{2CR!nClHpzS zqW)R$1KRC2{6YW3i|S+KTk@gX?Sf=-8KZL60O?YKAPXS0Z_~6zA0wrzz#H&VZwT|n z^??TF{{?{dt|7h?hbQQPZN>&Z%tJgA*#XM;Wkfb2DBYD%d8r_In1pTChP|gfHh_cr z!MCx+YC--c&)x&=O+!LUz_sAQ{_{Rt0Jq%#PC4cr$eriRIYpJ%3WO5@_BlHWHwE*K zX?Q)1m=H6^WWAvyv=+kZ)+!r7`Z$gFZXC*^Vi=n`O2O)|qIg?m15h3mL=#as1J3?M zTj7yC&6;Df{sa93@H<+>nf=QYWWk6UtO3%61aVsgp&^Mz4AuZ{ojy6ylH3b! zBWlpCP16=yxr=w9*=mkPM;ekcQf3%6CK8>yDQ_B;52NE38}RuQ=xP7q00000NkvXX Hu0mjf2Gi)F literal 0 HcmV?d00001 diff --git a/test/func/tests/screens/aec277d/chrome/retry-switcher.png b/test/func/tests/screens/aec277d/chrome/retry-switcher.png new file mode 100644 index 0000000000000000000000000000000000000000..d788a5e780f9ba001dc1a8ce5aab43f699361990 GIT binary patch literal 517 zcmV+g0{Z=lP)Kmdi^5K^(^i@lk{*u8-}3T?9#wo`O7d3Bi+SE9n>m zB{2CR!nClHpzS zqW)R$1KRC2{6YW3i|S+KTk@gX?Sf=-8KZL60O?YKAPXS0Z_~6zA0wrzz#H&VZwT|n z^??TF{{?{dt|7h?hbQQPZN>&Z%tJgA*#XM;Wkfb2DBYD%d8r_In1pTChP|gfHh_cr z!MCx+YC--c&)x&=O+!LUz_sAQ{_{Rt0Jq%#PC4cr$eriRIYpJ%3WO5@_BlHWHwE*K zX?Q)1m=H6^WWAvyv=+kZ)+!r7`Z$gFZXC*^Vi=n`O2O)|qIg?m15h3mL=#as1J3?M zTj7yC&6;Df{sa93@H<+>nf=QYWWk6UtO3%61aVsgp&^Mz4AuZ{ojy6ylH3b! zBWlpCP16=yxr=w9*=mkPM;ekcQf3%6CK8>yDQ_B;52NE38}RuQ=xP7q00000NkvXX Hu0mjf2Gi)F literal 0 HcmV?d00001 diff --git a/test/func/tests/screens/c9299b2/chrome/retry-switcher.png b/test/func/tests/screens/c9299b2/chrome/retry-switcher.png new file mode 100644 index 0000000000000000000000000000000000000000..3864eb30499d9b1697007f77fb18180f3aebbc5c GIT binary patch literal 491 zcmVJNR7i>KmQ70nK^(?Mlsfvd?zWX08d=v6tYcR_D55UtvVkEhvM`4q*oc1wYCYv+6-e5S=~Mzyrg;%lTAxlY+1SHiY27|#Yp#9Z`=o6vcXec#Hpr3Rpm6|1J*Bad0ZgjpoV?e*( zhvbu%ew5CU=n*j-4nb1clo2^^fO@4u(G(C?w^hnQ=SXkraBsL-ZwU9t=|q#}{}n*| zJPxPV3ER32@_`KHLg5F{A9cw4GV*tMR9jU1oK7eYgipoo6=FGL@LZlHf zuLn+#(>y@#HV2E(F* h*xj_2HX@%|#~%&u_^(+G->3ip002ovPDHLkV1gN5*lYj* literal 0 HcmV?d00001