diff --git a/lib/static/components/controls/control-button.tsx b/lib/static/components/controls/control-button.tsx index b089750e7..39fa189f1 100644 --- a/lib/static/components/controls/control-button.tsx +++ b/lib/static/components/controls/control-button.tsx @@ -13,6 +13,7 @@ interface ControlButtonProps { isDisabled?: boolean; isRunning?: boolean; extendClassNames?: string | string[]; + dataTestId?: string | number; } export default class ControlButton extends Component { @@ -26,7 +27,8 @@ export default class ControlButton extends Component { isSuiteControl: PropTypes.bool, isControlGroup: PropTypes.bool, isRunning: PropTypes.bool, - extendClassNames: PropTypes.oneOfType([PropTypes.array, PropTypes.string]) + extendClassNames: PropTypes.oneOfType([PropTypes.array, PropTypes.string]), + dataTestId: PropTypes.oneOfType([PropTypes.string, PropTypes.number]) }; render(): JSX.Element { @@ -40,7 +42,8 @@ export default class ControlButton extends Component { isControlGroup, isDisabled = false, isRunning = false, - extendClassNames + extendClassNames, + dataTestId } = this.props; const className = classNames( @@ -58,6 +61,7 @@ export default class ControlButton extends Component { onClick={handler} className={className} disabled={isDisabled} + data-test-id={dataTestId} > {label} ; diff --git a/lib/static/components/controls/strict-match-filter-input.jsx b/lib/static/components/controls/strict-match-filter-input.jsx index bf785041b..14dd226fd 100644 --- a/lib/static/components/controls/strict-match-filter-input.jsx +++ b/lib/static/components/controls/strict-match-filter-input.jsx @@ -23,6 +23,7 @@ const StrictMatchFilterInput = ({strictMatchFilter, actions}) => { label="Strict match" onChange={onChange} checked={checked} + data-test-id="header-strict-match" /> ); diff --git a/lib/static/components/controls/test-name-filter-input.jsx b/lib/static/components/controls/test-name-filter-input.jsx index 33ce6f076..f11facaa4 100644 --- a/lib/static/components/controls/test-name-filter-input.jsx +++ b/lib/static/components/controls/test-name-filter-input.jsx @@ -27,6 +27,7 @@ const TestNameFilterInput = ({actions, testNameFilter: testNameFilterProp}) => { value={testNameFilter} placeholder="filter by test name" onChange={onChange} + data-test-id="header-test-name-filter" /> ); }; diff --git a/lib/static/components/modals/screenshot-accepter/body.jsx b/lib/static/components/modals/screenshot-accepter/body.jsx index 76f0a0602..430ec4f45 100644 --- a/lib/static/components/modals/screenshot-accepter/body.jsx +++ b/lib/static/components/modals/screenshot-accepter/body.jsx @@ -51,7 +51,7 @@ class ScreenshotAccepterBody extends Component { return (
- {testName} + {testName} {'/'} {browserName} {'/'} diff --git a/lib/static/components/modals/screenshot-accepter/header.jsx b/lib/static/components/modals/screenshot-accepter/header.jsx index 8d05ff6ca..d9008265c 100644 --- a/lib/static/components/modals/screenshot-accepter/header.jsx +++ b/lib/static/components/modals/screenshot-accepter/header.jsx @@ -143,6 +143,7 @@ export default class ScreenshotAccepterHeader extends Component { isDisabled={images.length === 0} extendClassNames="screenshot-accepter__accept-btn" handler={this.handleScreenshotAccept} + dataTestId="screenshot-accepter-accept" /> - +
diff --git a/lib/static/components/progress-bar/index.jsx b/lib/static/components/progress-bar/index.jsx index d2c171c08..aa0e9ff40 100644 --- a/lib/static/components/progress-bar/index.jsx +++ b/lib/static/components/progress-bar/index.jsx @@ -2,11 +2,11 @@ import React from 'react'; import './index.styl'; -export default ({done, total}) => { +export default ({done, total, dataTestId}) => { const percent = (done / total).toFixed(2) * 100; return ( - + ); diff --git a/lib/static/components/section/body/index.jsx b/lib/static/components/section/body/index.jsx index 3c33e3136..f34e4b437 100644 --- a/lib/static/components/section/body/index.jsx +++ b/lib/static/components/section/body/index.jsx @@ -67,6 +67,7 @@ class Body extends Component { isSuiteControl={true} isDisabled={running} handler={this.onTestRetry} + dataTestId="test-retry" /> ) diff --git a/lib/static/components/state/index.jsx b/lib/static/components/state/index.jsx index dc8b64164..384d9cc6b 100644 --- a/lib/static/components/state/index.jsx +++ b/lib/static/components/state/index.jsx @@ -101,6 +101,7 @@ class State extends Component { isDisabled={isScreenshotAccepterDisabled} extendClassNames="screenshot-accepter__arrows-open-btn" handler={() => this.toggleModal()} + data-test-id="test-switch-accept-mode" /> ); diff --git a/test/func/fixtures/playwright/tests/screens/failed-describe-test-with-successful-assertView-and-error/chromium/header-success.png b/test/func/fixtures/playwright/tests/screens/failed-describe-test-with-successful-assertView-and-error/chromium/header-success.png index 2baf53cb5..3a5f9d589 100644 Binary files a/test/func/fixtures/playwright/tests/screens/failed-describe-test-with-successful-assertView-and-error/chromium/header-success.png and b/test/func/fixtures/playwright/tests/screens/failed-describe-test-with-successful-assertView-and-error/chromium/header-success.png differ diff --git a/test/func/tests/.hermione.conf.js b/test/func/tests/.hermione.conf.js index 5d72a170a..21ee6c65e 100644 --- a/test/func/tests/.hermione.conf.js +++ b/test/func/tests/.hermione.conf.js @@ -24,26 +24,20 @@ const commonConfig = getCommonConfig(__dirname); const config = _.merge(commonConfig, { baseUrl: `http://${serverHost}:${serverPort}/fixtures/${projectUnderTest}/report/index.html`, - browsers: { - // TODO: this is a hack to be able to have 2 sets of screenshots, for hermione-based report and pwt-based report - // currently, those have weird tiny diffs. Would be nice to figure out the cause and have common screenshots. - 'chrome-pwt': {...commonConfig.browsers.chrome} - }, - sets: { common: { files: 'common/**/*.hermione.js' }, 'common-gui': { - browsers: ['chrome'], files: 'common-gui/**/*.hermione.js' }, + 'common-tinder': { + files: 'common-tinder/**/*.hermione.js' + }, eye: { - browsers: ['chrome'], files: 'eye/**/*.hermione.js', }, plugins: { - browsers: ['chrome'], files: 'plugins/**/*.hermione.js' } }, diff --git a/test/func/tests/common-gui/index.hermione.js b/test/func/tests/common-gui/index.hermione.js index aee49712e..ddc87b890 100644 --- a/test/func/tests/common-gui/index.hermione.js +++ b/test/func/tests/common-gui/index.hermione.js @@ -6,7 +6,15 @@ const {promisify} = require('util'); const treeKill = promisify(require('tree-kill')); const {PORTS} = require('../../utils/constants'); -const {getTestSectionByNameSelector, getSpoilerByNameSelector, getElementWithTextSelector, hideScreenshots} = require('../utils'); +const { + getTestSectionByNameSelector, + getSpoilerByNameSelector, + getElementWithTextSelector, + hideScreenshots, + runGui, + waitForFsChanges, + getFsDiffFromVcs +} = require('../utils'); const serverHost = process.env.SERVER_HOST ?? 'host.docker.internal'; @@ -18,57 +26,6 @@ const reportDir = path.join(projectDir, 'report'); const reportBackupDir = path.join(projectDir, 'report-backup'); const screensDir = path.join(projectDir, 'screens'); -const runGui = async () => { - return new Promise((resolve, reject) => { - const child = childProcess.spawn('npm', ['run', 'gui'], {cwd: projectDir}); - - let processKillTimeoutId = setTimeout(() => { - treeKill(child.pid).then(() => { - reject(new Error('Couldn\'t start GUI: timed out')); - }); - }, 3000); - - child.stdout.on('data', (data) => { - if (data.toString().includes('GUI is running at')) { - clearTimeout(processKillTimeoutId); - resolve(child); - } - }); - - child.stderr.on('data', (data) => { - console.error(`stderr: ${data}`); - }); - - child.on('close', (code) => { - if (code !== 0) { - reject(new Error(`GUI process exited with code ${code}`)); - } - }); - }); -}; - -const getFsDiffFromVcs = (directory) => childProcess.execSync('git status . --porcelain=v2', {cwd: directory}); - -const waitForFsChanges = async (dirPath, condition = (output) => output.length > 0, {timeout = 1000, interval = 50} = {}) => { - let isTimedOut = false; - - const timeoutId = setTimeout(() => { - isTimedOut = true; - throw new Error(`Timed out while waiting for fs changes in ${dirPath} for ${timeout}ms`); - }, timeout); - - while (!isTimedOut) { - const output = getFsDiffFromVcs(dirPath); - - if (condition(output)) { - clearTimeout(timeoutId); - return; - } - - await new Promise(resolve => setTimeout(resolve, interval)); - } -}; - // These tests should not be launched in parallel describe('GUI mode', () => { let guiProcess; @@ -76,7 +33,7 @@ describe('GUI mode', () => { beforeEach(async ({browser}) => { await fs.cp(reportDir, reportBackupDir, {recursive: true}); - guiProcess = await runGui(); + guiProcess = await runGui(projectDir); await browser.url(guiUrl); await browser.$('button*=Expand all').click(); diff --git a/test/func/tests/common-tinder/index.hermione.js b/test/func/tests/common-tinder/index.hermione.js new file mode 100644 index 000000000..e17a33afe --- /dev/null +++ b/test/func/tests/common-tinder/index.hermione.js @@ -0,0 +1,134 @@ +const childProcess = require('child_process'); +const fs = require('fs/promises'); +const path = require('path'); +const treeKill = require('tree-kill'); + +const {PORTS} = require('../../utils/constants'); +const {hideScreenshots, runGui, waitForFsChanges, getFsDiffFromVcs} = require('../utils'); + +const serverHost = process.env.SERVER_HOST ?? 'host.docker.internal'; + +const projectName = process.env.PROJECT_UNDER_TEST; +const projectDir = path.resolve(__dirname, '../../fixtures', projectName); +const guiUrl = `http://${serverHost}:${PORTS[projectName].gui}`; + +const reportDir = path.join(projectDir, 'report'); +const reportBackupDir = path.join(projectDir, 'report-backup'); +const screensDir = path.join(projectDir, 'screens'); + +// These tests should not be launched in parallel +describe('Tinder mode', () => { + let guiProcess; + + beforeEach(async ({browser}) => { + await fs.cp(reportDir, reportBackupDir, {recursive: true}); + + guiProcess = await runGui(projectDir); + + await browser.url(guiUrl); + await browser.$('button*=Expand all').click(); + + await browser.$('button*=Switch accept mode').click(); + }); + + afterEach(async () => { + await treeKill(guiProcess.pid); + + await fs.rm(reportDir, {recursive: true, force: true, maxRetries: 3}); + await fs.rename(reportBackupDir, reportDir); + + childProcess.execSync('git restore .', {cwd: screensDir}); + childProcess.execSync('git clean -dfx .', {cwd: screensDir}); + }); + + describe(`accepting screenshot`, () => { + beforeEach(async ({browser}) => { + const testFullName = await browser.$('span[data-test-id="screenshot-accepter-test-name"]').getText(); + + const acceptButton = await browser.$('button[data-test-id="screenshot-accepter-accept"]'); + await acceptButton.click(); + + await browser.waitUntil(async () => { + const progress = await browser.$('span[data-test-id="screenshot-accepter-progress-bar"]').getAttribute('data-content'); + + return progress === '1/2'; + }, {interval: 100}); + + const switchAcceptModeButton = await browser.$('button[data-test-id="screenshot-accepter-switch-accept-mode"]'); + await switchAcceptModeButton.click(); + + const testNameFilterInput = await browser.$('input[data-test-id="header-test-name-filter"]'); + + await testNameFilterInput.setValue(testFullName); + await browser.$('div[data-test-id="header-strict-match"]').click(); + + await waitForFsChanges(screensDir); + }); + + it('should create a successful retry', async ({browser}) => { + const retrySwitcher = await browser.$(`(//button[@data-test-id="retry-switcher"])[last()]`); + await hideScreenshots(browser); + + await retrySwitcher.assertView('retry-switcher'); + }); + + it('should make the test pass on next run', async ({browser}) => { + const retryButton = await browser.$('button[data-test-id="test-retry"]'); + + // TODO: find a correct sign to wait for. Issue is that retry button is totally clickable, but doesn't + // work right away after switch accept mode and applying filtering for some reason. + await browser.pause(500); + await retryButton.click(); + + await retryButton.waitForClickable({reverse: true, timeout: 10000}); + await retryButton.waitForClickable({timeout: 10000}); + + const retrySwitcher = await browser.$(`(//button[@data-test-id="retry-switcher"])[last()]`); + await hideScreenshots(browser); + + await retrySwitcher.assertView('retry-switcher'); + }); + }); + + describe(`undo accepting screenshot`, () => { + it('should leave project files intact', async ({browser}) => { + const acceptButton = await browser.$('button[data-test-id="screenshot-accepter-accept"]'); + await acceptButton.click(); + + await browser.waitUntil(async () => { + const progress = await browser.$('span[data-test-id="screenshot-accepter-progress-bar"]').getAttribute('data-content'); + + return progress === '1/2'; + }, {interval: 100}); + + await waitForFsChanges(screensDir); + const fsDiffBeforeUndo = getFsDiffFromVcs(screensDir); + + const undoButton = await browser.$('button[data-test-id="screenshot-accepter-undo"]'); + await undoButton.click(); + + await waitForFsChanges(screensDir, (output) => output.length === 0); + + const fsDiffAfterUndo = getFsDiffFromVcs(screensDir); + + expect(fsDiffBeforeUndo.length > 0).toBeTruthy(); + expect(fsDiffAfterUndo.length === 0).toBeTruthy(); + }); + }); + + it('should show success screen after accepting all screenshots', async ({browser}) => { + const acceptButton = await browser.$('button[data-test-id="screenshot-accepter-accept"]'); + + for (let i = 1; i <= 2; i++) { + await acceptButton.click(); + + await browser.waitUntil(async () => { + const progress = await browser.$('span[data-test-id="screenshot-accepter-progress-bar"]').getAttribute('data-content'); + + return progress === `${i}/2`; + }, {interval: 100}); + } + + await expect(await browser.$('div*=All screenshots are accepted')).toBeDisplayed(); + }); +}); diff --git a/test/func/tests/common/tests-details.hermione.js b/test/func/tests/common/tests-details.hermione.js index 60832fa0a..bf75f8cc9 100644 --- a/test/func/tests/common/tests-details.hermione.js +++ b/test/func/tests/common/tests-details.hermione.js @@ -21,7 +21,7 @@ describe('Test details', function() { it('should prevent details summary overflow', async ({browser}) => { const selector = getTestSectionByNameSelector('test with long error message') + - `//summary[.${getElementWithTextSelector('span', 'message')}/..]`; + `//summary[.${getElementWithTextSelector('span', 'stack')}/..]`; await browser.$(selector).waitForDisplayed(); await browser.assertView('details summary', selector); diff --git a/test/func/tests/local.hermione.conf.js b/test/func/tests/local.hermione.conf.js new file mode 100644 index 000000000..f1de1cd60 --- /dev/null +++ b/test/func/tests/local.hermione.conf.js @@ -0,0 +1,31 @@ +/* +This hermione config may be useful for running tests on a local, non-headless Chromium browser while debugging. + +Use it as follows: +npm run gui:hermione-common -- -c local.hermione.conf.js +*/ + +process.env.SERVER_HOST = 'localhost'; + +const _ = require('lodash'); + +const mainConfig = require('./.hermione.conf.js'); + +const config = _.merge(mainConfig, { + browsers: { + chrome: { + automationProtocol: 'devtools', + desiredCapabilities: { + 'goog:chromeOptions': { + args: ['no-sandbox', 'hide-scrollbars'] + } + }, + waitTimeout: 3000 + } + } +}); + +delete config.gridUrl; +delete config.browsers.chrome.desiredCapabilities['goog:chromeOptions'].binary; + +module.exports = config; diff --git a/test/func/tests/package.json b/test/func/tests/package.json index 0d4765bea..c55294af5 100644 --- a/test/func/tests/package.json +++ b/test/func/tests/package.json @@ -3,16 +3,18 @@ "version": "0.0.0", "private": true, "scripts": { - "gui:hermione-common": "PROJECT_UNDER_TEST=hermione npx hermione --set common -b chrome gui", + "gui:hermione-common": "PROJECT_UNDER_TEST=hermione npx hermione --set common gui", "gui:hermione-eye": "PROJECT_UNDER_TEST=hermione-eye npx hermione --no --set eye gui", "gui:hermione-gui": "PROJECT_UNDER_TEST=hermione-gui npx hermione --no --set common-gui gui", - "gui:playwright": "PROJECT_UNDER_TEST=playwright npx hermione --set common -b chrome-pwt gui", + "gui:playwright": "PROJECT_UNDER_TEST=playwright npx hermione --set common gui", "gui:plugins": "PROJECT_UNDER_TEST=plugins SERVER_PORT=8084 npx hermione --set plugins gui", - "hermione:hermione-common": "PROJECT_UNDER_TEST=hermione SERVER_PORT=8061 npx hermione --set common -b chrome", + "gui:hermione-tinder": "PROJECT_UNDER_TEST=hermione-gui SERVER_PORT=8084 npx hermione --set common-tinder gui", + "hermione:hermione-common": "PROJECT_UNDER_TEST=hermione SERVER_PORT=8061 npx hermione --set common", "hermione:hermione-eye": "PROJECT_UNDER_TEST=hermione-eye SERVER_PORT=8062 npx hermione --set eye", "hermione:hermione-gui": "PROJECT_UNDER_TEST=hermione-gui SERVER_PORT=8063 npx hermione --no --set common-gui", - "hermione:playwright": "PROJECT_UNDER_TEST=playwright SERVER_PORT=8065 npx hermione --set common -b chrome-pwt", + "hermione:playwright": "PROJECT_UNDER_TEST=playwright SERVER_PORT=8065 npx hermione --set common", "hermione:plugins": "PROJECT_UNDER_TEST=plugins SERVER_PORT=8064 npx hermione --set plugins", + "hermione:hermione-tinder": "PROJECT_UNDER_TEST=hermione-gui SERVER_PORT=8084 npx hermione --set common-tinder", "test": "run-s hermione:*" } } diff --git a/test/func/tests/screens/1bb949f/chrome/retry-switcher.png b/test/func/tests/screens/1bb949f/chrome/retry-switcher.png new file mode 100644 index 000000000..996698da7 Binary files /dev/null and b/test/func/tests/screens/1bb949f/chrome/retry-switcher.png differ diff --git a/test/func/tests/screens/bdf4a21/chrome/retry-switcher.png b/test/func/tests/screens/bdf4a21/chrome/retry-switcher.png new file mode 100644 index 000000000..cf2e130cb Binary files /dev/null and b/test/func/tests/screens/bdf4a21/chrome/retry-switcher.png differ diff --git a/test/func/tests/screens/c0db305/chrome/details summary.png b/test/func/tests/screens/c0db305/chrome/details summary.png index 13aa740d6..cbf9826f5 100644 Binary files a/test/func/tests/screens/c0db305/chrome/details summary.png and b/test/func/tests/screens/c0db305/chrome/details summary.png differ diff --git a/test/func/tests/utils.js b/test/func/tests/utils.js index 5d045f93e..980516b61 100644 --- a/test/func/tests/utils.js +++ b/test/func/tests/utils.js @@ -1,3 +1,7 @@ +const childProcess = require('child_process'); +const {promisify} = require('util'); +const treeKill = promisify(require('tree-kill')); + /** Returns a div, which wraps the whole test section with specified name */ const getTestSectionByNameSelector = (testName) => `//div[contains(text(),'${testName}')]/..`; @@ -28,6 +32,57 @@ const hideScreenshots = async (browser) => { }); }; +const runGui = async (projectDir) => { + return new Promise((resolve, reject) => { + const child = childProcess.spawn('npm', ['run', 'gui'], {cwd: projectDir}); + + let processKillTimeoutId = setTimeout(() => { + treeKill(child.pid).then(() => { + reject(new Error('Couldn\'t start GUI: timed out')); + }); + }, 3000); + + child.stdout.on('data', (data) => { + if (data.toString().includes('GUI is running at')) { + clearTimeout(processKillTimeoutId); + resolve(child); + } + }); + + child.stderr.on('data', (data) => { + console.error(`stderr: ${data}`); + }); + + child.on('close', (code) => { + if (code !== 0) { + reject(new Error(`GUI process exited with code ${code}`)); + } + }); + }); +}; + +const getFsDiffFromVcs = (directory) => childProcess.execSync('git status . --porcelain=v2', {cwd: directory}); + +const waitForFsChanges = async (dirPath, condition = (output) => output.length > 0, {timeout = 1000, interval = 50} = {}) => { + let isTimedOut = false; + + const timeoutId = setTimeout(() => { + isTimedOut = true; + throw new Error(`Timed out while waiting for fs changes in ${dirPath} for ${timeout}ms`); + }, timeout); + + while (!isTimedOut) { + const output = getFsDiffFromVcs(dirPath); + + if (condition(output)) { + clearTimeout(timeoutId); + return; + } + + await new Promise(resolve => setTimeout(resolve, interval)); + } +}; + module.exports = { getTestSectionByNameSelector, getTestStateByNameSelector, @@ -35,5 +90,8 @@ module.exports = { getElementWithTextSelector, getSpoilerByNameSelector, hideHeader, - hideScreenshots + hideScreenshots, + runGui, + getFsDiffFromVcs, + waitForFsChanges };