From 33cb431471477c65d7c954eca0ef8cf5d55262ad Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Mon, 22 Mar 2021 13:04:34 +0100 Subject: [PATCH] [test] Create end-to-end testing CI job (#25405) --- .circleci/config.yml | 16 ++ package.json | 5 + .../Unstable_TrapFocus/Unstable_TrapFocus.js | 2 +- .../Unstable_TrapFocus.test.js | 53 +----- test/README.md | 5 + test/e2e/.mocharc.js | 8 + test/e2e/README.md | 29 +++ test/e2e/TestViewer.js | 30 ++++ test/e2e/index.js | 94 ++++++++++ test/e2e/index.test.ts | 62 +++++++ test/e2e/serve.json | 4 + test/e2e/template.html | 16 ++ .../Unstable_TrapFocus/OpenTrapFocus.tsx | 20 +++ test/e2e/webpack.config.js | 33 ++++ test/tsconfig.json | 2 +- test/utils/createClientRender.tsx | 3 +- test/utils/user-event/index.js | 166 ------------------ test/utils/user-event/index.test.js | 72 -------- 18 files changed, 327 insertions(+), 293 deletions(-) create mode 100644 test/e2e/.mocharc.js create mode 100644 test/e2e/README.md create mode 100644 test/e2e/TestViewer.js create mode 100644 test/e2e/index.js create mode 100644 test/e2e/index.test.ts create mode 100644 test/e2e/serve.json create mode 100644 test/e2e/template.html create mode 100644 test/e2e/tests/Unstable_TrapFocus/OpenTrapFocus.tsx create mode 100644 test/e2e/webpack.config.js delete mode 100644 test/utils/user-event/index.js delete mode 100644 test/utils/user-event/index.test.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 377dd5bc247fdc..c2a53c619b05de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -336,6 +336,19 @@ jobs: # hardcoded in karma-webpack path: /tmp/_karma_webpack_ destination: artifact-file + test_e2e: + <<: *defaults + docker: + - image: mcr.microsoft.com/playwright@sha256:1700531ce01a3d974cc440bb8efcf43d31d58ee5f1d354fc21563ea5fe4291e6 + environment: + NODE_ENV: development # Needed if playwright is in `devDependencies` + steps: + - checkout + - install_js: + browsers: true + - run: + name: yarn test:e2e + command: yarn test:e2e - run: name: Can we generate the @material-ui/core umd build? command: yarn workspace @material-ui/core build:umd @@ -410,6 +423,9 @@ workflows: - test_regressions: requires: - checkout + - test_e2e: + requires: + - checkout profile: when: equal: [profile, << pipeline.parameters.workflow >>] diff --git a/package.json b/package.json index fd898c005ffb1f..7a14da0fe4e4f3 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,11 @@ "test:coverage": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc --reporter=text mocha 'packages/**/*.test.{js,ts,tsx}' 'docs/**/*.test.{js,ts,tsx}' 'scripts/**/*.test.{js,ts,tsx}' 'test/utils/**/*.test.{js,ts,tsx}'", "test:coverage:ci": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc --reporter=lcov mocha 'packages/**/*.test.{js,ts,tsx}' 'docs/**/*.test.{js,ts,tsx}' 'scripts/**/*.test.{js,ts,tsx}' 'test/utils/**/*.test.{js,ts,tsx}'", "test:coverage:html": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc --reporter=html mocha 'packages/**/*.test.{js,ts,tsx}' 'docs/**/*.test.{js,ts,tsx}' 'scripts/**/*.test.{js,ts,tsx}' 'test/utils/**/*.test.{js,ts,tsx}'", + "test:e2e": "cross-env NODE_ENV=production yarn test:e2e:build && concurrently --success first --kill-others \"yarn test:e2e:run\" \"yarn test:e2e:server\"", + "test:e2e:build": "webpack --config test/e2e/webpack.config.js", + "test:e2e:dev": "concurrently \"yarn test:e2e:build --watch\" \"yarn test:e2e:server\"", + "test:e2e:run": "mocha --config test/e2e/.mocharc.js 'test/e2e/**/*.test.{js,ts,tsx}'", + "test:e2e:server": "serve test/e2e", "test:karma": "cross-env NODE_ENV=test karma start test/karma.conf.js", "test:karma:profile": "cross-env NODE_ENV=test karma start test/karma.conf.profile.js", "test:regressions": "cross-env NODE_ENV=production yarn test:regressions:build && concurrently --success first --kill-others \"yarn test:regressions:run\" \"yarn test:regressions:server\"", diff --git a/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.js b/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.js index 4e6a79d476a597..9583b554cfed18 100644 --- a/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.js +++ b/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.js @@ -78,7 +78,7 @@ function isNodeMatchingSelectorFocusable(node) { return true; } -export function defaultGetTabbable(root) { +function defaultGetTabbable(root) { const regularTabNodes = []; const orderedTabNodes = []; diff --git a/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.test.js b/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.test.js index 3d95a36edf4dce..1428c52beeea0a 100644 --- a/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.test.js +++ b/packages/material-ui-unstyled/src/Unstable_TrapFocus/Unstable_TrapFocus.test.js @@ -2,7 +2,7 @@ import * as React from 'react'; import * as ReactDOM from 'react-dom'; import { useFakeTimers } from 'sinon'; import { expect } from 'chai'; -import { act, createClientRender, screen, userEvent } from 'test/utils'; +import { act, createClientRender, screen } from 'test/utils'; import TrapFocus from './Unstable_TrapFocus'; import Portal from '../Portal'; @@ -93,55 +93,6 @@ describe('', () => { ); }); - it('should loop the tab key', () => { - render( - -
-
Title
- - - -
- , - ); - expect(screen.getByTestId('root')).toHaveFocus(); - - userEvent.tab(); - expect(screen.getByText('x')).toHaveFocus(); - userEvent.tab(); - expect(screen.getByText('cancel')).toHaveFocus(); - userEvent.tab(); - expect(screen.getByText('ok')).toHaveFocus(); - userEvent.tab(); - expect(screen.getByText('x')).toHaveFocus(); - - initialFocus.focus(); - expect(screen.getByTestId('root')).toHaveFocus(); - screen.getByText('x').focus(); - userEvent.tab({ shift: true }); - expect(screen.getByText('ok')).toHaveFocus(); - }); - - it('should focus on first focus element after last has received a tab click', () => { - render( - -
-
Title
- - - -
- , - ); - - userEvent.tab(); - expect(screen.getByText('x')).toHaveFocus(); - userEvent.tab(); - expect(screen.getByText('cancel')).toHaveFocus(); - userEvent.tab(); - expect(screen.getByText('ok')).toHaveFocus(); - }); - it('should focus rootRef if no tabbable children are rendered', () => { render( @@ -386,7 +337,7 @@ describe('', () => { expect(screen.getByTestId('outside-input')).toHaveFocus(); // the trap activates - userEvent.tab(); + screen.getByTestId('focus-input').focus(); expect(screen.getByTestId('focus-input')).toHaveFocus(); // the trap prevent to escape diff --git a/test/README.md b/test/README.md index 6236c4a7a449a1..bae71c0bca9a71 100644 --- a/test/README.md +++ b/test/README.md @@ -38,6 +38,7 @@ Deciding where to put a test is (like naming things) a hard problem: - If you find yourself using a lot of `data-testid` attributes or you're accessing a lot of styles consider adding a component (that doesn't require any interaction) to `test/regressions/tests/` e.g. `test/regressions/tests/List/ListWithSomeStyleProp` +- If you have to dispatch and compose many different DOM events prefer end-to-end tests (Checkout the [end-to-end testing readme](./e2e/README.md) for more information.) ### Unexpected calls to `console.error` or `console.warn` @@ -165,6 +166,10 @@ We are using [Playwright](https://playwright.dev/) to take screenshots and compa Here is an [example](https://github.com/mui-org/material-ui/blob/814fb60bbd8e500517b2307b6a297a638838ca89/test/regressions/tests/Menu/SimpleMenuList.js#L6-L16) with the `Menu` component. +#### end-to-end tests + +Checkout the [end-to-end testing readme](./e2e/README.md) for more information. + ##### Development When working on the visual regression tests you can run `yarn test:regressions:dev` in the background to constantly rebuild the views used for visual regression testing. diff --git a/test/e2e/.mocharc.js b/test/e2e/.mocharc.js new file mode 100644 index 00000000000000..e10f844d06e18a --- /dev/null +++ b/test/e2e/.mocharc.js @@ -0,0 +1,8 @@ +module.exports = { + extension: ['js', 'ts', 'tsx'], + recursive: true, + slow: 500, + timeout: (process.env.CIRCLECI === 'true' ? 4 : 2) * 1000, // Circle CI has low-performance CPUs. + reporter: 'dot', + require: [require.resolve('../utils/setupBabel')], +}; diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 00000000000000..31965a370ada1c --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,29 @@ +# end-to-end testing + +End-to-end tests (short e2e) are split into two parts: + +1. The rendered UI (short: fixture) +2. Instrumentation of that UI + +## Rendered UI + +The rendered UI is located inside a separate file in `./tests` and written as a React component. +If you're adding a new test prefer a new component instead of adding existing files since that might unknowingly alter existing tests. + +## Instrumentation + +We're using [`playwright`](https://playwright.dev) to replay user actions. +Each test tests only a single fixture. +A fixture can be loaded with `await renderFixture(fixturePath)` e.g. `renderFixture('Unstable_TrapFocus/OpenTrapFocus')`. + +## Commands + +For development `yarn test:e2e:dev` and `yarn test:e2e:run --watch` in separate terminals is recommended. + +| command | description | +| ---------------------- | --------------------------------------------------------------------------------------------- | +| `yarn test:e2e` | Full run | +| `yarn test:e2e:dev` | Prepares the fixtures to be able to test in watchmode | +| `yarn test:e2e:run` | Runs the tests (requires `yarn test:e2e:dev` or `yarn test:e2e:build`+`yarn test:e2e:server`) | +| `yarn test:e2e:build` | Builds the webpack bundle for viewing the fixtures | +| `yarn test:e2e:server` | Serves the fixture bundle. | diff --git a/test/e2e/TestViewer.js b/test/e2e/TestViewer.js new file mode 100644 index 00000000000000..5d69efec19e3d6 --- /dev/null +++ b/test/e2e/TestViewer.js @@ -0,0 +1,30 @@ +import * as React from 'react'; +import PropTypes from 'prop-types'; +import StyledEngineProvider from '@material-ui/core/StyledEngineProvider'; + +function TestViewer(props) { + const { children } = props; + + // We're simulating `act(() => ReactDOM.render(children))` + // In the end children passive effects should've been flushed. + // React doesn't have any such guarantee outside of `act()` so we're approximating it. + const [ready, setReady] = React.useState(false); + React.useEffect(() => { + setReady(true); + }, []); + + return ( + // TODO v5: remove once migration to emotion is completed + +
+ {children} +
+ + ); +} + +TestViewer.propTypes = { + children: PropTypes.node.isRequired, +}; + +export default TestViewer; diff --git a/test/e2e/index.js b/test/e2e/index.js new file mode 100644 index 00000000000000..05daf99fa6285c --- /dev/null +++ b/test/e2e/index.js @@ -0,0 +1,94 @@ +import * as React from 'react'; +import ReactDOM from 'react-dom'; +import { BrowserRouter as Router, Switch, Route, Link } from 'react-router-dom'; +import TestViewer from './TestViewer'; + +const fixtures = []; + +const requireFixtures = require.context('./tests', true, /\.(js|ts|tsx)$/); +requireFixtures.keys().forEach((path) => { + const [suite, name] = path + .replace('./', '') + .replace(/\.\w+$/, '') + .split('/'); + fixtures.push({ + path, + suite: `e2e/${suite}`, + name, + case: requireFixtures(path).default, + }); +}); + +function App() { + function computeIsDev() { + if (window.location.hash === '#dev') { + return true; + } + if (window.location.hash === '#no-dev') { + return false; + } + return process.env.NODE_ENV === 'development'; + } + const [isDev, setDev] = React.useState(computeIsDev); + React.useEffect(() => { + function handleHashChange() { + setDev(computeIsDev()); + } + window.addEventListener('hashchange', handleHashChange); + + return () => { + window.removeEventListener('hashchange', handleHashChange); + }; + }, []); + + function computePath(test) { + return `/${test.suite}/${test.name}`; + } + + return ( + + + {fixtures.map((test) => { + const path = computePath(test); + const TestCase = test.case; + if (TestCase === undefined) { + console.warn('Missing test.case for ', test); + return null; + } + + return ( + + + + + + ); + })} + + + + ); +} + +ReactDOM.render(, document.getElementById('react-root')); diff --git a/test/e2e/index.test.ts b/test/e2e/index.test.ts new file mode 100644 index 00000000000000..1b28dc7001dd38 --- /dev/null +++ b/test/e2e/index.test.ts @@ -0,0 +1,62 @@ +import { expect } from 'chai'; +import * as playwright from 'playwright'; + +describe('e2e', () => { + const baseUrl = 'http://localhost:5000'; + let browser: playwright.Browser; + let page: playwright.Page; + + async function renderFixture(fixturePath: string) { + await page.goto(`${baseUrl}/e2e/${fixturePath}#no-dev`); + } + + before(async () => { + browser = await playwright.chromium.launch({ + headless: true, + }); + page = await browser.newPage(); + await page.goto(`${baseUrl}#no-dev`); + }); + + after(async () => { + await browser.close(); + }); + + describe('', () => { + it('should loop the tab key', async () => { + await renderFixture('Unstable_TrapFocus/OpenTrapFocus'); + + expect( + await page.evaluate(() => document.activeElement?.getAttribute('data-testid')), + ).to.equal('root'); + + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('x'); + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('cancel'); + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('ok'); + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('x'); + + await page.focus('[data-testid="initial-focus"]'); + expect( + await page.evaluate(() => document.activeElement?.getAttribute('data-testid')), + ).to.equal('root'); + await page.focus('text=x'); + await page.keyboard.press('Shift+Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('ok'); + }); + + it('should focus on first focus element after last has received a tab click', async () => { + await renderFixture('Unstable_TrapFocus/OpenTrapFocus'); + + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('x'); + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('cancel'); + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('ok'); + }); + }); +}); diff --git a/test/e2e/serve.json b/test/e2e/serve.json new file mode 100644 index 00000000000000..ef9da9b562af53 --- /dev/null +++ b/test/e2e/serve.json @@ -0,0 +1,4 @@ +{ + "public": "build", + "rewrites": [{ "source": "**", "destination": "index.html" }] +} diff --git a/test/e2e/template.html b/test/e2e/template.html new file mode 100644 index 00000000000000..4bde5182495b03 --- /dev/null +++ b/test/e2e/template.html @@ -0,0 +1,16 @@ + + + + vrtest + + + + + +
+ + diff --git a/test/e2e/tests/Unstable_TrapFocus/OpenTrapFocus.tsx b/test/e2e/tests/Unstable_TrapFocus/OpenTrapFocus.tsx new file mode 100644 index 00000000000000..1a41153b39fad5 --- /dev/null +++ b/test/e2e/tests/Unstable_TrapFocus/OpenTrapFocus.tsx @@ -0,0 +1,20 @@ +import * as React from 'react'; +import TrapFocus from '@material-ui/core/Unstable_TrapFocus'; + +export default function BaseTrapFocus() { + return ( + + + document} isEnabled={() => true} open> +
+
Title
+ + + +
+ + + ); +} diff --git a/test/e2e/webpack.config.js b/test/e2e/webpack.config.js new file mode 100644 index 00000000000000..b62310ea158eef --- /dev/null +++ b/test/e2e/webpack.config.js @@ -0,0 +1,33 @@ +const HtmlWebpackPlugin = require('html-webpack-plugin'); +const path = require('path'); +const webpackBaseConfig = require('../../webpackBaseConfig'); + +module.exports = { + ...webpackBaseConfig, + entry: path.resolve(__dirname, 'index.js'), + mode: process.env.NODE_ENV || 'development', + optimization: { + // Helps debugging and build perf. + // Bundle size is irrelevant for local serving + minimize: false, + }, + output: { + path: path.resolve(__dirname, './build'), + publicPath: '/', + filename: 'tests.js', + }, + plugins: [ + new HtmlWebpackPlugin({ + template: path.resolve(__dirname, './template.html'), + }), + ], + module: { + ...webpackBaseConfig.module, + rules: webpackBaseConfig.module.rules.concat([ + { + test: /\.(jpg|gif|png)$/, + loader: 'url-loader', + }, + ]), + }, +}; diff --git a/test/tsconfig.json b/test/tsconfig.json index fb8f010733462e..798f9d363f0e6d 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "../tsconfig.json", - "include": ["regressions/**/*", "utils/**/*"], + "include": ["e2e/**/*", "regressions/**/*", "utils/**/*"], "compilerOptions": { "allowJs": true, "noEmit": true, diff --git a/test/utils/createClientRender.tsx b/test/utils/createClientRender.tsx index a6ba93fbe315df..854d775fb46932 100644 --- a/test/utils/createClientRender.tsx +++ b/test/utils/createClientRender.tsx @@ -15,7 +15,6 @@ import { RenderResult, } from '@testing-library/react/pure'; import { unstable_trace, Interaction } from 'scheduler/tracing'; -import userEvent from './user-event'; const enableDispatchingProfiler = process.env.TEST_GATE === 'enable-dispatching-profiler'; @@ -503,7 +502,7 @@ export function act(callback: () => void) { } export * from '@testing-library/react/pure'; -export { cleanup, fireEvent, userEvent }; +export { cleanup, fireEvent }; // We import from `@testing-library/react` and `@testing-library/dom` before creating a JSDOM. // At this point a global document isn't available yet. Now it is. export const screen = within(document.body); diff --git a/test/utils/user-event/index.js b/test/utils/user-event/index.js deleted file mode 100644 index 4d0f7d053a69f6..00000000000000 --- a/test/utils/user-event/index.js +++ /dev/null @@ -1,166 +0,0 @@ -import { fireEvent, getConfig } from '@testing-library/dom'; -// eslint-disable-next-line no-restricted-imports -import { defaultGetTabbable as getTabbable } from '@material-ui/unstyled/Unstable_TrapFocus/Unstable_TrapFocus'; - -// Absolutely NO events fire on label elements that contain their control -// if that control is disabled. NUTS! -// no joke. There are NO events for: