Skip to content

Commit

Permalink
[test] Create end-to-end testing CI job (mui#25405)
Browse files Browse the repository at this point in the history
  • Loading branch information
eps1lon authored Mar 22, 2021
1 parent 3bce6fa commit 33cb431
Show file tree
Hide file tree
Showing 18 changed files with 327 additions and 293 deletions.
16 changes: 16 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,19 @@ jobs:
# hardcoded in karma-webpack
path: /tmp/_karma_webpack_
destination: artifact-file
test_e2e:
<<: *defaults
docker:
- image: mcr.microsoft.com/playwright@sha256:1700531ce01a3d974cc440bb8efcf43d31d58ee5f1d354fc21563ea5fe4291e6
environment:
NODE_ENV: development # Needed if playwright is in `devDependencies`
steps:
- checkout
- install_js:
browsers: true
- run:
name: yarn test:e2e
command: yarn test:e2e
- run:
name: Can we generate the @material-ui/core umd build?
command: yarn workspace @material-ui/core build:umd
Expand Down Expand Up @@ -410,6 +423,9 @@ workflows:
- test_regressions:
requires:
- checkout
- test_e2e:
requires:
- checkout
profile:
when:
equal: [profile, << pipeline.parameters.workflow >>]
Expand Down
5 changes: 5 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@
"test:coverage": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc --reporter=text mocha 'packages/**/*.test.{js,ts,tsx}' 'docs/**/*.test.{js,ts,tsx}' 'scripts/**/*.test.{js,ts,tsx}' 'test/utils/**/*.test.{js,ts,tsx}'",
"test:coverage:ci": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc --reporter=lcov mocha 'packages/**/*.test.{js,ts,tsx}' 'docs/**/*.test.{js,ts,tsx}' 'scripts/**/*.test.{js,ts,tsx}' 'test/utils/**/*.test.{js,ts,tsx}'",
"test:coverage:html": "cross-env NODE_ENV=test BABEL_ENV=coverage nyc --reporter=html mocha 'packages/**/*.test.{js,ts,tsx}' 'docs/**/*.test.{js,ts,tsx}' 'scripts/**/*.test.{js,ts,tsx}' 'test/utils/**/*.test.{js,ts,tsx}'",
"test:e2e": "cross-env NODE_ENV=production yarn test:e2e:build && concurrently --success first --kill-others \"yarn test:e2e:run\" \"yarn test:e2e:server\"",
"test:e2e:build": "webpack --config test/e2e/webpack.config.js",
"test:e2e:dev": "concurrently \"yarn test:e2e:build --watch\" \"yarn test:e2e:server\"",
"test:e2e:run": "mocha --config test/e2e/.mocharc.js 'test/e2e/**/*.test.{js,ts,tsx}'",
"test:e2e:server": "serve test/e2e",
"test:karma": "cross-env NODE_ENV=test karma start test/karma.conf.js",
"test:karma:profile": "cross-env NODE_ENV=test karma start test/karma.conf.profile.js",
"test:regressions": "cross-env NODE_ENV=production yarn test:regressions:build && concurrently --success first --kill-others \"yarn test:regressions:run\" \"yarn test:regressions:server\"",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ function isNodeMatchingSelectorFocusable(node) {
return true;
}

export function defaultGetTabbable(root) {
function defaultGetTabbable(root) {
const regularTabNodes = [];
const orderedTabNodes = [];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as React from 'react';
import * as ReactDOM from 'react-dom';
import { useFakeTimers } from 'sinon';
import { expect } from 'chai';
import { act, createClientRender, screen, userEvent } from 'test/utils';
import { act, createClientRender, screen } from 'test/utils';
import TrapFocus from './Unstable_TrapFocus';
import Portal from '../Portal';

Expand Down Expand Up @@ -93,55 +93,6 @@ describe('<TrapFocus />', () => {
);
});

it('should loop the tab key', () => {
render(
<TrapFocus {...defaultProps} open>
<div tabIndex={-1} data-testid="root">
<div>Title</div>
<button type="button">x</button>
<button type="button">cancel</button>
<button type="button">ok</button>
</div>
</TrapFocus>,
);
expect(screen.getByTestId('root')).toHaveFocus();

userEvent.tab();
expect(screen.getByText('x')).toHaveFocus();
userEvent.tab();
expect(screen.getByText('cancel')).toHaveFocus();
userEvent.tab();
expect(screen.getByText('ok')).toHaveFocus();
userEvent.tab();
expect(screen.getByText('x')).toHaveFocus();

initialFocus.focus();
expect(screen.getByTestId('root')).toHaveFocus();
screen.getByText('x').focus();
userEvent.tab({ shift: true });
expect(screen.getByText('ok')).toHaveFocus();
});

it('should focus on first focus element after last has received a tab click', () => {
render(
<TrapFocus {...defaultProps} open>
<div tabIndex={-1} data-testid="root">
<div>Title</div>
<button type="button">x</button>
<button type="button">cancel</button>
<button type="button">ok</button>
</div>
</TrapFocus>,
);

userEvent.tab();
expect(screen.getByText('x')).toHaveFocus();
userEvent.tab();
expect(screen.getByText('cancel')).toHaveFocus();
userEvent.tab();
expect(screen.getByText('ok')).toHaveFocus();
});

it('should focus rootRef if no tabbable children are rendered', () => {
render(
<TrapFocus {...defaultProps} open>
Expand Down Expand Up @@ -386,7 +337,7 @@ describe('<TrapFocus />', () => {
expect(screen.getByTestId('outside-input')).toHaveFocus();

// the trap activates
userEvent.tab();
screen.getByTestId('focus-input').focus();
expect(screen.getByTestId('focus-input')).toHaveFocus();

// the trap prevent to escape
Expand Down
5 changes: 5 additions & 0 deletions test/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ Deciding where to put a test is (like naming things) a hard problem:
- If you find yourself using a lot of `data-testid` attributes or you're accessing
a lot of styles consider adding a component (that doesn't require any interaction)
to `test/regressions/tests/` e.g. `test/regressions/tests/List/ListWithSomeStyleProp`
- If you have to dispatch and compose many different DOM events prefer end-to-end tests (Checkout the [end-to-end testing readme](./e2e/README.md) for more information.)

### Unexpected calls to `console.error` or `console.warn`

Expand Down Expand Up @@ -165,6 +166,10 @@ We are using [Playwright](https://playwright.dev/) to take screenshots and compa

Here is an [example](https://github.com/mui-org/material-ui/blob/814fb60bbd8e500517b2307b6a297a638838ca89/test/regressions/tests/Menu/SimpleMenuList.js#L6-L16) with the `Menu` component.

#### end-to-end tests

Checkout the [end-to-end testing readme](./e2e/README.md) for more information.

##### Development

When working on the visual regression tests you can run `yarn test:regressions:dev` in the background to constantly rebuild the views used for visual regression testing.
Expand Down
8 changes: 8 additions & 0 deletions test/e2e/.mocharc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
module.exports = {
extension: ['js', 'ts', 'tsx'],
recursive: true,
slow: 500,
timeout: (process.env.CIRCLECI === 'true' ? 4 : 2) * 1000, // Circle CI has low-performance CPUs.
reporter: 'dot',
require: [require.resolve('../utils/setupBabel')],
};
29 changes: 29 additions & 0 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# end-to-end testing

End-to-end tests (short <abbr title="end-to-end">e2e</abbr>) are split into two parts:

1. The rendered UI (short: fixture)
2. Instrumentation of that UI

## Rendered UI

The rendered UI is located inside a separate file in `./tests` and written as a React component.
If you're adding a new test prefer a new component instead of adding existing files since that might unknowingly alter existing tests.

## Instrumentation

We're using [`playwright`](https://playwright.dev) to replay user actions.
Each test tests only a single fixture.
A fixture can be loaded with `await renderFixture(fixturePath)` e.g. `renderFixture('Unstable_TrapFocus/OpenTrapFocus')`.

## Commands

For development `yarn test:e2e:dev` and `yarn test:e2e:run --watch` in separate terminals is recommended.

| command | description |
| ---------------------- | --------------------------------------------------------------------------------------------- |
| `yarn test:e2e` | Full run |
| `yarn test:e2e:dev` | Prepares the fixtures to be able to test in watchmode |
| `yarn test:e2e:run` | Runs the tests (requires `yarn test:e2e:dev` or `yarn test:e2e:build`+`yarn test:e2e:server`) |
| `yarn test:e2e:build` | Builds the webpack bundle for viewing the fixtures |
| `yarn test:e2e:server` | Serves the fixture bundle. |
30 changes: 30 additions & 0 deletions test/e2e/TestViewer.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import * as React from 'react';
import PropTypes from 'prop-types';
import StyledEngineProvider from '@material-ui/core/StyledEngineProvider';

function TestViewer(props) {
const { children } = props;

// We're simulating `act(() => ReactDOM.render(children))`
// In the end children passive effects should've been flushed.
// React doesn't have any such guarantee outside of `act()` so we're approximating it.
const [ready, setReady] = React.useState(false);
React.useEffect(() => {
setReady(true);
}, []);

return (
// TODO v5: remove once migration to emotion is completed
<StyledEngineProvider injectFirst>
<div aria-busy={!ready} data-testid="testcase">
{children}
</div>
</StyledEngineProvider>
);
}

TestViewer.propTypes = {
children: PropTypes.node.isRequired,
};

export default TestViewer;
94 changes: 94 additions & 0 deletions test/e2e/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
import * as React from 'react';
import ReactDOM from 'react-dom';
import { BrowserRouter as Router, Switch, Route, Link } from 'react-router-dom';
import TestViewer from './TestViewer';

const fixtures = [];

const requireFixtures = require.context('./tests', true, /\.(js|ts|tsx)$/);
requireFixtures.keys().forEach((path) => {
const [suite, name] = path
.replace('./', '')
.replace(/\.\w+$/, '')
.split('/');
fixtures.push({
path,
suite: `e2e/${suite}`,
name,
case: requireFixtures(path).default,
});
});

function App() {
function computeIsDev() {
if (window.location.hash === '#dev') {
return true;
}
if (window.location.hash === '#no-dev') {
return false;
}
return process.env.NODE_ENV === 'development';
}
const [isDev, setDev] = React.useState(computeIsDev);
React.useEffect(() => {
function handleHashChange() {
setDev(computeIsDev());
}
window.addEventListener('hashchange', handleHashChange);

return () => {
window.removeEventListener('hashchange', handleHashChange);
};
}, []);

function computePath(test) {
return `/${test.suite}/${test.name}`;
}

return (
<Router>
<Switch>
{fixtures.map((test) => {
const path = computePath(test);
const TestCase = test.case;
if (TestCase === undefined) {
console.warn('Missing test.case for ', test);
return null;
}

return (
<Route key={path} exact path={path}>
<TestViewer>
<TestCase />
</TestViewer>
</Route>
);
})}
</Switch>
<div hidden={!isDev}>
<p>
Devtools can be enabled by appending <code>#dev</code> in the addressbar or disabled by
appending <code>#no-dev</code>.
</p>
<a href="#no-dev">Hide devtools</a>
<details>
<summary id="my-test-summary">nav for all tests</summary>
<nav id="tests">
<ol>
{fixtures.map((test) => {
const path = computePath(test);
return (
<li key={path}>
<Link to={path}>{path}</Link>
</li>
);
})}
</ol>
</nav>
</details>
</div>
</Router>
);
}

ReactDOM.render(<App />, document.getElementById('react-root'));
62 changes: 62 additions & 0 deletions test/e2e/index.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import { expect } from 'chai';
import * as playwright from 'playwright';

describe('e2e', () => {
const baseUrl = 'http://localhost:5000';
let browser: playwright.Browser;
let page: playwright.Page;

async function renderFixture(fixturePath: string) {
await page.goto(`${baseUrl}/e2e/${fixturePath}#no-dev`);
}

before(async () => {
browser = await playwright.chromium.launch({
headless: true,
});
page = await browser.newPage();
await page.goto(`${baseUrl}#no-dev`);
});

after(async () => {
await browser.close();
});

describe('<TrapFocus />', () => {
it('should loop the tab key', async () => {
await renderFixture('Unstable_TrapFocus/OpenTrapFocus');

expect(
await page.evaluate(() => document.activeElement?.getAttribute('data-testid')),
).to.equal('root');

await page.keyboard.press('Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('x');
await page.keyboard.press('Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('cancel');
await page.keyboard.press('Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('ok');
await page.keyboard.press('Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('x');

await page.focus('[data-testid="initial-focus"]');
expect(
await page.evaluate(() => document.activeElement?.getAttribute('data-testid')),
).to.equal('root');
await page.focus('text=x');
await page.keyboard.press('Shift+Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('ok');
});

it('should focus on first focus element after last has received a tab click', async () => {
await renderFixture('Unstable_TrapFocus/OpenTrapFocus');

await page.keyboard.press('Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('x');
await page.keyboard.press('Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('cancel');
await page.keyboard.press('Tab');
expect(await page.evaluate(() => document.activeElement?.textContent)).to.equal('ok');
});
});
});
4 changes: 4 additions & 0 deletions test/e2e/serve.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"public": "build",
"rewrites": [{ "source": "**", "destination": "index.html" }]
}
16 changes: 16 additions & 0 deletions test/e2e/template.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<title>vrtest</title>
<meta charset="utf-8" />
<meta name="viewport" content="minimum-scale=1, initial-scale=1, width=device-width" />
<style>
body {
background-color: white;
}
</style>
</head>
<body>
<div id="react-root"></div>
</body>
</html>
Loading

0 comments on commit 33cb431

Please sign in to comment.