[test] Add E2E website tests (mui#30128)

.circleci/config.yml

@@ -15,6 +15,10 @@ parameters:
     description: The name of the workflow to run
     type: string
     default: pipeline
+  e2e-base-url:
+    description: The base url for running end-to-end test
+    type: string
+    default: ''
 
 defaults: &defaults
   parameters:
@@ -26,6 +30,10 @@ defaults: &defaults
       description: A particular type of tests that should be run
       type: string
       default: undefined
+    e2e-base-url:
+      description: The base url for running end-to-end test
+      type: string
+      default: << pipeline.parameters.e2e-base-url >>
   environment:
     # Keep in sync with "Save playwright cache"
     PLAYWRIGHT_BROWSERS_PATH: /tmp/pw-browsers
@@ -373,6 +381,21 @@ jobs:
       - run:
           name: Test umd release
           command: yarn test:umd
+  test_e2e_website:
+    <<: *defaults
+    docker:
+      - image: mcr.microsoft.com/playwright@sha256:f08e263c95e83334104e6e2fee047ad92062a03af6ae94c0f8686ba2b3014823
+        environment:
+          NODE_ENV: development # Needed if playwright is in `devDependencies`
+    steps:
+      - checkout
+      - install_js:
+          browsers: true
+      - run:
+          name: yarn test:e2e-website
+          command: yarn test:e2e-website
+          environment:
+            PLAYWRIGHT_TEST_BASE_URL: << parameters.e2e-base-url >>
   test_profile:
     <<: *defaults
     docker:
@@ -750,6 +773,15 @@ workflows:
       - test_bundle_size_monitor:
           requires:
             - checkout
+  e2e-website:
+    when:
+      equal: [e2e-website, << pipeline.parameters.workflow >>]
+    jobs:
+      - checkout
+      - test_e2e_website:
+          requires:
+            - checkout
+
   bundling:
     when:
       equal: [bundling, << pipeline.parameters.workflow >>]

docs/src/featureToggle.ts

@@ -1,5 +1,6 @@
 const FEATURE_TOGGLE = {
   nav_products: true,
+  enable_product_scope: false,
   enable_website_banner: false,
 };
 

netlify/functions/deploy-succeeded.js

@@ -0,0 +1,43 @@
+const fetch = require('node-fetch');
+
+/**
+ * @param {object} event
+ * @param {string} event.body - https://jsoneditoronline.org/#left=cloud.fb1a4fa30a4f475fa6887071c682e2c1
+ */
+exports.handler = async (event) => {
+  const { payload } = JSON.parse(event.body);
+  const repo = payload.review_url.match(/github\.com\/(.*)\/pull\/(.*)/);
+  if (!repo) {
+    throw new Error(`No repo found at review_url: ${payload.review_url}`);
+  }
+
+  // eslint-disable-next-line no-console
+  console.info(`repo:`, repo[1]);
+  // eslint-disable-next-line no-console
+  console.info(`PR:`, repo[2]);
+  // eslint-disable-next-line no-console
+  console.info(`url:`, payload.deploy_ssl_url);
+
+  // for more details > https://circleci.com/docs/2.0/api-developers-guide/#
+  await fetch(`https://circleci.com/api/v2/project/gh/${repo[1]}/pipeline`, {
+    method: 'POST',
+    headers: {
+      'Content-type': 'application/json',
+      // token from https://app.netlify.com/sites/material-ui/settings/deploys#environment-variables
+      'Circle-Token': process.env.CIRCLE_CI_TOKEN,
+    },
+    body: JSON.stringify({
+      // For PR, /head is needed. https://support.circleci.com/hc/en-us/articles/360049841151
+      branch: `pull/${repo[2]}/head`,
+      parameters: {
+        // the parameters defined in .circleci/config.yml
+        workflow: 'e2e-website', // name of the workflow
+        'e2e-base-url': payload.deploy_ssl_url, // deploy preview url
+      },
+    }),
+  });
+  return {
+    statusCode: 200,
+    body: {},
+  };
+};

package.json

@@ -50,6 +50,8 @@
     "test:e2e:dev": "concurrently \"yarn test:e2e:build --watch\" \"yarn test:e2e:server\"",
     "test:e2e:run": "mocha --config test/e2e/.mocharc.js 'test/e2e/**/*.test.{js,ts,tsx}'",
     "test:e2e:server": "serve test/e2e",
+    "test:e2e-website": "playwright test test/e2e-website --config test/e2e-website/playwright.config.ts",
+    "test:e2e-website:dev": "PLAYWRIGHT_TEST_BASE_URL=http://localhost:3000 playwright test test/e2e-website --config test/e2e-website/playwright.config.ts",
     "test:karma": "cross-env NODE_ENV=test karma start test/karma.conf.js",
     "test:karma:profile": "cross-env NODE_ENV=test karma start test/karma.conf.profile.js",
     "test:regressions": "cross-env NODE_ENV=production yarn test:regressions:build && concurrently --success first --kill-others \"yarn test:regressions:run\" \"yarn test:regressions:server\"",
@@ -79,6 +81,7 @@
     "@emotion/styled": "^11.6.0",
     "@eps1lon/enzyme-adapter-react-17": "^0.1.0",
     "@octokit/rest": "^18.12.0",
+    "@playwright/test": "1.17.1",
     "@rollup/plugin-replace": "^3.0.0",
     "@testing-library/dom": "^8.11.1",
     "@testing-library/react": "^12.1.2",

test/e2e-website/README.md

@@ -0,0 +1,12 @@
+# Docs end-to-end testing
+
+## Running locally
+
+1. Run `yarn docs:dev` to start docs in development server.
+2. Run `yarn test:e2e-website` in a separate terminal to run the test suites (`*.spec.ts`) inside `test/e2e-website` folder.
+
+> use --headed to run tests in headed browsers, check out [Playwright CLI](https://playwright.dev/docs/intro#command-line) for more options
+
+## CI
+
+After netlify deploy the preview site, the `netlify/functions/deploy-succeeded.js` hook calls CircleCI API to run the `e2e-website` workflow against the deployed url.

test/e2e-website/material-docs.spec.ts

@@ -0,0 +1,134 @@
+import { test as base, expect } from '@playwright/test';
+import kebabCase from 'lodash/kebabCase';
+import FEATURE_TOGGLE from 'docs/src/featureToggle';
+import { TestFixture } from './playwright.config';
+
+const test = base.extend<TestFixture>({});
+
+test.beforeEach(async ({ materialUrlPrefix }) => {
+  test.skip(
+    !!materialUrlPrefix && !FEATURE_TOGGLE.enable_product_scope,
+    "Migration haven't started yet",
+  );
+});
+
+test.describe.parallel('Material docs', () => {
+  test('should have correct link with hash in the TOC', async ({ page, materialUrlPrefix }) => {
+    await page.goto(`${materialUrlPrefix}/getting-started/installation/`);
+
+    const anchors = page.locator('[aria-label="Page table of contents"] ul a');
+
+    const anchorTexts = await anchors.allTextContents();
+
+    await Promise.all(
+      anchorTexts.map((text, index) => {
+        return expect(anchors.nth(index)).toHaveAttribute(
+          'href',
+          `${materialUrlPrefix}/getting-started/installation/#${kebabCase(text)}`,
+        );
+      }),
+    );
+  });
+
+  test.describe.parallel('Demo page', () => {
+    test('should have correct link for API section', async ({ page, materialUrlPrefix }) => {
+      await page.goto(`${materialUrlPrefix}/components/cards/`);
+
+      const anchors = await page.locator('div > h2#heading-api ~ ul a');
+
+      const anchorTexts = await anchors.allTextContents();
+
+      await Promise.all(
+        anchorTexts.map((text, index) => {
+          return expect(anchors.nth(index)).toHaveAttribute(
+            'href',
+            `${materialUrlPrefix}/api/${kebabCase(text)}/`,
+          );
+        }),
+      );
+    });
+
+    test('should have correct link for sidebar anchor', async ({ page, materialUrlPrefix }) => {
+      await page.goto(`${materialUrlPrefix}/components/cards/`);
+
+      const anchor = await page.locator('nav[aria-label="documentation"] ul a:text-is("Card")');
+
+      await expect(anchor).toHaveAttribute('href', `${materialUrlPrefix}/components/cards/`);
+    });
+  });
+
+  test.describe.parallel('API page', () => {
+    test('should have correct link for sidebar anchor', async ({ page, materialUrlPrefix }) => {
+      await page.goto(`${materialUrlPrefix}/api/card/`);
+
+      const anchor = await page.locator('nav[aria-label="documentation"] ul a:text-is("Card")');
+
+      await expect(anchor).toHaveAttribute('app-drawer-active', '');
+      await expect(anchor).toHaveAttribute('href', `${materialUrlPrefix}/api/card/`);
+    });
+
+    test('all the links in the main content should have correct prefix', async ({
+      page,
+      materialUrlPrefix,
+    }) => {
+      await page.goto(`${materialUrlPrefix}/api/card/`);
+
+      const anchors = await page.locator('div#main-content a');
+
+      const handles = await anchors.elementHandles();
+
+      const links = await Promise.all(handles.map((elm) => elm.getAttribute('href')));
+
+      links.forEach((link) => {
+        if (
+          [
+            '/getting-started',
+            '/components',
+            '/api',
+            '/customization',
+            '/guides',
+            '/discover-more',
+          ].some((path) => link.replace(materialUrlPrefix, '').startsWith(path))
+        ) {
+          expect(link.startsWith(materialUrlPrefix)).toBeTruthy();
+        }
+
+        if (link.replace(materialUrlPrefix, '').startsWith('/system')) {
+          expect(link.startsWith('/system')).toBeTruthy();
+          expect(link.match(/\/system{1}/g)).toHaveLength(1); // should not have repeated `/system/system/*`
+        }
+
+        if (link.replace(materialUrlPrefix, '').startsWith('/styles')) {
+          expect(link.startsWith('/styles')).toBeTruthy();
+          expect(link.match(/\/styles{1}/g)).toHaveLength(1); // should not have repeated `/system/system/*`
+        }
+      });
+    });
+  });
+
+  test.describe.parallel('Search', () => {
+    test('should have correct link when searching component', async ({ page }) => {
+      await page.goto('/getting-started/installation/');
+
+      await page.keyboard.press('Meta+k');
+
+      await page.type('input#docsearch-input', 'card', { delay: 50 });
+
+      const anchor = await page.locator('.DocSearch-Hits a:has-text("Card")');
+
+      await expect(anchor.first()).toHaveAttribute('href', '/components/cards/#main-content');
+    });
+
+    test('should have correct link when searching API', async ({ page }) => {
+      await page.goto('/getting-started/installation/');
+
+      await page.keyboard.press('Meta+k');
+
+      await page.type('input#docsearch-input', 'card api', { delay: 50 });
+
+      const anchor = await page.locator('.DocSearch-Hits a:has-text("Card API")');
+
+      await expect(anchor.first()).toHaveAttribute('href', '/api/card/#main-content');
+    });
+  });
+});

test/e2e-website/playwright.config.ts

@@ -0,0 +1,25 @@
+import { PlaywrightTestConfig } from '@playwright/test';
+
+export type TestFixture = { materialUrlPrefix: string };
+
+const config: PlaywrightTestConfig<TestFixture> = {
+  use: {
+    baseURL: process.env.PLAYWRIGHT_TEST_BASE_URL || 'https://mui.com',
+  },
+  projects: [
+    {
+      name: 'current',
+      use: {
+        materialUrlPrefix: '',
+      },
+    },
+    {
+      name: 'new',
+      use: {
+        materialUrlPrefix: '/material',
+      },
+    },
+  ],
+};
+
+export default config;