Test(demo): Introduce E2E test for homepage refs #DS-869

.github/workflows/e2e.yml

@@ -7,7 +7,9 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
+    container:
+      image: mcr.microsoft.com/playwright:v1.38.0-jammy
     timeout-minutes: 60
 
     steps:
@@ -23,11 +25,8 @@ jobs:
         with:
           useRollingCache: true
 
-      - name: Install Playwright Browsers
-        run: yarn playwright install --with-deps
-
       - name: Run Playwright tests
-        run: yarn playwright test
+        run: HOME=/root yarn playwright test
 
       - uses: actions/upload-artifact@v3
         if: always()

CONTRIBUTING.md

@@ -8,6 +8,7 @@ Here are some tips how to make your contributing efforts efficient and eventuall
 - [Commit Conventions](#commit-conventions)
 - [Code style](#code-style)
 - [Documenting the Components](#documenting-the-components)
+- [Testing](#testing)
 - [Publishing](#publishing)
 
 ## General usage
@@ -161,6 +162,43 @@ This project uses Prettier for code formatting. You can run `make format` to for
       - `Description` — the description of the prop, e.g. `Title of the accordion`
    3. The props MUST be sorted alphabetically by their name.
 
+## Testing
+
+Each package contains a script called `test`.
+Using this you can test entire package and verify that all parts of the package are in good shape and all rules are met.
+
+Testing script includes:
+
+- linting using [ESlint][eslint]
+- checking formatting using [Prettier][prettier]
+- checking types using [Typescript][typescript] compiler
+- running unit test using [Jest][jest]
+
+### Unit testing
+
+You can run unit testing via `test:unit` scripts.
+For all available scripts see the package's `package.json` file.
+
+### Visual regression testing
+
+**Prerequisites:** [Docker][docker] 🐳
+
+You can run visual regression testing via `Makefile` in the project root.
+Using `make test-e2e` you will execute docker command that starts [Playwright][playwright] in containerized environment.
+
+👉 Visual snapshots are generated based on platform, so we need to use same platform locally and on CI (GitHub Actions).
+
+⚠️ Version number of the [Playwright][playwright] dependency must be the same in `package.json` file and in the `./bin/make/e2e.sh` to ensure that no additional [Playwright][playwright] dependencies will need to install (browsers are backed in the Docker image). See https://playwright.dev/docs/docker.
+
+⚠️ Visual regression testing currently targets only deploys on Netlify.
+We are investigating how we should handle visual testing and several paths like
+
+- running tests against localhost
+- running tests against Netlify preview branches
+- …
+
+So the visual tests do not provide full coverage now.
+
 ## Publishing
 
 This project uses GitHub Actions to publish the packages automatically to npm. New packages are published after the new tag is pushed to the main branch. PR can be merged only by the appropriate group of maintainers.
@@ -180,3 +218,9 @@ This project uses GitHub Actions to publish the packages automatically to npm. N
 [commitlint-config]: https://github.com/lmc-eu/code-quality-tools/tree/main/packages/commitlint-config
 [packages]: packages/
 [dictionary]: https://github.com/lmc-eu/spirit-design-system/tree/main/docs/DICTIONARIES.md
+[eslint]: https://eslint.org/
+[prettier]: https://prettier.io/
+[typescript]: https://www.typescriptlang.org/
+[jest]: https://jestjs.io/
+[docker]: https://www.docker.com/
+[playwright]: https://playwright.dev/

Makefile

@@ -70,6 +70,9 @@ lint: ## Lints all packages
 test: ## Run tests in all packages
 	$(PKG_MANAGER) test
 
+test-e2e: ## Run End-to-End tests
+	./bin/make/e2e.sh
+
 types: ## Check types in all packages
 	$(PKG_MANAGER) types
 

bin/make/e2e.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -o errexit
+
+PLAYWRIGHT_VERSION=1.38.1
+UBUNTU_VERSION=jammy
+
+docker run --rm --network host -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v$PLAYWRIGHT_VERSION-$UBUNTU_VERSION yarn test:e2e

package.json

@@ -31,6 +31,10 @@
     "storybook": "storybook dev -p 6006",
     "storybook:build": "storybook build -o ./.storybook/build",
     "test": "yarn packages:test",
+    "test:e2e": "yarn playwright test",
+    "test:e2e:update": "yarn playwright test --update-snapshots",
+    "test:e2e:report": "yarn playwright show-report",
+    "test:e2e:ui": "yarn playwright test --ui",
     "types": "yarn packages:types",
     "lint": "npm-run-all --parallel es:lint packages:lint",
     "packages:lint": "lerna run lint --parallel --no-sort",
@@ -54,7 +58,7 @@
     "@lmc-eu/eslint-config-react": "1.1.1",
     "@lmc-eu/prettier-config": "1.2.4",
     "@omlet/cli": "1.0.1-beta.34",
-    "@playwright/test": "^1.37.1",
+    "@playwright/test": "1.38.1",
     "@rollup/plugin-babel": "6.0.3",
     "@rollup/plugin-node-resolve": "15.2.1",
     "@rollup/plugin-replace": "5.0.2",

tests/e2e/demo-homepages.spec.ts

@@ -0,0 +1,21 @@
+import { expect, test } from '@playwright/test';
+
+test.describe('Demo Homepages', () => {
+  const demos = [
+    {
+      url: 'https://spirit-design-system-demo.netlify.app/',
+      package: 'web',
+    },
+    {
+      url: 'https://spirit-design-system-react.netlify.app/',
+      package: 'web-react',
+    },
+  ];
+
+  for (const demo of demos) {
+    test(`test demo homepage ${demo.package}`, async ({ page }) => {
+      await page.goto(demo.url);
+      await expect(page).toHaveScreenshot(`${demo.package}.png`, { fullPage: true, maxDiffPixelRatio: 0.01 });
+    });
+  }
+});

yarn.lock

@@ -4438,15 +4438,12 @@
   resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
   integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
 
-"@playwright/test@^1.37.1":
-  version "1.37.1"
-  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.37.1.tgz#e7f44ae0faf1be52d6360c6bbf689fd0057d9b6f"
-  integrity sha512-bq9zTli3vWJo8S3LwB91U0qDNQDpEXnw7knhxLM0nwDvexQAwx9tO8iKDZSqqneVq+URd/WIoz+BALMqUTgdSg==
+"@playwright/test@1.38.1":
+  version "1.38.1"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.38.1.tgz#8ef4263e355cd1d8ad7905d471d268e8acb82ed6"
+  integrity sha512-NqRp8XMwj3AK+zKLbZShl0r/9wKgzqI/527bkptKXomtuo+dOjU9NdMASQ8DNC9z9zLOMbG53T4eihYr3XR+BQ==
   dependencies:
-    "@types/node" "*"
-    playwright-core "1.37.1"
-  optionalDependencies:
-    fsevents "2.3.2"
+    playwright "1.38.1"
 
 "@pnpm/network.ca-file@^1.0.1":
   version "1.0.1"
@@ -19389,10 +19386,19 @@ pkg-dir@^7.0.0:
   dependencies:
     find-up "^6.3.0"
 
-[email protected]:
-  version "1.37.1"
-  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.37.1.tgz#cb517d52e2e8cb4fa71957639f1cd105d1683126"
-  integrity sha512-17EuQxlSIYCmEMwzMqusJ2ztDgJePjrbttaefgdsiqeLWidjYz9BxXaTaZWxH1J95SHGk6tjE+dwgWILJoUZfA==
+[email protected]:
+  version "1.38.1"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.38.1.tgz#75a3c470aa9576b7d7c4e274de3d79977448ba08"
+  integrity sha512-tQqNFUKa3OfMf4b2jQ7aGLB8o9bS3bOY0yMEtldtC2+spf8QXG9zvXLTXUeRsoNuxEYMgLYR+NXfAa1rjKRcrg==
+
+[email protected]:
+  version "1.38.1"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.38.1.tgz#82ecd9bc4f4f64dbeee8a11c31793748e2528130"
+  integrity sha512-oRMSJmZrOu1FP5iu3UrCx8JEFRIMxLDM0c/3o4bpzU5Tz97BypefWf7TuTNPWeCe279TPal5RtPPZ+9lW/Qkow==
+  dependencies:
+    playwright-core "1.38.1"
+  optionalDependencies:
+    fsevents "2.3.2"
 
 polished@^4.2.2:
   version "4.2.2"