Test(web,web-react,web-twig): Introduce visual tests of all component…

…s #DS-660

CONTRIBUTING.md

@@ -185,27 +185,32 @@ For all available scripts see the package's `package.json` file.
 **Prerequisites:** [Docker][docker] 🐳
 
 You can run visual regression testing via `Makefile` in the project root.
-Using `make test-e2e` you will execute docker command that starts [Playwright][playwright] in containerized environment.
+All commands will execute Docker command that starts [Playwright][playwright] in containerized environment.
+
+- Use `make test-e2e` to run the tests.
+- Use `make test-e2e-update` to update the snapshots.
+- Use `make test-e2e-report` to generate and serve report of visual regression testing. Find report URL in the terminal output.
 
 👉 Visual snapshots are generated based on platform, so we need to use same platform locally and on CI (GitHub Actions).
 
-⚠️ Version number of the [Playwright][playwright] dependency must be the same in `package.json` file and in the `./bin/make/e2e.sh` to ensure that no additional [Playwright][playwright] dependencies will need to install (browsers are backed in the Docker image). See https://playwright.dev/docs/docker.
+⚠️ Version number of the Playwright dependency must be the same in `package.json` file and in the `./bin/make/e2e.sh` to ensure that no additional Playwright dependencies will need to install (browsers are backed in the Docker image). See https://playwright.dev/docs/docker.
+
+We run visual regression testing locally against our demo apps. Web and Web React packages are served using Vite.
+Web Twig package is served using Symfony app.
 
-⚠️ Visual regression testing currently targets only deploys on Netlify.
-We are investigating how we should handle visual testing and several paths like
+In CI we use Netlify to test against.
 
-- running tests against localhost
-- running tests against Netlify preview branches
-- …
+⚠️ Currently we do not deploy the Web Twig package to any environment, so you can only test it locally.
 
-So the visual tests do not provide full coverage now.
+We have two test suites and you can find them in the `./tests/e2e` directory:
 
-### Speed up CI performance
+- `demo-homepages` - tests the homepages of our demo apps.
+  - This test is used to verify that the demo apps are working properly and their homepages are not broken.
+- `demo-components-compare` - tests components' pages of our demo apps'.
+  - This test gets the list of components from file system for each package and then it goes through each component and compares its page in all demo apps.
+  - Only one screenshot is taken for each component. If you run the update command, only the last screenshot will be saved.
 
-This project uses [Nx Cloud](https://nx.app/) for speeding up CI runs by caching its results.
-You can also benefit from this feature locally by adding `nx-cloud.env` to root of this project with `NX_CLOUD_ACCESS_TOKEN=<token>`.
-Read-write token can be generated inside the Nx Cloud App.
-Otherwise only read access is used so you cannot [upload your local cache](https://lerna.js.org/docs/features/share-your-cache).
+👉 To save time and repository size, we test only in Chromium browser and only on desktop viewport.
 
 ## Publishing
 

playwright.config.ts

@@ -30,48 +30,20 @@ export default defineConfig({
     trace: 'on-first-retry',
   },
 
+  /* Timeout configuration in ms */
+  timeout: 120000,
+
+  /* Configure expect() to use in tests */
+  expect: {
+    /* Setting the timeout for each test */
+    timeout: 120000,
+  },
+
   /* Configure projects for major browsers */
   projects: [
     {
       name: 'chromium',
-      use: { ...devices['Desktop Chrome'] },
-    },
-
-    {
-      name: 'firefox',
-      use: { ...devices['Desktop Firefox'] },
+      use: { ...devices['Desktop Chrome'], ignoreHTTPSErrors: true },
     },
-
-    {
-      name: 'webkit',
-      use: { ...devices['Desktop Safari'] },
-    },
-
-    /* Test against mobile viewports. */
-    // {
-    //   name: 'Mobile Chrome',
-    //   use: { ...devices['Pixel 5'] },
-    // },
-    // {
-    //   name: 'Mobile Safari',
-    //   use: { ...devices['iPhone 12'] },
-    // },
-
-    /* Test against branded browsers. */
-    // {
-    //   name: 'Microsoft Edge',
-    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
-    // },
-    // {
-    //   name: 'Google Chrome',
-    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
-    // },
   ],
-
-  /* Run your local dev server before starting the tests */
-  // webServer: {
-  //   command: 'npm run start',
-  //   url: 'http://127.0.0.1:3000',
-  //   reuseExistingServer: !process.env.CI,
-  // },
 });

tests/e2e/demo-components-compare.spec.ts

@@ -0,0 +1,73 @@
+import { isTesting as isTestingEnvironment } from '@lmc-eu/spirit-common/constants/environments';
+import { SERVERS, getDevelopmentEndpointUri } from '@lmc-eu/spirit-common/constants/servers';
+import { readdirSync } from 'fs';
+import { expect, test } from '@playwright/test';
+
+// Tests that are intentionally broken, but will be fixed in the future
+const IGNORED_TESTS = [
+  'Tooltip', // Should be fixed when DS-1087 and DS-1088 are merged
+];
+
+const runComponentCompareTests = (testConfig) => {
+  const { packageDir, componentsDir, srcDir = '', ignoredComponents = [], packageName } = testConfig;
+  if (packageName) {
+    const formattedPackageName = packageName
+      .split('-')
+      .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+      .join(' ');
+
+    test.describe(`Demo ${formattedPackageName} Components`, () => {
+      const componentDirs = readdirSync(`${packageDir}${srcDir}${componentsDir}`, { withFileTypes: true })
+        .filter((item) => item.isDirectory())
+        .filter((item) => !ignoredComponents.includes(item.name))
+        .filter((item) => !IGNORED_TESTS.includes(item.name))
+        .map((item) => item.name.toLowerCase());
+
+      for (const component of componentDirs) {
+        test(`test demo ${formattedPackageName} component ${component}`, async ({ page }) => {
+          await page.goto(
+            `${
+              isTestingEnvironment()
+                ? SERVERS.TESTING[packageName]
+                : getDevelopmentEndpointUri(packageName, { isDocker: packageName !== 'web-twig' })
+            }${componentsDir}/${component}/${packageName === 'web-twig' ? '?HIDE_TOOLBAR' : ''}`,
+          );
+          // wait for fonts to load
+          await page.evaluate(() => document.fonts.ready);
+          // wait for transitions to finish
+          await page.waitForLoadState();
+          // disable animations to avoid flaky screenshots
+          await page.addStyleTag({ content: '*, *::before, *::after { animation-iteration-count: 1 !important }' });
+          await expect(page).toHaveScreenshot(`${component}.png`, {
+            animations: 'disabled',
+            fullPage: true,
+          });
+        });
+      }
+    });
+  }
+};
+
+const testConfigs = [
+  {
+    componentsDir: '/src/scss/components',
+    packageDir: 'packages/web',
+    packageName: 'web',
+  },
+  {
+    componentsDir: '/src/components',
+    ignoredComponents: ['Field', 'Dialog', 'Icon', 'TextFieldBase', 'VisuallyHidden'],
+    packageDir: 'packages/web-react',
+    packageName: 'web-react',
+  },
+  // Disable web-twig tests for now on CI, because we don't have a way to run them in CI yet.
+  !isTestingEnvironment() && {
+    componentsDir: '/components',
+    ignoredComponents: ['Field', 'Icon', 'TextFieldBase', 'VisuallyHidden'],
+    packageDir: 'packages/web-twig',
+    packageName: 'web-twig',
+    srcDir: '/src/Resources',
+  },
+];
+
+testConfigs.forEach(runComponentCompareTests);