Playwright e2e reconfiguration

.github/workflows/client.yml

@@ -8,13 +8,14 @@ on:
     paths:
       - 'client/**'
   workflow_dispatch:
-    paths:
-      - 'client/**'
 
 jobs:
   client:
     name: Test react website
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: client/
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -26,29 +27,49 @@ jobs:
           cache: "yarn"
           cache-dependency-path: "**/yarn.lock"
 
+      - name: Install dependencies and setup env
+        run: |
+          yarn config:test
+          yarn install
+          yarn playwright install --with-deps
+
       - name: Run the linting checks
         run: |
-          cd client/	
-          yarn install	
           yarn syntax
+
       - name: Build the React SPA website
         run: |
-          cd client/	
-          yarn install	
           yarn build
+
       - name: Run client unit and integration tests
         run: |
-          cd client/	
-          yarn install	
-          yarn config:test
           yarn test:int
           yarn test:unit
 
       - name: Upload unit and integration test coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
           files: client/coverage/clover.xml
-          flags: client-unit-integration-tests
+          flags: client-unit-integration-tests 
+
+      - name: Run e2e tests
+        continue-on-error: true
+        run: |
+          yarn test:e2e
+
+      - name: Upload Playwright Report
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: client/playwright-report/**
+          retention-days: 30
+
+      - name: Upload e2e test coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          files: client/coverage/e2e/codecov.json
+          flags: client-e2e-tests
 
   get_version:
     name: Get version

.gitignore

@@ -121,7 +121,7 @@ build/
 # Test reports
 coverage/
 playwright-report/
-playwright/
+client/playwright/
 test_results/
 
 # Mac temp files

client/.eslintignore

@@ -2,4 +2,5 @@ api/
 build/
 config/
 node_modules/
-script/
+script/
+coverage/

client/.gitignore

@@ -1 +1,2 @@
-playwright/.auth/
+playwright/.auth/
+test-results/

client/package.json

@@ -28,7 +28,8 @@
     "stop": "npx kill-port 4000",
     "syntax": "npx eslint . --fix",
     "test:all": "yarn test:unit && yarn test:int && yarn test:e2e",
-    "test:e2e": "yarn build && yarn config:test && npx kill-port 4000 && yarn start >/dev/null & playwright test && npx kill-port 4000",
+    "test:e2e:int": "cross-env int=true yarn test:e2e",
+    "test:e2e": "playwright test -c ./playwright.config.ts",
     "test:int": "jest -c ./jest.config.json ../test/integration --setupFilesAfterEnv ./test/integration/jest.setup.ts",
     "test:unit": "jest -c ./jest.config.json ../test/unit --setupFilesAfterEnv ./test/unit/jest.setup.ts"
   },
@@ -50,6 +51,7 @@
     "@types/styled-components": "^5.1.32",
     "@typescript-eslint/eslint-plugin": "^6.12.0",
     "@typescript-eslint/parser": "^6.12.0",
+    "cross-env": "^7.0.3",
     "dotenv": "^16.1.4",
     "eslint": "8.54.0",
     "eslint-config-airbnb-base": "^15.0.0",
@@ -89,6 +91,7 @@
     "@types/react-dom": "^18.2.17",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
+    "monocart-coverage-reports": "^2.10.2",
     "playwright": "^1.32.1",
     "prettier": "^3.2.4",
     "shx": "^0.3.4",

client/playwright.config.ts

@@ -5,26 +5,48 @@
 import { defineConfig, devices } from '@playwright/test';
 import * as dotenv from 'dotenv';
 
+// Check if playwright was called with 'int' flag.
+const useIntServer = process.env.int === 'true';
+
 dotenv.config({ path: './test/.env' });
 // import fs from 'fs';
 // import path from 'path';
 
 // const storeState = JSON.parse(fs.readFileSync(path.resolve('./playwright/.auth/user.json'), 'utf-8'));
-const BASE_URI = process.env.REACT_APP_URL.toString() ?? '';
+const BASE_URI = process.env.REACT_APP_URL ?? 'http://localhost:4000/';
 
 export default defineConfig({
-  timeout: 45000,
-  globalTimeout: 600000,
-  testDir: './test/e2e',
+  webServer: useIntServer ? undefined : {
+    command: 'yarn start'
+  },
+  timeout: 60 * 1000,
+  globalTimeout: 10 * 60 * 1000,
+  testDir: './test/e2e/tests',
   testMatch: /.*\.test\.ts/,
   reporter: [
-    ['html', { outputFile: 'playwright-report/index.html' }],
+    [
+      'html',
+      {
+        outputFile: 'playwright-report/index.html',
+      },
+    ],
     ['list'],
-    ['junit', { outputFile: 'playwright-report/results.xml' }],
-    ['json', { outputFile: 'playwright-report/results.json' }],
-  ],
+    [
+      'junit',
+      {
+        outputFile: 'playwright-report/results.xml',
+      },
+    ],
+    [
+      'json',
+      {
+        outputFile: 'playwright-report/results.json',
+      },
+    ],
+  ], // Codecov handled through Monocart-Reporter https://github.com/cenfun/monocart-reporter
   use: {
     baseURL: BASE_URI,
+    trace: 'retain-on-failure',
   },
   projects: [
     // Setup project
@@ -51,4 +73,6 @@ export default defineConfig({
       dependencies: ['setup'],
     },
   ],
+  globalSetup: 'test/e2e/setup/global.setup.ts',
+  globalTeardown: 'test/e2e/setup/global-teardown.ts',
 });

client/test/README.md

@@ -1,24 +1,28 @@
 # End-to-End (E2E) Tests
 
-The E2E tests require a functional gitlab oauth setup, traefik gateway and a
-live react client website. The E2E tests do not launch the react client website.
-So it is important to launch the website using `yarn start`. Keep this server
-running while performing the E2E tests with `yarn test:e2e` command.
+The E2E tests require playwright test runner, an on-premise GitLab OAuth setup and
+configured `config/test.js` and `test/.env` files. When everything is set up, you
+can run the tests by running `yarn test:e2e`.
+
+By default, Playwright launches the react client website as specified in `config/test.js`
+and terminates it afterwards. Running `yarn test:e2e:int` allows you to skip this so
+you can host the website yourself.
+
+## Playwright
 
 The E2E tests use playwright test runner. You also need to have the software
-installed. If it is not installed, you can install with the following command.
+installed. If it is not installed, you can install it with the following command.
 
 ```bash
-sudo npx playwright install-deps
+yarn playwright install --with-deps
 ```
 
 ## OAuth Setup
 
-You can follow the instructions in
-[authorization page](../../docs/admin/client/auth.md) to setup oauth for the
+You can follow the instructions in [authorization page](../../docs/admin/client/auth.md) to setup OAuth for the
 react client website. Remember to add the `http://localhost:4000` as callback URL
-in the oauth application. The gitlab will still be running on a remote machine.
-It is not possible to run both the gitlab and react client website on localhost.
+in the OAuth application. The GitLab will still be running on a remote machine.
+It is not possible to run both the GitLab and react client website on localhost.
 
 ## config/test.js file
 
@@ -35,8 +39,7 @@ the details of your testing environment. For instance, you need to adjust:
 * `REACT_APP_LOGOUT_REDIRECT_URI`
 
 to reflect your test setup. More information on about the environment settings is
-available in [authorization](../../docs/admin/client/auth.md) and
-[client deployment](../../docs/admin/client/CLIENT.md) pages.
+available in [authorization](../../docs/admin/client/auth.md) and [client deployment](../../docs/admin/client/CLIENT.md) pages.
 
 Here's an example of relevant values for variables. This example is suitable for
 testing on developer computer.
@@ -48,6 +51,8 @@ REACT_APP_REDIRECT_URI="http://localhost:4000/Library"
 REACT_APP_LOGOUT_REDIRECT_URI="http://localhost:4000"
 ```
 
+Finally, run `yarn config:test` to copy the config file into the `build` and `public` folders.
+
 ## env file
 
 You need to create a `test/.env` file where you will store the GitLab user
@@ -77,7 +82,8 @@ REACT_APP_URL='http://localhost:4000'
 
 ## Testing on localhost
 
-There are two possible testing setups you can create.
+If you want to handle starting the react client server yourself, there are two
+possible testing setups you can create.
 
 1. Host website on the developer computer and test from developer computer
 1. Host website on the integration server and test from the integration server
@@ -127,7 +133,7 @@ credentials on `gitlab.foo.com`.
 ## Testing on the integration server
 
 In this test setup, the DTaaS application is running at `https://foo.com` and
-the gitlab instance is running at `https://gitlab.foo.com`. The E2E test shall
+the GitLab instance is running at `https://gitlab.foo.com`. The E2E test shall
 be run from the developer computer. The codebase commit should be the same on
 both the developer computer and integration server.
 
@@ -150,7 +156,7 @@ window.env = {
   REACT_APP_AUTH_AUTHORITY: 'https://gitlab.foo.com/',
   REACT_APP_REDIRECT_URI: 'https://foo.com/Library',
   REACT_APP_LOGOUT_REDIRECT_UR: 'https://foo.com/',
-  REACT_APP_GITLAB_SCOPES: 'openid profile read_user read_repository api',
+  REACT_APP_GitLab_SCOPES: 'openid profile read_user read_repository api',
 };
 ```
 
@@ -171,12 +177,19 @@ gateway and the client website hosted behind traefik.
 
 ## Running the Tests
 
-Once you've properly set up your .env file, you can now run the end-to-end tests.
+Once you've properly set up your .env file, you can run the end-to-end tests as follows:
 
 ```bash
 yarn test:e2e
 ```
+Or with manual website launch:
+```bash
+yarn test:e2e:int
+```
+
+These commands launch the test runner and execute all end-to-end tests. The first
+command also runs the `yarn start` command to start the client website
+and terminates it after testing.
 
-This command launches the test runner and executes all end-to-end tests.
 Make sure you have an active internet connection while running these tests,
 as they simulate real user interactions with your GitLab account.

client/test/e2e/setup/fixtures.ts

@@ -0,0 +1,47 @@
+import { test as testBase } from '@playwright/test';
+import MCR from 'monocart-coverage-reports';
+import coverageOptions from './mcr.config';
+
+// fixtures
+const test = testBase.extend<{
+  autoTestFixture: string;
+}>({
+  autoTestFixture: [
+    async ({ page }, use) => {
+      const isChromium = test.info().project.name === 'chromium';
+
+      // console.log('autoTestFixture setup...');
+      // coverage API is chromium only
+      if (isChromium) {
+        await Promise.all([
+          page.coverage.startJSCoverage({
+            resetOnNavigation: false,
+          }),
+          page.coverage.startCSSCoverage({
+            resetOnNavigation: false,
+          }),
+        ]);
+      }
+
+      await use('autoTestFixture');
+
+      // console.log('autoTestFixture teardown...');
+      if (isChromium) {
+        const [jsCoverage, cssCoverage] = await Promise.all([
+          page.coverage.stopJSCoverage(),
+          page.coverage.stopCSSCoverage(),
+        ]);
+        const coverageList = [...jsCoverage, ...cssCoverage];
+        // console.log(coverageList.map((item) => item.url));
+        const mcr = MCR(coverageOptions);
+        await mcr.add(coverageList);
+      }
+    },
+    {
+      scope: 'test',
+      auto: true,
+    },
+  ],
+});
+
+export default test;

client/test/e2e/setup/global-teardown.ts

@@ -0,0 +1,9 @@
+import MCR from 'monocart-coverage-reports';
+import coverageOptions from './mcr.config';
+
+async function globalTeardown() {
+  const mcr = MCR(coverageOptions);
+  await mcr.generate();
+}
+
+export default globalTeardown;

client/test/e2e/setup/global.setup.ts

@@ -0,0 +1,9 @@
+import MCR from 'monocart-coverage-reports';
+import coverageOptions from './mcr.config';
+
+async function globalSetup() {
+  const mcr = MCR(coverageOptions);
+  await mcr.cleanCache();
+}
+
+export default globalSetup;

client/test/e2e/setup/mcr.config.ts

@@ -0,0 +1,12 @@
+import { CoverageReportOptions } from 'monocart-coverage-reports';
+
+// https://github.com/cenfun/monocart-coverage-reports
+const coverageOptions: CoverageReportOptions = {
+  name: 'Playwright Monocart Coverage Reports',
+
+  reports: ['codecov', 'v8', 'console-summary'],
+
+  outputDir: './coverage/e2e',
+};
+
+export default coverageOptions;