test: fix and refactor e2e tests

.circleci/config.yml

@@ -2,25 +2,39 @@ version: 2
 jobs:
   build:
     docker:
-      - image: cimg/node:12.18-browsers
+      - image: cimg/node:16.20-browsers
 
     steps:
-      - checkout
+      - checkout:
+          path: 'html-reporter'
 
-      - run: npm install
+      - run: cd html-reporter && npm ci
+
+      - run:
+          name: Download Chromium
+          command: >-
+            cd test/func/docker/browser-utils &&
+            npm ci &&
+            CHROME_PATH=$(node download-chromium.js |& grep '^/home') &&
+            CHROME_PATH=$(dirname $CHROME_PATH) &&
+            mkdir ~/browsers &&
+            mv $CHROME_PATH ~/browsers/chrome-linux
+            cd ~/html-reporter
 
       - run:
           name: Download Selenium
-          command: sudo npm install selenium-standalone@6.17.0 -g
+          command: npm install selenium-standalone@9.1.1 -g
       - run:
           name: Start Selenium
-          command: selenium-standalone install && selenium-standalone start
+          command: >-
+            selenium-standalone install --drivers.chrome.version=116 &&
+            selenium-standalone start --drivers.chrome.version=116
           background: true
 
       - run:
           name: Functional tests
-          command: npm run test-func
+          command: npm run e2e
 
       - store_artifacts:
-          path: hermione-report/
-          destination: /hermione-report
+          path: reports/
+          destination: /reports

.gitignore

@@ -23,3 +23,4 @@ sqlite.db
 
 hermione-report
 /test/func/fixtures/report
+test/func/fixtures/plugins/html-reporter-plugins/*/plugin.js

CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contribution guide
+
+> This document is currently a work in progress and is not yet comprehensive.
+> Additional info will be added over time.
+
+### Running e2e tests
+
+End-to-end testing of html-reporter consists of two stages: generating fixture reports using different tools and tests,
+then running hermione tests on these reports.
+
+In order to make e2e/screenshot tests stable and reproducible across different environments,
+you need to launch browsers inside a Docker container.
+
+1. Make sure you have Docker installed.
+    <details><summary>How to?</summary>
+    1. If you want to make a personal open-source contribution, you may use Docker free of charge and follow the [official guide](https://docs.docker.com/get-docker/).
+    2. If you are acting on behalf of a company, you may not have access to Docker Desktop. In this case:
+        - On Linux, you may follow the official installation guide.
+        - On Mac, you may use [colima](https://github.com/abiosoft/colima) as a replacement for Docker Desktop.
+        - On Windows, you may use Windows Subsystem for Linux to run the Docker CLI without the Desktop application.
+    </details>
+2. Run e2e tests:
+    ```bash
+    npm run e2e
+    ```
+
+If you want a finer-grained control over the process, the following commands are available:
+- `npm run e2e:build-browsers` — build Docker image with browsers
+- `npm run e2e:launch-browsers` — launch selenium standalone server inside Docker
+- `npm run e2e:generate-fixture-report` — generate fixture report to run tests on
+- `npm run e2e:test` — run e2e tests

package.json

@@ -16,17 +16,14 @@
     "copy-static": "copyfiles 'lib/static/icons/*' .npmignore build",
     "check-types": "tsc --project tsconfig.spec.json",
     "coverage": "nyc npm run test-unit",
+    "e2e": "npm run e2e:launch-browsers && npm run e2e:generate-fixture-report -- --all && npm run e2e:test -- --all",
+    "e2e:build-browsers": "docker build -f test/func/docker/Dockerfile -t html-reporter-browsers:0.0.1 test/func/docker",
+    "e2e:generate-fixture-report": "node test/func/fixtures/cli.js",
+    "e2e:launch-browsers": "docker run -it --rm -p 4444:4444 html-reporter-browsers:0.0.1",
+    "e2e:test": "node test/func/tests/cli.js",
     "lint": "eslint .",
     "test": "npm run lint && npm run test-unit && npm run check-types",
     "test-unit": "cross-env NODE_ENV=test mocha test/unit",
-    "test-func": "npm run test-func-main && npm run test-func-plugins",
-    "test-func-main": "npm run make-main-fixtures; cross-env NODE_ENV=test hermione",
-    "test-func-plugins": "npm run make-plugins-fixtures; cross-env NODE_ENV=test hermione -c test/func/plugins/.hermione.plugins.js",
-    "make-main-fixtures": "npm run clean-fixtures && cross-env NODE_ENV=test hermione -c test/func/fixtures/.hermione.fixtures.js",
-    "make-plugins-fixtures": "npm run clean-fixtures && cross-env NODE_ENV=test html_reporter_plugins_enabled=true hermione -c test/func/fixtures/.hermione.fixtures.js",
-    "make-main-fixtures-gui": "npm run make-main-fixtures -- gui",
-    "make-plugins-fixtures-gui": "npm run make-plugins-fixtures -- gui",
-    "clean-fixtures": "rimraf test/func/fixtures/report",
     "precommit": "npm run lint",
     "prepublishOnly": "npm run build",
     "start": "NODE_ENV=development webpack serve --config=webpack.dev.js --progress",
@@ -127,7 +124,6 @@
     "fork-ts-checker-webpack-plugin": "^6.5.3",
     "hermione": "^8.0.0-beta.2",
     "hermione-global-hook": "^1.0.1",
-    "hermione-headless-chrome": "0.0.2",
     "html-react-parser": "^0.4.0",
     "html-reporter-basic-plugin": "file:test/func/html-reporter-plugins/basic",
     "html-reporter-menu-bar-plugin": "file:test/func/html-reporter-plugins/menu-bar",

test/func/docker/.dockerignore

@@ -0,0 +1 @@
+**/node_modules

test/func/docker/Dockerfile

@@ -0,0 +1,20 @@
+FROM cimg/node:16.20-browsers
+
+ENV CHROME_VERSION=116
+
+USER circleci
+
+COPY --chown=circleci browser-utils browser-utils
+WORKDIR browser-utils
+
+RUN npm ci && \
+    # The download script outputs progress messages, but we need only final path, hence we filter output here
+    CHROME_PATH=$(node ./download-chromium.js |& grep '^/home') && \
+    CHROME_PATH=$(dirname $CHROME_PATH) && \
+    mkdir ~/browsers && \
+    mv $CHROME_PATH ~/browsers/chrome-linux
+
+RUN npm install  [email protected] -g && \
+    selenium-standalone  install --drivers.chrome.version=$CHROME_VERSION
+
+ENTRYPOINT selenium-standalone start --drivers.chrome.version=$CHROME_VERSION

test/func/docker/browser-utils/download-chromium.js

@@ -0,0 +1,6 @@
+const os = require('os');
+const path = require('path');
+const downloadChromium = require('hermione-headless-chrome/lib/download-chromium-by-version');
+
+downloadChromium(process.env.CHROME_VERSION, path.join(os.homedir(), 'chrome-cache'), 10)
+    .then(chromePath => console.log(chromePath));

test/func/docker/browser-utils/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "browser-utils",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {},
+  "dependencies": {
+    "hermione-headless-chrome": "1.1.0"
+  }
+}

test/func/fixtures/cli.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const fs = require('fs/promises');
+const path = require('path');
+const util = require('util');
+const _ = require('lodash');
+const inquirer = require('inquirer');
+const program = require('@gemini-testing/commander');
+
+const exec = util.promisify(require('child_process').exec);
+
+const tasks = [
+    {project: 'hermione', task: () => exec('npx hermione', {cwd: path.resolve(__dirname, 'hermione')})},
+    {project: 'plugins', task: async () => {
+        const pluginNames = await fs.readdir('plugins');
+        await Promise.all(pluginNames.map(pluginName =>
+            exec('npm run build', {cwd: path.resolve(__dirname, 'hermione', pluginName)})
+        ));
+
+        return exec('npx hermione', {cwd: path.resolve(__dirname, 'plugins')});
+    }}
+];
+const allProjects = tasks.map(_.property('project'));
+
+(async () => {
+    program
+        .option('--all')
+        .option('-p --project [value]', '')
+        .parse(process.argv);
+
+    let selectedProjects = [];
+    if (program.all) {
+        selectedProjects = allProjects;
+    } else if (program.project) {
+        selectedProjects = program.project;
+    } else {
+        const {projects} = await inquirer.prompt([{
+            type: 'checkbox',
+            name: 'projects',
+            message: 'Select projects for which you want to generate fixture reports:',
+            choices: allProjects
+        }]);
+
+        selectedProjects = projects;
+    }
+
+    if (selectedProjects.some(p => !allProjects.includes(p))) {
+        throw new Error(`Project must be one of ${allProjects}`);
+    }
+
+    await Promise.all(tasks
+        .filter(task => selectedProjects.includes(task.project))
+        .map(task => task.task()));
+
+    console.log('All done!');
+})();

test/func/fixtures/hermione/.hermione.conf.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const {GRID_URL, CHROME_BINARY_PATH} = require('../../utils/constants');
+global.assert = require('chai').assert;
+
+const serverPort = 8080;
+const fixturesPath = 'test/func/fixtures/hermione/report';
+
+module.exports = {
+    gridUrl: GRID_URL,
+    baseUrl: `http://localhost:${serverPort}/test/func/fixtures/hermione/index.html`,
+
+    screenshotsDir: 'test/func/fixtures/screens',
+
+    sets: {
+        fixtures: {
+            files: 'test/func/fixtures/**/*.hermione.js'
+        }
+    },
+
+    browsers: {
+        chrome: {
+            windowSize: '1280x1024',
+            desiredCapabilities: {
+                browserName: 'chrome',
+                'goog:chromeOptions': {
+                    args: ['headless', 'no-sandbox'],
+                    binary: CHROME_BINARY_PATH,
+                }
+            }
+        }
+    },
+
+    plugins: {
+        'html-reporter-test-server': {
+            enabled: true,
+            port: serverPort
+        },
+        'html-reporter-tester': {
+            enabled: true,
+            path: fixturesPath,
+        },
+    }
+};

test/func/fixtures/hermione/failed-describe.hermione.js

@@ -0,0 +1,17 @@
+describe('failed describe', function() {
+    it('successfully passed test', async function() {
+        await this.browser.url(this.browser.options.baseUrl);
+
+        assert.isTrue(true);
+    });
+
+    it('test without screenshot', async function() {
+        await this.browser.url(this.browser.options.baseUrl);
+
+        await this.browser.assertView('header', 'header');
+    });
+
+    it('test with long error message', async function() {
+        throw new Error(`long_error_message ${'0123456789'.repeat(20)}\n message content`);
+    });
+});

test/func/fixtures/hermione/success-describe.hermione.js

@@ -0,0 +1,13 @@
+describe('success describe', function() {
+    it('succesfully passed test', async function() {
+        await this.browser.url(this.browser.options.baseUrl);
+
+        assert.isTrue(true);
+    });
+
+    it('test with screenshot', async function() {
+        await this.browser.url(this.browser.options.baseUrl);
+
+        await this.browser.assertView('header', 'header');
+    });
+});

test/func/fixtures/.hermione.fixtures.js → test/func/fixtures/plugins/.hermione.conf.js

@@ -1,26 +1,32 @@
 'use strict';
 
+const {GRID_URL, CHROME_BINARY_PATH} = require('../../utils/constants');
 global.assert = require('chai').assert;
 
 const serverPort = 8080;
 const fixturesPath = 'test/func/fixtures/report';
 
 module.exports = {
-    baseUrl: `http://localhost:${serverPort}/test/func/fixtures/index.html`,
+    gridUrl: GRID_URL,
+    baseUrl: `http://localhost:${serverPort}/test/func/fixtures/plugins/index.html`,
 
     screenshotsDir: 'test/func/fixtures/screens',
 
     sets: {
         fixtures: {
-            files: 'test/func/fixtures/**/*.hermione.js'
+            files: 'test/func/fixtures/plugins/**/*.hermione.js'
         }
     },
 
     browsers: {
         chrome: {
             windowSize: '1280x1024',
             desiredCapabilities: {
-                browserName: 'chrome'
+                browserName: 'chrome',
+                'goog:chromeOptions': {
+                    args: ['headless', 'no-sandbox'],
+                    binary: CHROME_BINARY_PATH,
+                }
             }
         }
     },
@@ -33,7 +39,7 @@ module.exports = {
         'html-reporter-tester': {
             enabled: true,
             path: fixturesPath,
-            pluginsEnabled: false,
+            pluginsEnabled: true,
             plugins: [
                 {
                     name: 'html-reporter-basic-plugin',
@@ -90,9 +96,5 @@ module.exports = {
                 }
             ]
         },
-        'hermione-headless-chrome': {
-            browserId: 'chrome',
-            version: '77'
-        }
     }
 };