js: test: Decouple screenshots from regular e2e tests

Makefile

@@ -120,17 +120,25 @@ docker-js-integration:
 		--network container:isso-server \
 		$(TESTBED_IMAGE) npm run test-integration
 
-docker-compare-screenshots:
+docker-generate-screenshots:
 	docker run \
 		--mount type=bind,source=$(PWD)/package.json,target=/src/package.json,readonly \
-		--mount type=bind,source=$(PWD)/isso/js/,target=/src/isso/js/,readonly \
-		$(TESTBED_IMAGE) bash isso/js/tests/integration/compare-hashes.sh
+		--mount type=bind,source=$(PWD)/isso/js/,target=/src/isso/js/ \
+		--network container:isso-server \
+		--env ISSO_ENDPOINT='http://isso-dev.local:8080' \
+		$(TESTBED_IMAGE) npm run test-screenshots
 
-docker-update-screenshots:
+docker-compare-screenshots: docker-generate-screenshots
 	docker run \
 		--mount type=bind,source=$(PWD)/package.json,target=/src/package.json,readonly \
-		--mount type=bind,source=$(PWD)/isso/js/,target=/src/isso/js/,readonly \
-		$(TESTBED_IMAGE) bash isso/js/tests/integration/compare-hashes.sh -u
+		--mount type=bind,source=$(PWD)/isso/js/,target=/src/isso/js/ \
+		$(TESTBED_IMAGE) bash isso/js/tests/screenshots/compare-hashes.sh
+
+docker-update-screenshots: docker-generate-screenshots
+	docker run \
+		--mount type=bind,source=$(PWD)/package.json,target=/src/package.json,readonly \
+		--mount type=bind,source=$(PWD)/isso/js/,target=/src/isso/js/ \
+		$(TESTBED_IMAGE) bash isso/js/tests/screenshots/compare-hashes.sh -u
 
 clean:
 	rm -f $(ISSO_JS_DST)

docker-compose.yml

@@ -59,8 +59,8 @@ services:
         target: /src/isso/js/
         read_only: true
       - type: bind
-        source: ./isso/js/tests/integration/screenshots/
-        target: /src/isso/js/tests/integration/screenshots/
+        source: ./isso/js/tests/screenshots/reference/
+        target: /src/isso/js/tests/screenshots/reference/
         read_only: false  # needed for generating screenshots during e2e tests
     depends_on:
       - isso-server

docs/docs/technical-docs/testing-client.rst

@@ -178,7 +178,7 @@ To generate updated screenshots, you must run the following commands.
 
    $ make docker-js-integration
 
-This will run the test ``isso/js/tests/integration/screenshots.test.js`` and output
+This will run the test ``isso/js/tests/screenshots/screenshots.test.js`` and output
 ``.png`` files. Now compare those screenshots against the known-to-be-good
 ``.hash`` files:
 
@@ -195,7 +195,7 @@ new hashes:
    $ make docker-compare-screenshots -u
 
 Then commit those generated ``.hash`` files in
-``isso/js/tests/integration/screenshots/*.png.hash`` to git.
+``isso/js/tests/screenshots/reference/*.png.hash`` to git.
 
 .. warning:: Please only check in hashes rendered through the Docker
    environment as otherwise the screenshots could be skewed by aliasing and
@@ -205,7 +205,7 @@ Then commit those generated ``.hash`` files in
    older Ubuntu versions - another reason to stick to the Docker environment
    for consistency.
 
-.. _integration tests folder: https://github.com/posativ/isso/tree/master/isso/js/tests/integration/screenshots
+.. _integration tests folder: https://github.com/posativ/isso/tree/master/isso/js/tests/screenshots
 
 Testing standards
 -----------------

isso/js/tests/integration/compare-hashes.sh → isso/js/tests/screenshots/compare-hashes.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/sh
+#!/bin/bash
 
 # This script takes the output of the screenshot puppeteer test and compares
 # the generated images against hashes of known-to-be-good images.
@@ -13,7 +13,7 @@
 # - imagemagick (for "identify" program)
 # - bash        (hopefully this script should run on any shell though)
 
-SCREENSHOTS_PATH=isso/js/tests/integration/screenshots
+SCREENSHOTS_PATH=isso/js/tests/screenshots/reference
 update=false
 
 for i in "$@"; do
@@ -79,8 +79,15 @@ if [ $thread_hash != $expected_thread ]; then
 fi
 
 if [ $failed = "true" ]; then
-    echo "One or more screenshots do not match (see above)."
+    echo ""
+    echo "*** ERROR: *** One or more screenshots do not match (see above)."
+    echo ""
     echo "To update screenshot hashes, run this script with the '-u' flag"
+    echo "You can do this e.g. via Docker. Run:"
+    echo "$ make docker"
+    echo "$ docker-compose up -d"
+    echo "$ make docker-update-screenshots"
+    echo ""
     exit 1
 else
     echo "Everything fine, screenshots have not changed."

isso/js/tests/integration/screenshots.test.js → isso/js/tests/screenshots/screenshots.test.js

@@ -5,7 +5,7 @@
 
 const ISSO_ENDPOINT = process.env.ISSO_ENDPOINT ?
   process.env.ISSO_ENDPOINT : 'http://localhost:8080';
-const SCREENSHOTS_PATH = 'isso/js/tests/integration/screenshots';
+const SCREENSHOTS_PATH = 'isso/js/tests/screenshots/reference';
 
 beforeEach(async () => {
   await page.goto(

package.json

@@ -8,6 +8,7 @@
     "test": "jest --config isso/js/jest.config.js isso/js/tests/",
     "test-unit": "jest --config isso/js/jest-unit.config.js isso/js/tests/unit/",
     "test-integration": "jest --runInBand --config isso/js/jest-integration.config.js isso/js/tests/integration/",
+    "test-screenshots": "jest --runInBand --config isso/js/jest-integration.config.js isso/js/tests/screenshots/",
     "build-dev": "webpack --config isso/js/webpack.config.js --config-name dev",
     "watch-dev": "webpack --config isso/js/webpack.config.js --config-name dev --watch",
     "build-prod": "webpack --config isso/js/webpack.config.js --merge --config-name dev --config-name prod"