Install backstopjs and git lfs for visual regression testing

.gitattributes

@@ -0,0 +1 @@
+backstop_data/bitmaps_reference/*.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/tests.yml

@@ -1,24 +1,28 @@
 name: Run macro, script and visual tests
 on: [push]
-env:
-  RUNNING_ON_CI: true
-  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-  PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
-  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
 jobs:
-  run-tests:
-    name: Tests
+  run-tests-and-deploy:
     runs-on: ubuntu-latest
+    env:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      RUNNING_IN_CI: true
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          lfs: true
+          url: https://github.com/ONSdigital/design-system.git
       - name: Install Node
         uses: actions/setup-node@v2
         with:
-          node-version: '14'
+          node-version: "16"
       - name: Install dependencies
         run: yarn install
       - name: Run macro and script tests
         run: yarn test
+      - name: Run visual regression tests
+        run: yarn test-visual
       - name: Create Slack notification
         if: always()
         uses: edge/simple-slack-notify@master

backstop.config.js

@@ -0,0 +1,33 @@
+module.exports = {
+  id: 'ds-vr-test',
+  viewports: [
+    {
+      name: 'desktop',
+      width: 1920,
+      height: 1080,
+    },
+    {
+      name: 'tablet',
+      width: 768,
+      height: 1024,
+    },
+    {
+      name: 'mobile',
+      width: 375,
+      height: 667,
+    },
+  ],
+  scenarios: [],
+  paths: {
+    bitmaps_reference: 'backstop_data/bitmaps_reference',
+    bitmaps_test: 'backstop_data/bitmaps_test',
+    engine_scripts: 'backstop_data/engine_scripts',
+    html_report: 'backstop_data/html_report',
+    ci_report: 'backstop_data/ci_report',
+  },
+  engine: 'puppeteer',
+  engineOptions: {
+    args: ['--no-sandbox'],
+  },
+  report: process.env.RUNNING_IN_CI === 'true' ? [] : ['browser'],
+};

gulpfile.js

@@ -8,12 +8,14 @@ const gulpSourcemaps = require('gulp-sourcemaps');
 const gulpTerser = require('gulp-terser');
 const buffer = require('vinyl-buffer');
 const source = require('vinyl-source-stream');
+const backstop = require('backstopjs');
 
 require('@babel/register');
 
 const babelEsmConfig = require('./babel.conf.esm');
 const babelNomoduleConfig = require('./babel.conf.nomodule');
 const postCssPlugins = require('./postcss.config').default;
+const generateURLs = require('./src/tests/helpers/url-generator.js').default;
 const generateStaticPages = require('./lib/generate-static-pages').default;
 
 const isProduction = process.env.NODE_ENV === 'production';
@@ -89,6 +91,25 @@ gulp.task('generate-pages', async function() {
   await generateStaticPages();
 });
 
+gulp.task('generate-urls', async () => {
+  const urls = await generateURLs();
+  return urls;
+});
+
+function createBackstopTask(task) {
+  return async () => {
+    const urls = await generateURLs();
+    const backstopConfig = require('./backstop.config.js');
+    backstopConfig.scenarios = urls;
+    await backstop(task, {
+      config: backstopConfig,
+    });
+    setTimeout(() => {
+      process.exit();
+    }, 0);
+  };
+}
+
 gulp.task('watch-and-build', async () => {
   browserSync.init({
     proxy: 'localhost:3010',
@@ -110,3 +131,6 @@ gulp.task('watch', gulp.series('watch-and-build', 'start-dev-server'));
 gulp.task('build', gulp.series('copy-static-files', 'build-assets', 'generate-pages'));
 gulp.task('generate', gulp.series('generate-pages'));
 gulp.task('build-package', gulp.series('copy-static-files', 'copy-js-files', 'build-assets'));
+gulp.task('run-backstop-tests', gulp.series('start-dev-server', createBackstopTask('test')));
+gulp.task('run-backstop-reference', gulp.series('start-dev-server', createBackstopTask('reference')));
+gulp.task('run-backstop-approve', createBackstopTask('approve'));

package.json

@@ -15,8 +15,12 @@
     "test:with-log": "yarn test --no-color 2>test.log",
     "test:start-server": "TEST_PORT=3020 gulp start-dev-server",
     "build": "yarn && yarn tidy-clean && NODE_ENV=production gulp build",
+    "build-serve": "yarn build && gulp start-dev-server",
     "npm-bundle": "NODE_ENV=production yarn tidy-clean && NODE_ENV=production gulp build-package && babel-node ci/generate-npm-package.js",
     "cdn-bundle": "NODE_ENV=production yarn tidy-clean && NODE_ENV=production gulp build-package && babel-node ci/prepare-templates-for-zip.js",
+    "test-visual": "yarn build && gulp run-backstop-tests",
+    "test-visual:reference": "yarn build && gulp run-backstop-reference",
+    "test-visual:approve": "gulp run-backstop-approve",
     "tidy-clean": "rm -rf build css favicons fonts img components layout scripts coverage scss js",
     "check-unused": "npx npm-check-unused",
     "dedupe-deps": "npx yarn-deduplicate yarn.lock",
@@ -64,15 +68,12 @@
     "@babel/preset-env": "^7.14.7",
     "@babel/register": "^7.14.5",
     "@babel/runtime": "^7.14.6",
-    "@percy/cli": "^1.0.0-beta.71",
-    "@percy/puppeteer": "^2.0.0",
-    "@percy/script": "^1.1.0",
     "abortcontroller-polyfill": "^1.2.3",
     "autoprefixer": "^9.3.1",
-    "autosize": "^4.0.2",
     "babel-eslint": "^10.0.1",
     "babel-plugin-istanbul": "^6.0.0",
     "babelify": "^10.0.0",
+    "backstopjs": "^6.1.4",
     "browser-sync": "^2.27.3",
     "browserify": "^17.0.0",
     "chalk": "^4.1.2",
@@ -141,5 +142,6 @@
   "resolutions": {
     "minimist": "^1.2.5",
     "**/@percy/agent": "^0.28.6"
-  }
+  },
+  "dependencies": {}
 }

src/tests/helpers/url-generator.js

@@ -0,0 +1,35 @@
+const fs = require('fs');
+const util = require('util');
+const glob = require('glob');
+
+const globUtil = util.promisify(glob);
+const readdir = util.promisify(fs.readdir);
+
+const port = process.env.TEST_PORT_NUMBER || 3010;
+const testURL = `http://localhost:${port}`;
+
+export default async () => {
+  let urls = [];
+  const directories = [
+    {
+      path: './src/components',
+    },
+    {
+      path: './src/patterns',
+    },
+    {
+      path: './src/foundations',
+    },
+  ];
+  for (const directory of directories) {
+    const folders = await readdir(directory.path);
+    for (const folder of folders) {
+      const files = await globUtil(`${directory.path}/${folder}/**/example-*.njk`);
+      for (const file of files) {
+        const urlPath = file.replace(/^\.\/src\/(.*\/example-.*?)\.njk$/, '/$1');
+        urls.push({ url: `${testURL}${urlPath}`, label: urlPath, delay: 1000 });
+      }
+    }
+  }
+  return urls;
+};