ci: add smoke test run for every integration

.github/workflows/test.yml

@@ -13,3 +13,54 @@ jobs:
           node-version: 18
       - run: npm install
       - run: npm run lint
+
+  smoke-tests:
+    needs: lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      # We don't want to fail the whole matrix if one job fails
+      # Makes it harder to debug what went wrong if it ends early
+      fail-fast: false
+      matrix:
+        include:
+          - directory: cypress/basic
+          - directory: cypress/iframes
+            additional-steps: cd cypress/iframes && node server.js &
+          - directory: cypress/manual-mode
+          - directory: cypress/multi-page
+          - directory: playwright/basic
+            playwright: true
+          - directory: playwright/manual-mode
+            playwright: true
+          - directory: playwright/typescript-multi-page
+            playwright: true
+          - directory: playwright-test/basic
+            playwright: true
+          - directory: playwright-test/multi-page
+            playwright: true
+          - directory: puppeteer/basic
+          - directory: puppeteer/typescript-multi-page
+          - directory: wdio/test-runner/typescript
+          - directory: wdio/typescript-multi-page
+          - directory: wdio/v7/typescript-basic
+            # WDIO v7 does not work on Node > 16. Please refer to `setup.ts` for more information.
+            node-version: 16
+          - directory: webdriverjs/basic
+          - directory: webdriverjs/testing
+          - directory: webdriverjs/typescript-multi-page
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version || '20' }}
+      - name: Install Playwright
+        if: ${{ matrix.playwright }}
+        run: npx playwright install
+      - name: Additional steps
+        if: ${{ matrix.additional-steps }}
+        run: ${{ matrix.additional-steps }}
+      - name: Run tests
+        run: cd ${{ matrix.directory }} && npm install && npm test
+        env:
+          API_KEY: ${{ secrets.AXE_DEVHUB_API_KEY }}

cypress/basic/package.json

@@ -5,7 +5,7 @@
     "test": "cypress run --headed --browser=chrome"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "cypress": "^12.12.0"
   }
 }

cypress/iframes/package.json

@@ -3,7 +3,7 @@
     "test": "cypress run --headed --browser=chrome"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "cypress": "^12.13.0"
   }
 }

cypress/manual-mode/package.json

@@ -5,7 +5,7 @@
     "test": "cypress run --headed --browser=chrome"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "cypress": "^12.12.0"
   }
 }

cypress/multi-page/package.json

@@ -5,7 +5,7 @@
     "test": "cypress run --headed --browser chrome"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "cypress": "^12.12.0"
   }
 }

playwright-test/basic/package.json

@@ -5,7 +5,7 @@
     "test": "playwright test"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@playwright/test": "^1.34.3"
   }
 }

playwright-test/basic/tests/fixtures.js

@@ -9,5 +9,6 @@ module.exports = playwrightTest({
     apiKey: API_KEY,
     serverURL: SERVER_URL
   },
-  headless: false
+  headless: false,
+  args: ['--headless=new']
 })

playwright-test/multi-page/package.json

@@ -5,7 +5,7 @@
     "test": "playwright test"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@playwright/test": "^1.34.3",
     "typescript": "^5.0.4"
   }

playwright-test/multi-page/tests/fixtures.ts

@@ -9,7 +9,8 @@ const { test, expect } = playwrightTest({
     apiKey: API_KEY,
     serverURL: SERVER_URL
   },
-  headless: false
+  headless: false,
+  args: ['--headless=new']
 })
 
 export { test, expect }

playwright/basic/package.json

@@ -5,7 +5,7 @@
     "test": "mocha --timeout 5000 'tests/login.js'"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "chai": "^4.3.7",
     "mocha": "^10.2.0",
     "playwright": "^1.33.0"

playwright/manual-mode/package.json

@@ -5,7 +5,7 @@
     "test": "mocha --timeout 5000 'tests/login.js'"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "chai": "^4.3.7",
     "mocha": "^10.2.0",
     "playwright": "^1.33.0"

playwright/typescript-multi-page/package.json

@@ -5,7 +5,7 @@
     "test": "mocha --require ts-node/register --timeout 10s tests/*.test.ts"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@types/chai": "^4.3.5",
     "@types/mocha": "^10.0.1",
     "chai": "^4.3.7",

puppeteer/basic/package.json

@@ -5,7 +5,7 @@
     "test": "mocha tests/*.js --timeout 10s"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "chai": "^4.3.7",
     "mocha": "^10.2.0",
     "puppeteer": "^20.5.0"

puppeteer/basic/tests/login.js

@@ -21,7 +21,8 @@ describe('My Login Application', () => {
           apiKey: API_KEY,
           serverURL: SERVER_URL
         },
-        headless: false
+        headless: false,
+        args: ['--headless=new']
       })
     )
     // Create a page instance, using your browser instance.

puppeteer/typescript-multi-page/package.json

@@ -5,7 +5,7 @@
     "test": "mocha --require ts-node/register --timeout 10s tests/*.test.ts"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@types/chai": "^4.3.5",
     "@types/mocha": "^10.0.1",
     "chai": "^4.3.7",

puppeteer/typescript-multi-page/tests/setup.ts

@@ -21,7 +21,8 @@ before(async () => {
         apiKey: API_KEY,
         serverURL: SERVER_URL
       },
-      headless: false
+      headless: false,
+      args: ['--headless=new']
     })
   )
 })

wdio/test-runner/typescript/package.json

@@ -5,7 +5,7 @@
     "test": "wdio run wdio.config.ts"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@types/chai": "^4.3.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^20.3.1",
@@ -15,7 +15,7 @@
     "@wdio/mocha-framework": "^8.11.0",
     "@wdio/spec-reporter": "^8.11.2",
     "chai": "^4.3.7",
-    "chromedriver": "^122.0.4",
+    "chromedriver": "^126.0.4",
     "ts-node": "^10.9.1",
     "typescript": "^5.1.3",
     "wdio-chromedriver-service": "^8.1.1"

wdio/test-runner/typescript/wdio.config.ts

@@ -6,7 +6,12 @@ const { API_KEY, SERVER_URL = 'https://axe.deque.com' } = process.env
 export const config = wdioTestRunner({
   axe: { apiKey: API_KEY as string, serverURL: SERVER_URL },
   specs: ['forgot-password.test.ts', 'login.test.ts', 'home.test.ts'],
-  capabilities: [{ browserName: 'chrome' }],
+  capabilities: [
+    {
+      browserName: 'chrome',
+      'goog:chromeOptions': { args: ['--headless=new'] }
+    }
+  ],
   baseUrl: 'https://the-internet.herokuapp.com',
   services: ['chromedriver'],
   framework: 'mocha',

wdio/typescript-multi-page/package.json

@@ -5,12 +5,12 @@
     "test": "mocha --require ts-node/register --timeout 10s '*.test.ts'"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@types/chai": "^4.3.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^20.3.1",
     "chai": "^4.3.7",
-    "chromedriver": "^122.0.4",
+    "chromedriver": "^126.0.4",
     "mocha": "^10.2.0",
     "ts-node": "^10.9.1",
     "typescript": "^5.1.3",

wdio/typescript-multi-page/setup.ts

@@ -16,7 +16,8 @@ before(async () => {
         serverURL: SERVER_URL
       },
       capabilities: {
-        browserName: 'chrome'
+        browserName: 'chrome',
+        'goog:chromeOptions': { args: ['--headless=new'] }
       }
     })
   )

wdio/v7/typescript-basic/package.json

@@ -5,12 +5,12 @@
     "test": "mocha --require ts-node/register --timeout 10s '**/*.test.ts'"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@types/chai": "^4.3.5",
     "@types/mocha": "^10.0.1",
     "@types/node": "^20.3.1",
     "chai": "^4.3.7",
-    "chromedriver": "^114.0.2",
+    "chromedriver": "^126.0.4",
     "mocha": "^10.2.0",
     "ts-node": "^10.9.1",
     "typescript": "^5.1.3",

wdio/v7/typescript-basic/setup.ts

@@ -31,7 +31,8 @@ before(async () => {
         serverURL: SERVER_URL
       },
       capabilities: {
-        browserName: 'chrome'
+        browserName: 'chrome',
+        'goog:chromeOptions': { args: ['--headless=new'] }
       }
     })
   )

webdriverjs/basic/package.json

@@ -2,12 +2,12 @@
   "name": "basic",
   "private": true,
   "scripts": {
-    "test": "mocha --timeout 10s tests/*.js"
+    "test": "mocha --timeout 20s tests/*.js"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "chai": "^4.3.7",
-    "chromedriver": "^114.0.1",
+    "chromedriver": "^126.0.4",
     "mocha": "^10.2.0",
     "selenium-webdriver": "^4.9.2"
   }

webdriverjs/basic/tests/login.js

@@ -4,6 +4,7 @@ const {
   webdriverConfig,
   WebdriverController
 } = require('@axe-core/watcher')
+const { Options } = require('selenium-webdriver/chrome')
 
 /* Get your configuration from environment variables. */
 const { API_KEY, SERVER_URL = 'https://axe.deque.com' } = process.env
@@ -13,14 +14,17 @@ describe('My Login Application', () => {
   let controller
 
   before(async () => {
+    const options = new Options()
+    options.addArguments('--headless=new')
     browser = await new Builder()
       .forBrowser('chrome')
       .setChromeOptions(
         webdriverConfig({
           axe: {
             apiKey: API_KEY,
             serverURL: SERVER_URL
-          }
+          },
+          options
         })
       )
       .build()

webdriverjs/testing/package.json

@@ -2,12 +2,12 @@
   "name": "testing",
   "private": true,
   "scripts": {
-    "test": "mocha --timeout 10s tests/*.js"
+    "test": "SELENIUM_BROWSER=chrome mocha --timeout 20s tests/*.js"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@types/selenium-webdriver": "^4.1.15",
-    "chromedriver": "^114.0.2",
+    "chromedriver": "^126.0.4",
     "mocha": "^10.2.0",
     "selenium-webdriver": "^4.10.0"
   }

webdriverjs/testing/tests/login.js

@@ -5,6 +5,7 @@ const {
   webdriverConfig,
   WebdriverController
 } = require('@axe-core/watcher')
+const { Options } = require('selenium-webdriver/chrome')
 
 /* Get your configuration from environment variables. */
 const { API_KEY, SERVER_URL = 'https://axe.deque.com' } = process.env
@@ -15,6 +16,8 @@ suite(env => {
     let controller
 
     before(async () => {
+      const options = new Options()
+      options.addArguments('--headless=new')
       const builder = env.builder()
 
       // axe Watcher only supports Chrome.
@@ -27,7 +30,8 @@ suite(env => {
               axe: {
                 apiKey: API_KEY,
                 serverURL: SERVER_URL
-              }
+              },
+              options
             })
           )
           .build()

webdriverjs/typescript-multi-page/package.json

@@ -5,12 +5,12 @@
     "test": "mocha --require ts-node/register --timeout 10s tests/*.test.ts"
   },
   "devDependencies": {
-    "@axe-core/watcher": "^3.11.1",
+    "@axe-core/watcher": "^3.12.0",
     "@types/chai": "^4.3.5",
     "@types/mocha": "^10.0.1",
     "@types/selenium-webdriver": "^4.1.15",
     "chai": "^4.3.7",
-    "chromedriver": "^114.0.1",
+    "chromedriver": "^126.0.4",
     "mocha": "^10.2.0",
     "selenium-webdriver": "^4.9.2",
     "ts-node": "^10.9.1",