Update sitemap a11y scan

.github/workflows/a11y.yml

@@ -0,0 +1,155 @@
+name: A11y Tests
+on:
+  # Manual trigger.
+  workflow_dispatch:
+  # Schedule; currently once nightly, at 3:20 UTC.
+  schedule:
+    - cron: '20 3 * * *'
+jobs:
+  playwright-tests:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    env:
+      BASE_URL: 'https://va.gov'
+      USE_PROXY: false
+      PW_BROWSER: '["chromium", "firefox", "webkit"]'
+      PW_WIDTH: '[320, 768, 1024, 1280, 1920]'
+      PW_HEIGHT: '[720, 1080, 1440]'
+      TOTAL_SEGMENTS: 128
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Install jq
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y jq
+
+      - name: Run command with random values from arrays
+        run: |
+          arrays=('${{ env.PW_BROWSER }}' '${{ env.PW_HEIGHT }}' '${{ env.PW_WIDTH }}')
+          array_names=('PW_BROWSER' 'PW_HEIGHT' 'PW_WIDTH')
+
+          for i in "${!arrays[@]}"; do
+            json_string="${arrays[$i]}"
+            array_name="${array_names[$i]}"
+
+            array=$(echo "$json_string" | jq -c '.')
+            random_index=$((RANDOM % $(echo "$array" | jq 'length')))
+            random_value=$(echo "$array" | jq -r ".[$random_index]")
+            echo "Selected random value from $array_name: $random_value"
+            echo "${array_name}_VALUE=$random_value" >> $GITHUB_ENV
+          done
+
+      - name: Ensure random values are set
+        run: |
+          echo "Using random values: $PW_BROWSER_VALUE, $PW_WIDTH_VALUE, $PW_HEIGHT_VALUE"
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps
+
+      # This is necessary in order for fetch to work.
+      # Finally, made fetch happen...
+      - name: Build proxy-fetcher dist
+        run: |
+          yarn tsc -b ./packages/proxy-fetcher/tsconfig.json
+
+      - name: Run Playwright tests
+        run: |
+          SEGMENT_INDEX=${{ matrix.shardIndex }} yarn playwright test --project=a11y
+
+      - name: Upload scan report to GitHub Actions Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: segment-${{ matrix.shardIndex }}.json
+          path: segment-${{ matrix.shardIndex }}.json
+          retention-days: 7
+          if-no-files-found: ignore
+
+      - name: Upload failed pages report to GitHub Actions Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-pages-segment-${{ matrix.shardIndex }}.json
+          path: failed-pages-segment-${{ matrix.shardIndex }}.json
+          retention-days: 7
+          if-no-files-found: ignore
+
+  merge-scan-reports:
+    if: always()
+    needs: [playwright-tests]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: segment-*
+          merge-multiple: true
+
+      - name: Merge reports in valid JSON format
+        run: |
+          if [ ! -f segment-*.json ]; then
+              echo "No segment-*.json files found."
+              exit 0
+          fi
+          jq -s '.' segment-*.json | jq 'flatten' > all-segments.json
+
+      - name: Upload report
+        uses: actions/upload-artifact@v4
+        with:
+          name: a11y-scan-${{ github.run_attempt }}
+          path: all-segments.json
+          retention-days: 14
+          if-no-files-found: ignore
+
+  merge-failed-pages-reports:
+    if: always()
+    needs: [playwright-tests]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: failed-pages-segment-*
+          merge-multiple: true
+
+      - name: Merge reports in valid JSON format
+        run: |
+          if [ ! -f failed-pages-segment-*.json ]; then
+              echo "No failed-pages-segment-*.json files found."
+              exit 0
+          fi
+          jq -s '.' failed-pages-segment-*.json | jq 'flatten' > all-failed-pages.json
+
+      - name: Upload report
+        uses: actions/upload-artifact@v4
+        with:
+          name: a11y-scan-failed-pages-${{ github.run_attempt }}
+          path: all-failed-pages.json
+          retention-days: 14
+          if-no-files-found: ignore
+

READMEs/testing.md

@@ -75,12 +75,136 @@ This project can be tested for a11y compliance in several ways:
         expect(accessibilityScanResults.violations).toEqual([])
     ```
 
-- A full site scan of all urls known to next-build (generated in the sitemap) using Playwright: `yarn test:playwright:a11y`
-
-The full scan will run (TKTK: some cadence, weekly?) in CI.
-You can run it manually after generating the sitemap with a few steps:
-
-1. `yarn export` to generate the static pages for the site
-2. `yarn build:sitemap` to generate the sitemap for pages from step 1.
-3. `yarn export:serve` to host the static pages locally
-4. `yarn test:playwright:a11y` to run the scan. This runs `playwright/tests/a11y.spec.js` which loops over the sitemap and tests each page individually using `@axe-core/playwright`.
+- A full site scan of all urls known to next-build (generated in the sitemap) using Playwright:
+
+### Full Site Scan
+
+The full scan will run daily in CI using a GitHub Workflow and multiple
+runners, but you can also run it locally for testing and debugging purposes.
+
+- Workflow file: `.github/workflows/a11y.yml`
+- Test file: `playwright/tests/a11y.spec.js`
+- Yarn command with plenty of env var config options: `yarn 
+test:playwright:a11y`
+
+#### GitHub Workflow
+
+We designed the scan to run against the production va.gov/sitemap.xml to
+ensure accessibility issues facing actual users are caught. Testing locally
+and in lower environments is great, but to get the most bang for your buck,
+running it on the actual output ensures that any fixes carry through to
+production.
+
+The workflow file `.github/workflows/a11y.yml` has all the contents of the
+test run which currently uses 64 runners in a matrix. Each runner passes in
+a `SEGMENT_INDEX` that the test uses to split the sitemap into multiple
+pages based on the total number of runners used.
+
+Several environmental variables control different parts of the test
+configuration and can be found at the top-level of the `playwright-tests` job.
+
+- `BASE_URL`: Where to go look for "/sitemap.xml"
+- `USE_PROXY`: Whether to use the proxy in the custom fetcher. We don't want
+  to use it for the a11y scan.
+- `PW_BROWSER`: An array of Playwright browser types. These are randomly
+  read per runner and used as `PW_BROWSER_VALUE` in test config.
+- `PW_WIDTH`: An array of viewport widths to test against. These are randomly
+  read per runner and used as `PW_WIDTH_VALUE` in test config.
+- `PW_HEIGHT`: An array of viewport widths to test against. These are randomly
+  read per runner and used as `PW_WIDTH_VALUE` in test config. The viewport
+  height might not matter for a11y testing, I'm not sure.
+- `TOTAL_SEGMENTS`: The number of total segments. This was in the matrix
+  variables but pulled out since it is always one value and not a list of
+  values.
+
+You can alter those values to be picked up by each test run.
+
+Random values are picked in the "Run command with random values from arrays"
+step using some shell commands and `jq` foo. Then, those are used in the
+test run as env vars with `_VALUE` as a suffix.
+
+More detail about configuration and how the env vars are used is located in the
+[Test Config](#test-config) section.
+
+After the run the JSON output from the test run is uploaded into a file with
+the segment name included, e.g. `segment-${{ matrix.shardIndex }}.json`.
+
+To deal with an issue where pages were redirecting when Axe was trying to
+scan them, we added a `try/catch` to log those to a separate
+`failed-pages-segment-${{ matrix.shardIndex }}.json` report. Some pages were
+still trouble so those are excluded from the scan entirely.
+
+Then the `merge-reports` job waits for all the runners to finish before
+concatenating the results of both reports and making sure the final JSON
+output is valid.
+
+In the future, the report will be uploaded and sent to who needs it, but for
+now you can download it from GitHub on the Actions summary view for the a11y
+test workflow.
+
+#### Test Config
+
+The `playwright.config.ts` file contains all the Playwright config used for
+any test using Playwright in the next-build repo. We set up a different
+Playwright Project named `a11y` to filter tests for accessibility as well as
+provide different configuration. Let's go over the configuration differences:
+
+- `retries` - The test performs no assertions and retrying would scan all
+  the pages again. Plus, there was a weird error on GitHub where the URLs
+  were being scanned multiple times. Might be worth revisiting in the future.
+- `browserName`: Allows processing `PW_BROWSER_VALUE` to run different browser
+  types.
+- `trace`: No need to trace for code coverage.
+- `screenshot`: Taking screenshots slowed down the tests and isn't necessary
+  to report violations
+- `video`: No need to take video
+- `viewport`: Allows `PW_WIDTH_VALUE` and `PW_HEIGHT_VALUE`
+
+It might be useful to look over the Playwright config and see if there are
+more config options that would be helpful to use in testing.
+
+#### The Full Scan "Test"
+
+The actual test doesn't actually run any assertions. The reason for this is
+that we are scanning many pages in one test. We could split it up so each
+URL is run in an isolated test but that would require more setup and
+teardown costing time. Also...just being honest...passing data to each test
+via async/await was finicky, and I never went back to try and investigate
+further.
+
+Some pages end up redirecting
+
+Test run workflow:
+
+1. Get pages of the sitemap.
+2. Split the pages into segments if the segment index is not zero. This
+   allows to run against all pages locally or use segment indexes on GH.
+3. Pages are looped through and navigated to as long as the page isn't in
+   the list of excluded pages.
+4. `AxeBuilder` analyzes each page.
+5. If violations are found, they are pushed into an array with the URL and
+   violations as keys.
+6. If an error occurs, then catch it and log to a separate failed pages array.
+7. After all pages are scanned, the final results are written to a JSON file
+   for pages scanned and failures. The root element is stripped to make
+   merging the reports easier as well as adding a trailing comma.
+
+#### Local Testing
+
+You don't have to build a site locally, but if you want to test against a
+fresh next-build instance, you can follow these steps:
+
+1. Run all the steps needed to set up next-build listed in the root README.
+   md file.
+2. `yarn export` to generate the static pages for the site
+3. `yarn build:sitemap` to generate the sitemap for pages from step 1.
+4. `yarn export:serve` to host the static pages locally
+5. Take note of the port being used for the `BASE_URL` variable you will
+   pass in.
+6. `BASE_URL=${your-url} yarn test:playwright:a11y` to run the scan. This runs
+   `playwright/tests/a11y.spec.js` which loops over the sitemap and tests each page individually using `@axe-core/playwright`.
+
+You should also add the config values you want locally to end up with
+something like: `BASE_URL=${...} USE_PROXY=false PW_WIDTH_VALUE=768 
+PW_HEIGHT_VALUE=720 PW_BROWSER_VALUE=firefox yarn playwright test --project=a11y
+`

package.json

@@ -33,7 +33,8 @@
     "test:format": "APP_ENV=test prettier --check .",
     "test:playwright": "playwright test --project=main",
     "test:playwright:interactive": "playwright test --ui",
-    "test:playwright:a11y": "playwright test --project=a11y",
+    "test:playwright:a11y:prod": "BASE_URL=https://va.gov USE_PROXY=false PW_WIDTH_VALUE=768 PW_HEIGHT_VALUE=720 PW_BROWSER_VALUE=chromium yarn playwright test --project=a11y",
+    "test:playwright:a11y:local": "USE_PROXY=false PW_WIDTH_VALUE=768 PW_HEIGHT_VALUE=720 PW_BROWSER_VALUE=chromium yarn playwright test --project=a11y",
     "test:types": "yarn build:env-loader && APP_ENV=test tsc --noEmit",
     "storybook": "storybook dev -p 6006",
     "storybook:build": "storybook build -o out/storybook-static --quiet",

playwright.config.ts

@@ -1,10 +1,10 @@
-import { defineConfig } from '@playwright/test'
+import { defineConfig, ViewportSize } from '@playwright/test'
 
 export default defineConfig({
   testDir: './playwright/tests',
   outputDir: './playwright/test-results',
   reporter: [['html', { outputFolder: './playwright/test-report' }]],
-  fullyParallel: process.env.CI ? false : true,
+  fullyParallel: !process.env.CI,
 
   // Fail the build on CI if you accidentally left test.only in the source code.
   forbidOnly: !!process.env.CI,
@@ -39,6 +39,21 @@ export default defineConfig({
     {
       name: 'a11y',
       testMatch: /.a11y.spec.js/,
+      // No retries since no assertions are made.
+      retries: 0,
+      timeout: 4200000,
+      use: {
+        browserName:
+          (process.env.PW_BROWSER_VALUE as 'chromium' | 'firefox' | 'webkit') ||
+          'chromium',
+        trace: 'off',
+        screenshot: 'off',
+        video: 'off',
+        viewport: {
+          width: Number(process.env.PW_WIDTH_VALUE),
+          height: Number(process.env.PW_HEIGHT_VALUE),
+        } as ViewportSize,
+      },
     },
   ],
 })