chore(ci): shard e2e components tests (#7581)

### Description

<!--
What changes are introduced?
Why are these changes introduced?
What issue(s) does this solve? (with link, if possible)
-->

This PR shards e2e components tests so they run on 2 shards per browser.
This should significantly speed up the test run.

### What to review

<!--
What steps should the reviewer take in order to review?
What parts/flows of the application/packages/tooling is affected?
-->

Changes makes sense and the tests pass

### Testing

<!--
Did you add sufficient testing for this change?
If not, please explain how you tested this change and why it was not
possible/practical for writing an automated test
-->

N/A

### Notes for release

<!--
Engineers do not need to worry about the final copy,
but they must provide the docs team with enough context on:

* What changed
* How does one use it (code snippets, etc)
* Are there limitations we should be aware of

If this is PR is a partial implementation of a feature and is not
enabled by default or if
this PR does not contain changes that needs mention in the release notes
(tooling chores etc),
please call this out explicitly by writing "Part of feature X" or "Not
required" in this section.
-->

N/A

.github/workflows/e2e-ct.yml

@@ -6,12 +6,82 @@ on:
   push:
     branches: [next]
 jobs:
-  test:
+  install:
     timeout-minutes: 30
     runs-on: ubuntu-latest
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
+        project: [chromium, firefox, webkit]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}-
+            v1-${{ runner.os }}-pnpm-store-
+            v1-${{ runner.os }}-
+
+      - name: Install project dependencies
+        run: pnpm install
+
+      - name: Build packages
+        run: pnpm build --output-logs=full --log-order=grouped
+
+      # Caches build from either PR or next
+      - name: Cache build
+        id: cache-e2e-build
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-e2e-build
+        with:
+          path: |
+            ./*
+            !**/node_modules/**
+          # Unique key for a workflow run. Should be invalidated in the next run
+          key: ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }}
+
+  playwright-ct-test:
+    timeout-minutes: 30
+    needs: [install]
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
+        project: [chromium, firefox, webkit]
+        # Add more shards here if needed
+        shardIndex: [1, 2]
+        shardTotal: [2]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -58,38 +128,145 @@ jobs:
           path: ~/.cache/ms-playwright
           key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }}
       - name: Install Playwright Browsers
+        # TODO: Fix webkit caching when downloading from cache
+        # for some reason it doesn't work without installing again
+        # if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
         run: npx playwright install --with-deps
-      - name: Build packages
-        run: pnpm build --output-logs=full --log-order=grouped
+
+      - name: Restore build cache
+        uses: actions/cache/restore@v4
+        id: restore-build
+        env:
+          cache-name: cache-e2e-build
+        with:
+          path: |
+            ./*
+            !**/node_modules/**
+          key: ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }}
+          # If the cached build from the pervious step is not available. Fail the build
+          fail-on-cache-miss: true
+
       - name: Run end-to-end tests
-        run: pnpm --filter sanity test:ct
         env:
+          PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }}
           NODE_OPTIONS: --max_old_space_size=8192
+        run: pnpm --filter sanity test:ct --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-ct-report
+          path: ${{ github.workspace }}/packages/sanity/blob-report
+          retention-days: 30
+
+  merge-reports:
+    if: always()
+    needs: [playwright-ct-test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}-
+            v1-${{ runner.os }}-pnpm-store-
+            v1-${{ runner.os }}-
+
+      - name: Install project dependencies
+        run: pnpm install
+
+      - name: Download blob reports from Github Actions Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: playwright-ct-report
+          path: ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report
+
+      - name: Merge into HTML Report
+        run: npx playwright merge-reports --reporter json ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report >> ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report/playwright-ct-test-results.json
+
       - name: Get Current Job Log URL
         uses: Tiryoh/gha-jobid-action@c1d1cf7334b70c29374ae382b91053674c8049a2 # v1
         id: job_html_url
         if: always()
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          job_name: "test"
+          # Get the first run's ID even though it might be correct it should bring to the right place
+          job_name: "playwright-ct-test (chromium, 1, 2)"
+
       - name: Parse playwright report
         id: parse-playwright-report
         if: always()
         run: pnpm --filter sanity write:playwright-report-as-pr-comment
         env:
           GITHUB_WORKFLOW_URL: "${{ steps.job_html_url.outputs.html_url }}"
+
       - name: PR comment with report
         uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2
         if: ${{ always() && steps.parse-playwright-report.conclusion == 'success' && github.event_name == 'pull_request' }}
         with:
           comment_tag: "playwright-ct-report"
-          filePath: ${{ github.workspace }}/packages/sanity/playwright-ct/results/playwright-report-pr-comment.md
+          filePath: ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report/playwright-report-pr-comment.md
 
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: playwright-ct-report
           path: |
             ${{ github.workspace }}/packages/sanity/playwright-ct/report
+            ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report
             ${{ github.workspace }}/packages/sanity/playwright-ct/results
           retention-days: 30
+
+  cleanup:
+    permissions:
+      contents: read
+      actions: write # needed to delete the cache
+    timeout-minutes: 30
+    name: Cleanup (${{ matrix.project }})
+    runs-on: ubuntu-latest
+    needs: [playwright-ct-test]
+
+    strategy:
+      # we want to know if a test fails on a specific node version
+      fail-fast: false
+      matrix:
+        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
+        project: [chromium, firefox, webkit]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      # Delete the cache so it is only used once
+      - name: Delete Cache
+        run: gh cache delete ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }}
+        env:
+          cache-name: cache-e2e-build
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

packages/sanity/.gitignore

@@ -26,6 +26,7 @@
 /playwright-ct/report
 /playwright-ct/results
 /playwright-ct/template/.cache/
+/playwright-ct/playwright-ct-report
 
 # Deleted legacy exports
 /migrate.js

packages/sanity/playwright-ct.config.ts

@@ -25,16 +25,18 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: [
-    ['list'],
-    ['html', {outputFolder: HTML_REPORT_PATH}],
-    [
-      'json',
-      {
-        outputFile: path.join(ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json'),
-      },
-    ],
-  ],
+  reporter: isCI
+    ? [['list'], ['blob']]
+    : [
+        ['list'],
+        ['html', {outputFolder: HTML_REPORT_PATH}],
+        [
+          'json',
+          {
+            outputFile: path.join(ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json'),
+          },
+        ],
+      ],
 
   /* Maximum time one test can run for. */
   timeout: 30 * 1000,

packages/sanity/playwright-ct/scripts/parsePlaywrightReportJson.ts

@@ -7,7 +7,9 @@ import {inspect} from 'node:util'
 import {generateOutput, groupTests, readJsonFile} from './utils'
 
 const DEBUG = Boolean(parseInt(process.env.DEBUG || '0', 2))
-const DEFAULT_ARTIFACT_OUTPUT_PATH = path.resolve(path.join(__dirname, '..', 'results'))
+const DEFAULT_ARTIFACT_OUTPUT_PATH = path.resolve(
+  path.join(__dirname, '..', 'playwright-ct-report'),
+)
 
 /**
  * Summarize the Playwright report JSON into a Markdown string that can be posted as a comment to PRs