diff --git a/.github/workflows/e2e-ct.yml b/.github/workflows/e2e-ct.yml index 8d001df4ce1..6e967aa5659 100644 --- a/.github/workflows/e2e-ct.yml +++ b/.github/workflows/e2e-ct.yml @@ -6,12 +6,82 @@ on: push: branches: [next] jobs: - test: + install: timeout-minutes: 30 runs-on: ubuntu-latest env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} + strategy: + fail-fast: false + matrix: + # Be sure to update all instances in this file and `pr-cleanup.yml` if updated + project: [chromium, firefox, webkit] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - uses: pnpm/action-setup@v4 + name: Install pnpm + id: pnpm-install + with: + run_install: false + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Cache node modules + id: cache-node-modules + uses: actions/cache@v4 + env: + cache-name: cache-node-modules + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}- + v1-${{ runner.os }}-pnpm-store- + v1-${{ runner.os }}- + + - name: Install project dependencies + run: pnpm install + + - name: Build packages + run: pnpm build --output-logs=full --log-order=grouped + + # Caches build from either PR or next + - name: Cache build + id: cache-e2e-build + uses: actions/cache@v4 + env: + cache-name: cache-e2e-build + with: + path: | + ./* + !**/node_modules/** + # Unique key for a workflow run. Should be invalidated in the next run + key: ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }} + + playwright-ct-test: + timeout-minutes: 30 + needs: [install] + runs-on: ubuntu-latest + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + strategy: + fail-fast: false + matrix: + # Be sure to update all instances in this file and `pr-cleanup.yml` if updated + project: [chromium, firefox, webkit] + # Add more shards here if needed + shardIndex: [1, 2] + shardTotal: [2] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -58,32 +128,106 @@ jobs: path: ~/.cache/ms-playwright key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }} - name: Install Playwright Browsers + # TODO: Fix webkit caching when downloading from cache + # for some reason it doesn't work without installing again + # if: steps.cache-playwright-browsers.outputs.cache-hit != 'true' run: npx playwright install --with-deps - - name: Build packages - run: pnpm build --output-logs=full --log-order=grouped + + - name: Restore build cache + uses: actions/cache/restore@v4 + id: restore-build + env: + cache-name: cache-e2e-build + with: + path: | + ./* + !**/node_modules/** + key: ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }} + # If the cached build from the pervious step is not available. Fail the build + fail-on-cache-miss: true + - name: Run end-to-end tests - run: pnpm --filter sanity test:ct env: + PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }} NODE_OPTIONS: --max_old_space_size=8192 + run: pnpm --filter sanity test:ct --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-ct-report + path: ${{ github.workspace }}/packages/sanity/blob-report + retention-days: 30 + + merge-reports: + if: always() + needs: [playwright-ct-test] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + + - uses: pnpm/action-setup@v4 + name: Install pnpm + id: pnpm-install + with: + run_install: false + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - name: Cache node modules + id: cache-node-modules + uses: actions/cache@v4 + env: + cache-name: cache-node-modules + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}- + v1-${{ runner.os }}-pnpm-store- + v1-${{ runner.os }}- + + - name: Install project dependencies + run: pnpm install + + - name: Download blob reports from Github Actions Artifacts + uses: actions/download-artifact@v3 + with: + name: playwright-ct-report + path: ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report + + - name: Merge into HTML Report + run: npx playwright merge-reports --reporter json ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report >> ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report/playwright-ct-test-results.json + - name: Get Current Job Log URL uses: Tiryoh/gha-jobid-action@c1d1cf7334b70c29374ae382b91053674c8049a2 # v1 id: job_html_url if: always() with: github_token: ${{ secrets.GITHUB_TOKEN }} - job_name: "test" + # Get the first run's ID even though it might be correct it should bring to the right place + job_name: "playwright-ct-test (chromium, 1, 2)" + - name: Parse playwright report id: parse-playwright-report if: always() run: pnpm --filter sanity write:playwright-report-as-pr-comment env: GITHUB_WORKFLOW_URL: "${{ steps.job_html_url.outputs.html_url }}" + - name: PR comment with report uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2 if: ${{ always() && steps.parse-playwright-report.conclusion == 'success' && github.event_name == 'pull_request' }} with: comment_tag: "playwright-ct-report" - filePath: ${{ github.workspace }}/packages/sanity/playwright-ct/results/playwright-report-pr-comment.md + filePath: ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report/playwright-report-pr-comment.md - uses: actions/upload-artifact@v3 if: always() @@ -91,5 +235,38 @@ jobs: name: playwright-ct-report path: | ${{ github.workspace }}/packages/sanity/playwright-ct/report + ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report ${{ github.workspace }}/packages/sanity/playwright-ct/results retention-days: 30 + + cleanup: + permissions: + contents: read + actions: write # needed to delete the cache + timeout-minutes: 30 + name: Cleanup (${{ matrix.project }}) + runs-on: ubuntu-latest + needs: [playwright-ct-test] + + strategy: + # we want to know if a test fails on a specific node version + fail-fast: false + matrix: + # Be sure to update all instances in this file and `pr-cleanup.yml` if updated + project: [chromium, firefox, webkit] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 18 + + # Delete the cache so it is only used once + - name: Delete Cache + run: gh cache delete ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }} + env: + cache-name: cache-e2e-build + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/packages/sanity/.gitignore b/packages/sanity/.gitignore index 5ab7703b1c9..e97750a7b8c 100644 --- a/packages/sanity/.gitignore +++ b/packages/sanity/.gitignore @@ -26,6 +26,7 @@ /playwright-ct/report /playwright-ct/results /playwright-ct/template/.cache/ +/playwright-ct/playwright-ct-report # Deleted legacy exports /migrate.js diff --git a/packages/sanity/playwright-ct.config.ts b/packages/sanity/playwright-ct.config.ts index 2a56c987491..158091b1e66 100644 --- a/packages/sanity/playwright-ct.config.ts +++ b/packages/sanity/playwright-ct.config.ts @@ -25,16 +25,18 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: [ - ['list'], - ['html', {outputFolder: HTML_REPORT_PATH}], - [ - 'json', - { - outputFile: path.join(ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json'), - }, - ], - ], + reporter: isCI + ? [['list'], ['blob']] + : [ + ['list'], + ['html', {outputFolder: HTML_REPORT_PATH}], + [ + 'json', + { + outputFile: path.join(ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json'), + }, + ], + ], /* Maximum time one test can run for. */ timeout: 30 * 1000, diff --git a/packages/sanity/playwright-ct/scripts/parsePlaywrightReportJson.ts b/packages/sanity/playwright-ct/scripts/parsePlaywrightReportJson.ts index 69693423fa7..8745bb2ab93 100644 --- a/packages/sanity/playwright-ct/scripts/parsePlaywrightReportJson.ts +++ b/packages/sanity/playwright-ct/scripts/parsePlaywrightReportJson.ts @@ -7,7 +7,9 @@ import {inspect} from 'node:util' import {generateOutput, groupTests, readJsonFile} from './utils' const DEBUG = Boolean(parseInt(process.env.DEBUG || '0', 2)) -const DEFAULT_ARTIFACT_OUTPUT_PATH = path.resolve(path.join(__dirname, '..', 'results')) +const DEFAULT_ARTIFACT_OUTPUT_PATH = path.resolve( + path.join(__dirname, '..', 'playwright-ct-report'), +) /** * Summarize the Playwright report JSON into a Markdown string that can be posted as a comment to PRs