diff --git a/.github/workflows/playwright_comment.yml b/.github/workflows/playwright_comment.yml index 013eb2517..9a9938e24 100644 --- a/.github/workflows/playwright_comment.yml +++ b/.github/workflows/playwright_comment.yml @@ -21,7 +21,7 @@ jobs: egress-policy: audit - name: Grab playwright-output from PR run - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # pin@v2 + uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 continue-on-error: true with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -29,20 +29,39 @@ jobs: name: playwright-output - name: Grab master-screenshots-outcome from PR run - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # pin@v2 + uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 continue-on-error: true with: github_token: ${{ secrets.GITHUB_TOKEN }} run_id: ${{ github.event.workflow_run.id }} name: master-screenshots-outcome + - name: Grab playwright-results-json from PR run - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # pin@v2 + uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 continue-on-error: true with: github_token: ${{ secrets.GITHUB_TOKEN }} run_id: ${{ github.event.workflow_run.id }} name: playwright-results-json + - name: Dry-run grab playwright-report from PR run so we have its ID + uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 + id: playwright-report + continue-on-error: true + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ github.event.workflow_run.id }} + name: playwright-report + dry_run: true + + - name: Store playwright-report ID + id: playwright-report-artifact-id + env: + ARTIFACTS_JSON: ${{ steps.playwright-report.outputs.artifacts }} + run: | + ID=$(echo "$ARTIFACTS_JSON" | jq -r '.[0].id'); + echo "id=$ID" >> "$GITHUB_OUTPUT" + - name: Generate summary from playwright-results.json (expected to fail to comment) id: playwright-summary uses: daun/playwright-report-summary@v3 @@ -65,32 +84,59 @@ jobs: } >> "$GITHUB_OUTPUT" # this is required because github.event.workflow_run.pull_requests is not available for PRs from forks - - name: "Get PR information" - uses: potiuk/get-workflow-origin@e2dae063368361e4cd1f510e8785cd73bca9352e # pin@v1_5 + - name: Get PR context id: source-run-info - with: - token: ${{ secrets.GITHUB_TOKEN }} - sourceRunId: ${{ github.event.workflow_run.id }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Find the most recently updated open PR at the repo with the requested commit: + SEARCH_QUERY: >- + type:pr state:open sort:updated-desc + repo:${{ github.repository }} + ${{ github.event.workflow_run.head_sha }} + # Minimal graphql search query to fetch the PR `number` field: + GQL: |- + query($filter: String!) { + search( query: $filter, type: ISSUE, first: 1) { + nodes { ... on PullRequest { number } } + } + } + # Formats the GQL response into a `key=value` string + basic error handling + JQ_FILTER: >- + .data.search.nodes[0] + | if (.number == null) then error("Could not find PR number") end + | "pullRequestNumber=\(.number)" + run: | + gh api graphql --field "filter=$SEARCH_QUERY" --raw-field "query=$GQL" --jq "$JQ_FILTER" >> "${GITHUB_OUTPUT}" - name: "[Comment] Couldn't download screenshots from master branch" - uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure' with: issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} message: | :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging. - - name: "[Comment] Warning: Flaky tests or visual differences caused by this PR; please check the playwright report" - uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 - if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && (steps.playwright.outputs.FLAKY != 0 || steps.playwright.outputs.FAILED != 0) + - name: "[Comment] Warning: Visual differences caused by this PR; please check the playwright report" + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 + if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FAILED != 0 + with: + issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} + message: | + :warning: Visual changes detected by playwright; please check the report to verify if they are desirable. + + - name: "[Comment] Success (but flaky): No visual differences introduced by this PR (but flaky tests detected)" + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 + if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY != 0 && steps.playwright.outputs.FAILED == 0 with: issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - message: ${{ steps.playwright-summary.outputs.summary }} + message: | + :heavy_check_mark: No visual changes detected by playwright, but flaky tests were detected; please try to fix the tests. - name: "[Comment] Success: No visual differences introduced by this PR" - uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0 with: issue: ${{ steps.source-run-info.outputs.pullRequestNumber }} - message: ${{ steps.playwright-summary.outputs.summary }} + message: | + :heavy_check_mark: No visual changes detected by playwright. update-only: true