diff --git a/.github/workflows/playwright_comment.yml b/.github/workflows/playwright_comment.yml
index 013eb2517..9a9938e24 100644
--- a/.github/workflows/playwright_comment.yml
+++ b/.github/workflows/playwright_comment.yml
@@ -21,7 +21,7 @@ jobs:
egress-policy: audit
- name: Grab playwright-output from PR run
- uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # pin@v2
+ uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
continue-on-error: true
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
@@ -29,20 +29,39 @@ jobs:
name: playwright-output
- name: Grab master-screenshots-outcome from PR run
- uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # pin@v2
+ uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
continue-on-error: true
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ github.event.workflow_run.id }}
name: master-screenshots-outcome
+
- name: Grab playwright-results-json from PR run
- uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # pin@v2
+ uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
continue-on-error: true
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ github.event.workflow_run.id }}
name: playwright-results-json
+ - name: Dry-run grab playwright-report from PR run so we have its ID
+ uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
+ id: playwright-report
+ continue-on-error: true
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ run_id: ${{ github.event.workflow_run.id }}
+ name: playwright-report
+ dry_run: true
+
+ - name: Store playwright-report ID
+ id: playwright-report-artifact-id
+ env:
+ ARTIFACTS_JSON: ${{ steps.playwright-report.outputs.artifacts }}
+ run: |
+ ID=$(echo "$ARTIFACTS_JSON" | jq -r '.[0].id');
+ echo "id=$ID" >> "$GITHUB_OUTPUT"
+
- name: Generate summary from playwright-results.json (expected to fail to comment)
id: playwright-summary
uses: daun/playwright-report-summary@v3
@@ -65,32 +84,59 @@ jobs:
} >> "$GITHUB_OUTPUT"
# this is required because github.event.workflow_run.pull_requests is not available for PRs from forks
- - name: "Get PR information"
- uses: potiuk/get-workflow-origin@e2dae063368361e4cd1f510e8785cd73bca9352e # pin@v1_5
+ - name: Get PR context
id: source-run-info
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- sourceRunId: ${{ github.event.workflow_run.id }}
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Find the most recently updated open PR at the repo with the requested commit:
+ SEARCH_QUERY: >-
+ type:pr state:open sort:updated-desc
+ repo:${{ github.repository }}
+ ${{ github.event.workflow_run.head_sha }}
+ # Minimal graphql search query to fetch the PR `number` field:
+ GQL: |-
+ query($filter: String!) {
+ search( query: $filter, type: ISSUE, first: 1) {
+ nodes { ... on PullRequest { number } }
+ }
+ }
+ # Formats the GQL response into a `key=value` string + basic error handling
+ JQ_FILTER: >-
+ .data.search.nodes[0]
+ | if (.number == null) then error("Could not find PR number") end
+ | "pullRequestNumber=\(.number)"
+ run: |
+ gh api graphql --field "filter=$SEARCH_QUERY" --raw-field "query=$GQL" --jq "$JQ_FILTER" >> "${GITHUB_OUTPUT}"
- name: "[Comment] Couldn't download screenshots from master branch"
- uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2
+ uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME == 'failure'
with:
issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
message: |
:heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging.
- - name: "[Comment] Warning: Flaky tests or visual differences caused by this PR; please check the playwright report"
- uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2
- if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && (steps.playwright.outputs.FLAKY != 0 || steps.playwright.outputs.FAILED != 0)
+ - name: "[Comment] Warning: Visual differences caused by this PR; please check the playwright report"
+ uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
+ if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FAILED != 0
+ with:
+ issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
+ message: |
+ :warning: Visual changes detected by playwright; please check the report to verify if they are desirable.
+
+ - name: "[Comment] Success (but flaky): No visual differences introduced by this PR (but flaky tests detected)"
+ uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
+ if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY != 0 && steps.playwright.outputs.FAILED == 0
with:
issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- message: ${{ steps.playwright-summary.outputs.summary }}
+ message: |
+ :heavy_check_mark: No visual changes detected by playwright, but flaky tests were detected; please try to fix the tests.
- name: "[Comment] Success: No visual differences introduced by this PR"
- uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2
+ uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2
if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY == 0 && steps.playwright.outputs.FAILED == 0
with:
issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- message: ${{ steps.playwright-summary.outputs.summary }}
+ message: |
+ :heavy_check_mark: No visual changes detected by playwright.
update-only: true