From 086e043856ad54cf13cb65f9ba4c63afe8a22cc3 Mon Sep 17 00:00:00 2001
From: marv <rotrobmin@gmail.com>
Date: Sun, 27 Oct 2024 10:22:28 -0700
Subject: [PATCH] Playwright Comment on Flaky Tests (#1531)

* Playwright Comment on Flaky Tests

* More specific regex

* Fix if statement
---
 .github/workflows/playwright_comment.yml | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/playwright_comment.yml b/.github/workflows/playwright_comment.yml
index a118f980b6..992ffcea5f 100644
--- a/.github/workflows/playwright_comment.yml
+++ b/.github/workflows/playwright_comment.yml
@@ -47,6 +47,7 @@ jobs:
           cat ./master-screenshots-outcome >> $GITHUB_OUTPUT
           echo "$EOF" >> $GITHUB_OUTPUT
           echo "FAILED=$(grep -c '^ *[0-9] failed$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT
+          echo "FLAKY=$(grep -c '^ *[0-9] flaky$' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT
 
       # this is required because github.event.workflow_run.pull_requests is not available for PRs from forks
       - name: "Get PR information"
@@ -64,16 +65,15 @@ jobs:
           message: |
             :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "master-screenshots" step in the workflow run and rerun it before merging.
 
-      - name: "[Comment] Success: No visual differences introduced by this PR"
+      - name: "[Comment] Warning: Flaky tests caused by this PR; please check the playwright report"
         uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2
-        if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FAILED == 0
+        if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FLAKY != 0
         with:
           issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
           message: |
-            :heavy_check_mark: No visual differences introduced by this PR.
+            :warning: Flaky tests caused by this PR; please check the playwright report.
 
             <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}#artifacts">View Playwright Report</a> (note: open the "playwright-report" artifact)
-          update-only: true
 
       - name: "[Comment] Warning: Visual differences introduced by this PR"
         uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2
@@ -84,3 +84,14 @@ jobs:
             :warning: Visual differences introduced by this PR; please validate if they are desirable.
 
             <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}#artifacts">View Playwright Report</a> (note: open the "playwright-report" artifact)
+
+      - name: "[Comment] Success: No visual differences introduced by this PR"
+        uses: mshick/add-pr-comment@dd126dd8c253650d181ad9538d8b4fa218fc31e8 # pin@v2
+        if: steps.playwright.outputs.MASTER_SCREENSHOTS_OUTCOME != 'failure' && steps.playwright.outputs.FAILED == 0
+        with:
+          issue: ${{ steps.source-run-info.outputs.pullRequestNumber }}
+          message: |
+            :heavy_check_mark: No visual differences introduced by this PR.
+
+            <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}#artifacts">View Playwright Report</a> (note: open the "playwright-report" artifact)
+          update-only: true