Test/ds 980 e2e tests agains netlify builds

.github/workflows/e2e.yml

@@ -3,19 +3,41 @@ on:
   push:
     branches: [main]
   pull_request:
-    types: [labeled]
+    types:
+      - opened
+      - synchronize
+      - labeled
 
 env:
   NODE_ENV: testing
 
 jobs:
+  haslabel:
+    name: analyse labels
+    runs-on: ubuntu-latest
+    outputs:
+      e2e: ${{ steps.haslabel.outputs.labeled-run-visual-tests }}
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - uses: actions/checkout@v4
+      - name: Labeled to e2e
+        id: haslabel
+        uses: DanielTamkin/[email protected]
+        with:
+          contains: 'run-visual-tests'
+
   test:
-    if: ${{ (github.event.label.name == 'run-visual-tests') || (github.ref == 'refs/heads/main') }}
+    needs: hasLabel
+    if: ${{ needs.haslabel.outputs.e2e || (github.ref == 'refs/heads/main') }}
     runs-on: ubuntu-22.04
-    container:
-      image: mcr.microsoft.com/playwright:v1.40.1-jammy
     timeout-minutes: 60
 
+    env:
+      NETLIFY_DEPLOY_PREVIEW_URL: 'https://deploy-preview-${{ github.event.pull_request.number }}--spirit-design-system.netlify.app/'
+
     steps:
       - uses: actions/checkout@v4
 
@@ -29,15 +51,58 @@ jobs:
         with:
           useRollingCache: true
 
+      # Wait for the Netlify preview URL to be ready
+      - name: Wait for Netlify Deploy
+        uses: mmazzarolo/wait-for-netlify-action@8a7a8d8cf5b313c916d805b76cc498380062d268
+        id: get-netlify-preview-url
+        continue-on-error: true
+        with:
+          site_id: ${{ secrets.NETLIFY_SITE_SPIRIT_DESIGN_SYSTEM }}
+          max_timeout: 1
+        env:
+          NETLIFY_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
+      # The Netlify preview URL is now available
+      # as `steps.get-netlify-preview-url.outputs.url`
+
       - name: Run Playwright tests
-        run: HOME=/root yarn playwright test
+        uses: docker://mcr.microsoft.com/playwright:v1.42.1-jammy
+        with:
+          args: yarn playwright test
+        env:
+          WEBSITE_URL: ${{ steps.get-netlify-preview-url.outcome == 'success' && steps.get-netlify-preview-url.outputs.url || env.NETLIFY_DEPLOY_PREVIEW_URL }}
+
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        id: artifact-upload-step
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
 
-      - name: Notify slack if tests fail
+      - name: Comment PR with artifact URL
+        if: failure()
+        uses: actions/github-script@v5
+        with:
+          script: |
+            const issue_number = context.issue.number;
+            const repo = context.repo.repo;
+            const owner = context.repo.owner;
+
+            await github.rest.issues.createComment({
+              owner,
+              repo,
+              issue_number,
+              body: `Here is the URL of the uploaded artifact: ${{ steps.artifact-upload-step.outputs.artifact-url }}`
+            });
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Notify Slack when failure
         id: Slack
         uses: slackapi/[email protected]
-        if: ${{ failure() }}
+        if: failure()
         with:
-          channel-id: 'C050BHERJTW' # Slack channel-id of #spirit-design-system-notifications_en
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }} # Slack channel-id of #spirit-design-system-notifications_en
           payload: |
             {
               "attachments": [
@@ -73,17 +138,25 @@ jobs:
                           "type": "button",
                           "text": {
                             "type": "plain_text",
-                            "text": ":github: Open pull request"
+                            "text": ":github: Open Pull request"
                           },
                           "url": "${{ github.event.pull_request.html_url }}"
                         },
                         {
                           "type": "button",
                           "text": {
                             "type": "plain_text",
-                            "text": ":github: Open failed github action"
+                            "text": ":github: Open failed GitHub Action"
                           },
                           "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                        },
+                        {
+                          "type": "button",
+                          "text": {
+                            "type": "plain_text",
+                            "text": ":github: Download Report"
+                          },
+                          "url": "${{ steps.artifact-upload-step.outputs.artifact-url }}"
                         }
                       ]
                     }
@@ -94,10 +167,3 @@ jobs:
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFICATIONS_TOKEN }} # Webhook from Slack Application
           SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
-
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 30

CONTRIBUTING.md

@@ -93,6 +93,7 @@ The following is the list of supported scopes:
   - `exporter-svg`
 - Packages:
   - `analytics`
+  - `codemods`
   - `design-tokens`
   - `form-validations`
   - `icons`
@@ -202,10 +203,6 @@ All commands will execute Docker command that starts [Playwright][playwright] in
 We run visual regression testing locally against our demo apps. Web and Web React packages are served using Vite.
 Web Twig package is served using Symfony app.
 
-In CI we use Netlify to test against.
-
-⚠️ Currently we do not deploy the Web Twig package to any environment, so you can only test it locally.
-
 We have two test suites and you can find them in the `./tests/e2e` directory:
 
 - `demo-homepages` - tests the homepages of our demo apps.
@@ -216,6 +213,28 @@ We have two test suites and you can find them in the `./tests/e2e` directory:
 
 👉 To save time and repository size, we test only in Chromium browser and only on desktop viewport.
 
+#### On CI pipeline
+
+In CI we use Netlify to test against.
+
+To enable visual regression testing on CI, you need to add label `run-visual-tests` to your PR.
+Then the tests will be run against the Netlify deployment on every change pushed.
+
+When the tests fail, there will be a comment in the PR with a link to the report and a message about the failure will be raised in the Slack channel `#spirit-design-system-notifications_en`.
+You can also find the report URL in the GitHub Actions run under the `Run actions/upload-artifact` section.
+Look for `Artifact download URL: https://github.com/lmc-eu/spirit-design-system/actions/runs/<run-id>/artifacts/<artifact-id>`.
+
+⚠️ Currently we do not deploy the Web Twig package to any environment, so you can only test it locally.
+
+### Developing and Testing GitHub Actions
+
+It can be time-consuming and painful to test GitHub Actions.
+First, you have to change the GitHub Actions file locally, push your local code into GitHub repository, and wait for the result.
+
+To solve this issue, you can use [act][act] CLI tool to test and write the GitHub actions locally.
+
+For more, please read the article [How to Run GitHub Actions Locally with the act CLI tool][act-article].
+
 ## Publishing
 
 This project uses GitHub Actions to publish the packages automatically to npm. New packages are published after the new tag is pushed to the main branch. PR can be merged only by the appropriate group of maintainers.
@@ -230,6 +249,8 @@ This project uses GitHub Actions to publish the packages automatically to npm. N
 
 > If you have further questions do not hesitate to open an issue and ask us! ❤️
 
+[act]: https://github.com/nektos/act
+[act-article]: https://www.freecodecamp.org/news/how-to-run-github-actions-locally/
 [conventional-commits]: https://www.conventionalcommits.org
 [lerna-home]: https://lernajs.io
 [commitlint-config]: https://github.com/lmc-eu/code-quality-tools/tree/main/packages/commitlint-config

apps/demo/netlify.toml

@@ -20,7 +20,7 @@
     yarn lerna run --scope @lmc-eu/spirit-design-tokens build --skip-nx-cache \
     && yarn lerna run --scope @lmc-eu/spirit-icons build --skip-nx-cache \
     && cp -r packages/icons/dist/svg/. static/assets/icons/svg \
-    && yarn lerna run --scope @lmc-eu/spirit-docs build --skip-nx-cache \
+    && yarn lerna run --scope @lmc-eu/spirit-demo-app build --skip-nx-cache \
     """
 
   # Ignore builds on Netlify

apps/demo/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@lmc-eu/spirit-docs",
+  "name": "@lmc-eu/spirit-demo-app",
   "version": "0.0.0",
   "description": "Demo platform of the Spirit Design System",
   "license": "MIT",