run e2e tests for beta-release unless skipped

.github/workflows/beta-release.yaml

@@ -5,12 +5,21 @@ on:
     inputs:
       # This is the branch that will trigger the workflow
       branch:
-        description: "Branch to build. Should be a release/* branch"
+        description: "Branch to publish. Should be a release/* branch"
         required: true
+      skip_tests:
+        description: "Skip the end-to-end tests"
+        required: true
+        default: false
 
 env:
   # Creates and uploads sourcemaps to Application error telemetry, and save the built extension as an artifact
   PUBLIC_RELEASE: true
+  CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs9/BXaFQsYPDxqbVvi11xhWdxfygfrF5YCLiboJooQyTkqIGpGxtI3JF/hkAXDcoqI+N5ATsGPYW34CdOc7uBCU91Ig+gHFiicnkzJaoOBjIwqx452l2/mp7cqNdavtCq40YENkF13ouj5loPwMMYY0L/sSvab+6eO20i1+Ulbsn9onS/fDd16clOaIbUVJ1PhyYvrU0HGVUqW5wUIDLyRezr3aTQLtDIQp/7DTBQ60S2G5KPpAW1UEphnXRLwl6cR5MiYw20OStfTZaA2qpWQvLAQtBoPNjP0Ld6rzI/e3uaC5qUMMCusitKeCA5HOFQDz2IJ0kS8Cn5fxzhXFi6QIDAQAB
+  MV: 3
+  PUBLIC_NAME: -beta
+  DATADOG_APPLICATION_ID: ${{ secrets.DATADOG_APPLICATION_ID }}
+  DATADOG_CLIENT_TOKEN: ${{ secrets.DATADOG_CLIENT_TOKEN }}
 
 jobs:
   build:
@@ -35,11 +44,6 @@ jobs:
           cache: npm
       - run: npm ci
       - run: npm run build
-        env:
-          MV: 3
-          PUBLIC_NAME: -beta
-          DATADOG_APPLICATION_ID: ${{ secrets.DATADOG_APPLICATION_ID }}
-          DATADOG_CLIENT_TOKEN: ${{ secrets.DATADOG_CLIENT_TOKEN }}
       - run: bash scripts/upload-sourcemaps.sh
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.SOURCEMAP_USER_ID }}
@@ -56,7 +60,7 @@ jobs:
           retention-days: 30
           if-no-files-found: error
       - name: Create production version
-        run: npx [email protected] dist/manifest.json key 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs9/BXaFQsYPDxqbVvi11xhWdxfygfrF5YCLiboJooQyTkqIGpGxtI3JF/hkAXDcoqI+N5ATsGPYW34CdOc7uBCU91Ig+gHFiicnkzJaoOBjIwqx452l2/mp7cqNdavtCq40YENkF13ouj5loPwMMYY0L/sSvab+6eO20i1+Ulbsn9onS/fDd16clOaIbUVJ1PhyYvrU0HGVUqW5wUIDLyRezr3aTQLtDIQp/7DTBQ60S2G5KPpAW1UEphnXRLwl6cR5MiYw20OStfTZaA2qpWQvLAQtBoPNjP0Ld6rzI/e3uaC5qUMMCusitKeCA5HOFQDz2IJ0kS8Cn5fxzhXFi6QIDAQAB'
+        run: npx [email protected] dist/manifest.json key '${{ env.CHROME_MANIFEST_KEY }}'
       - name: Save production extension
         uses: actions/upload-artifact@v4
         with:
@@ -65,9 +69,61 @@ jobs:
           retention-days: 30
           if-no-files-found: error
 
+  end-to-end-tests:
+    name: end-to-end-tests
+    if: ${{ github.event.inputs.skip_tests != 'true' }}
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    env:
+      SHADOW_DOM: open
+      SERVICE_URL: https://app.pixiebrix.com
+      REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST: 1
+      E2E_TEST_USER_EMAIL_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_EMAIL_UNAFFILIATED }}
+      E2E_TEST_USER_PASSWORD_UNAFFILIATED: ${{ secrets.E2E_TEST_USER_PASSWORD_UNAFFILIATED }}
+      DEV_EVENT_TELEMETRY: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: package.json
+          cache: npm
+      - name: Install dependencies
+        run: npm ci
+      - name: Install playwright browsers
+        uses: Wandalen/[email protected]
+        with:
+          command: npx playwright install chrome msedge
+          with: |
+            fail_ci_if_error: true
+            verbose: true
+            token: ${{ secrets.CODECOV_TOKEN }}
+          # Installing msedge is flaky due to failed checksums from the Microsoft CDN. Unclear why this happens.
+          # Retry every 15 seconds, for up to 10 minutes
+          attempt_delay: 15000
+          attempt_limit: 40
+      - name: Build the extension
+        run: npm run build:webpack
+      - name: Run end to end tests
+        # Xvfb is required to run the tests in headed mode. Headed mode is required to run tests for browser extensions
+        # in Playwright, see https://playwright.dev/docs/ci#running-headed
+        run: xvfb-run npm run test:e2e
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: end-to-end-tests-report
+          path: end-to-end-tests/.report
+          retention-days: 5
+
   publish:
     # https://github.com/fregante/chrome-webstore-upload-keys
-    needs: build
+    needs:
+      - build
+      - end-to-end-tests
+    # https://stackoverflow.com/a/69354134
+    if: |
+      always() &&
+      needs.build.result == 'success' &&
+      (needs.end-to-end-tests.result == 'success' || needs.end-to-end-tests.result == 'skipped')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v4

.github/workflows/ci.yml

@@ -189,17 +189,15 @@ jobs:
         include:
           - MV: 2
             PUBLIC_NAME: ""
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
           - MV: 3
             PUBLIC_NAME: "-mv3"
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs9/BXaFQsYPDxqbVvi11xhWdxfygfrF5YCLiboJooQyTkqIGpGxtI3JF/hkAXDcoqI+N5ATsGPYW34CdOc7uBCU91Ig+gHFiicnkzJaoOBjIwqx452l2/mp7cqNdavtCq40YENkF13ouj5loPwMMYY0L/sSvab+6eO20i1+Ulbsn9onS/fDd16clOaIbUVJ1PhyYvrU0HGVUqW5wUIDLyRezr3aTQLtDIQp/7DTBQ60S2G5KPpAW1UEphnXRLwl6cR5MiYw20OStfTZaA2qpWQvLAQtBoPNjP0Ld6rzI/e3uaC5qUMMCusitKeCA5HOFQDz2IJ0kS8Cn5fxzhXFi6QIDAQAB
     timeout-minutes: 60
     runs-on: ubuntu-latest
     env:
       SHADOW_DOM: open
       SERVICE_URL: https://app.pixiebrix.com
       MV: ${{ matrix.MV }}
-      CHROME_MANIFEST_KEY: ${{ matrix.CHROME_MANIFEST_KEY }}
+      CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
       DATADOG_APPLICATION_ID: ${{ secrets.DATADOG_APPLICATION_ID }}
       DATADOG_CLIENT_TOKEN: ${{ secrets.DATADOG_CLIENT_TOKEN }}
       REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST: 1

.github/workflows/release.yaml

@@ -9,6 +9,7 @@ on:
 env:
   # Creates and uploads sourcemaps to Application error telemetry, and save the built extension as an artifact
   PUBLIC_RELEASE: true
+  CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
 
 jobs:
   build:
@@ -25,10 +26,8 @@ jobs:
         include:
           - MV: 2
             PUBLIC_NAME: "-mv2"
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
           - MV: 3
             PUBLIC_NAME: "-mv3"
-            CHROME_MANIFEST_KEY: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAhYpgz6Nt3vv5n3d8jyrsWXjkvAxh7bz8WJW05RFrtJx9t0BiVVclO+WpAmhKanB2BiTDw4+Dnlf2lQfTo62LIBnkfTiGzukKqTu3plF0D/Tl/yG1st1xKaQ6dekeThcsgxrFD8+kIUwF4Vq0wPpQ5upl+vf6kX4t9eDev8Eg86mHzUEG/QoS/bu5evN3I1Z0HsiF84VWlrV0b/1GSqpn+dMrFFdcwo2Sn0Ec65nSNfzauDUm5n0NToQ8iYdHkuottREXKJ7/Uy4tO0eMmfokVixbm0i2m9aHEOior5CmNG9X/yGtR2CiM1N4DSEY5mTFu5hPOrALspJ+t7+Is7YnFwIDAQAB
 
     name: build${{ matrix.PUBLIC_NAME }}
     steps:
@@ -60,7 +59,7 @@ jobs:
           retention-days: 30
           if-no-files-found: error
       - name: Create production version
-        run: npx [email protected] dist/manifest.json key '${{ matrix.CHROME_MANIFEST_KEY }}'
+        run: npx [email protected] dist/manifest.json key '${{ env.CHROME_MANIFEST_KEY }}'
       - name: Save production extension
         uses: actions/upload-artifact@v4
         with: