Split tests in multiple jobs

duckduckgo · Aug 15, 2024 · 17fba95 · 17fba95
1 parent 2d72b91
commit 17fba95
Showing 1 changed file with 50 additions and 26 deletions.
diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml
@@ -6,10 +6,11 @@ on:
   workflow_dispatch:
 
 jobs:
-  end-to-end-tests:
-    name: End to end Tests
+  build-end-to-end-tests:
+    name: Build End to End Tests
     runs-on: macos-14-xlarge
-
+    timeout-minutes: 30
+
     steps:
     - name: Check out the code
       uses: actions/checkout@v3
@@ -48,26 +49,59 @@ jobs:
           ONLY_ACTIVE_ARCH=NO \
         | tee xcodebuild.log
 
-    - name: Install Maestro
-      run: |
-        export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
+    - name: Store Binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: failure() || cancelled()
+      with:
+        name: BuildLogs
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+        retention-days: 7
 
-    - name: Release tests
-      run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=release DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+  end-to-end-tests:
+    name: End to end Tests
+    needs: build-end-to-end-tests
+    runs-on: macos-14-xlarge
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        test-tag: [release, privacy, securityTest, adClick]
+      max-parallel: 1 # Uncomment this line to run tests sequentially. 
+      fail-fast: false
 
-    - name: Privacy tests
-      run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=privacy DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+    steps:
+
+    - name: Check out the code
+      uses: actions/checkout@v3 # Don't need submodules here as this is only for the tests folder
+
+    - name: Retrieve Binary
+      uses: actions/download-artifact@v4
+      with:
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
 
-    - name: Security tests
+    - name: Install Maestro
       run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=securityTest DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+        export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
 
-    - name: Ad Click Detection Flow tests
+    - name: End to End tests
       run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=adClick DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=${{ matrix.test-tag }} DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
       
+  notify-failure:
+    name: Notify on failure
+    if: ${{ always() && contains(join(needs.*.result, ','), 'failure') && github.ref_name == 'main' }}
+    needs: [build-end-to-end-tests, end-to-end-tests]
+    runs-on: ubuntu-latest
+
+    steps:
     - name: Create Asana task when workflow failed
       if: ${{ failure() }}
       run: |
@@ -76,13 +110,3 @@ jobs:
           --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
           --header "Content-Type: application/json" \
           --data ' { "data": { "name": "GH Workflow Failure - End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}. For instructions on how to handle the failure(s), check https://app.asana.com/0/0/1206423571874502/f" } }'
-
-    - name: Upload logs when workflow failed
-      uses: actions/upload-artifact@v4
-      if: failure() || cancelled()
-      with:
-        name: BuildLogs
-        path: |
-          xcodebuild.log
-          DerivedData/Logs/Test/*.xcresult
-        retention-days: 7