From c60fbe6238e6ff23b13c6cd71de3ab087741e674 Mon Sep 17 00:00:00 2001 From: Niklas Berglund Date: Thu, 27 Jun 2024 14:15:15 +0200 Subject: [PATCH] Change the way iOS end to end tests workflows are set up --- .../action.yml | 40 ++-- .github/actions/run-ios-e2e-tests/action.yml | 74 ++++++++ .../workflows/ios-end-to-end-tests-api.yml | 9 + .../ios-end-to-end-tests-merge-to-main.yml | 17 ++ .../ios-end-to-end-tests-nightly.yml | 16 ++ ...os-end-to-end-tests-settings-migration.yml | 18 +- .github/workflows/ios-end-to-end-tests.yml | 178 ++++++++++++------ .github/workflows/ios.yml | 28 ++- ios/Configurations/UITests.xcconfig.template | 6 + .../xcschemes/MullvadVPNUITests.xcscheme | 12 -- .../Base/BaseUITestCase.swift | 49 ++++- ios/MullvadVPNUITests/ConnectivityTests.swift | 5 +- ios/MullvadVPNUITests/Info.plist | 4 + .../Pages/SelectLocationPage.swift | 7 + ios/MullvadVPNUITests/tests.json | 19 ++ 15 files changed, 369 insertions(+), 113 deletions(-) rename .github/actions/{ios-end-to-end-tests => build-ios-e2e-tests}/action.yml (72%) create mode 100644 .github/actions/run-ios-e2e-tests/action.yml create mode 100644 .github/workflows/ios-end-to-end-tests-api.yml create mode 100644 .github/workflows/ios-end-to-end-tests-merge-to-main.yml create mode 100644 .github/workflows/ios-end-to-end-tests-nightly.yml create mode 100644 ios/MullvadVPNUITests/tests.json diff --git a/.github/actions/ios-end-to-end-tests/action.yml b/.github/actions/build-ios-e2e-tests/action.yml similarity index 72% rename from .github/actions/ios-end-to-end-tests/action.yml rename to .github/actions/build-ios-e2e-tests/action.yml index fa2d198a44a5..4d65cfbe3682 100644 --- a/.github/actions/ios-end-to-end-tests/action.yml +++ b/.github/actions/build-ios-e2e-tests/action.yml @@ -1,5 +1,5 @@ -name: 'iOS end to end tests action' -description: 'Prepares and runs end to end tests on iOS device' +name: 'Build iOS end to end tests action' +description: 'Prepares and builds end to end tests on iOS device' inputs: ios_device_pin_code: description: 'iOS Device Pin Code' @@ -16,25 +16,19 @@ inputs: test_device_udid: description: 'Test Device UDID' required: true - xcode_test_plan: - description: 'Xcode Test Plan to run' - required: true partner_api_token: description: 'Partner API Token' required: true test_name: description: 'Test case/suite name. Will run all tests in the test plan if not provided.' required: false + outputs_path: + description: 'Path to store outputs. This should be unique for each job run in order to avoid concurrency issues.' + required: true runs: using: 'composite' steps: - - name: Make sure app is not installed - run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN - shell: bash - env: - IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} - - name: Configure Xcode project run: | for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done @@ -46,12 +40,21 @@ runs: sed -i "" \ "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \ UITests.xcconfig + sed -i "" \ + "s#^// PARTNER_API_TOKEN =#PARTNER_API_TOKEN =#" \ + UITests.xcconfig sed -i "" \ "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \ UITests.xcconfig sed -i "" \ "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \ UITests.xcconfig + sed -i "" \ + "/TEST_DEVICE_IS_IPAD =/ s#= .*#= 0#" \ + UITests.xcconfig + sed -i "" \ + "/UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN =/ s#= .*#= 0#" \ + UITests.xcconfig shell: bash working-directory: ios/Configurations env: @@ -61,7 +64,7 @@ runs: NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }} PARTNER_API_TOKEN: ${{ inputs.partner_api_token }} - - name: Run end-to-end-tests + - name: Build app and tests for testing run: | if [ -n "$TEST_NAME" ]; then TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME" @@ -71,19 +74,12 @@ runs: set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ -project MullvadVPN.xcodeproj \ -scheme MullvadVPNUITests \ - -testPlan $XCODE_TEST_PLAN $TEST_NAME_ARGUMENT \ - -resultBundlePath xcode-test-report \ + -testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT \ -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ - clean test 2>&1 | xcbeautify --report junit --report-path junit-test-report + -derivedDataPath derived-data \ + clean build-for-testing 2>&1 shell: bash working-directory: ios/ env: - XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }} TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} TEST_NAME: ${{ inputs.test_name }} - - - name: Uninstall app if still installed - run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN - shell: bash - env: - IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} diff --git a/.github/actions/run-ios-e2e-tests/action.yml b/.github/actions/run-ios-e2e-tests/action.yml new file mode 100644 index 000000000000..4c6634085a56 --- /dev/null +++ b/.github/actions/run-ios-e2e-tests/action.yml @@ -0,0 +1,74 @@ +name: 'Run iOS end to end tests action' +description: 'Runs end to end tests on iOS device' +inputs: + test_name: + description: 'Test case/suite name. Will run all tests in the test plan if not provided.' + required: false + test_device_udid: + description: 'Test Device UDID' + required: true + outputs_path: + description: > + Path to where outputs are stored - both build outputs and outputs from running tests. + This should be unique for each job run in order to avoid concurrency issues. + required: true + +runs: + using: 'composite' + steps: + # Set up a unique output directory + - name: Set up outputs directory + run: | + if [ -n "$TEST_NAME" ]; then + # Strip slashes to avoid creating subdirectories + test_name_sanitized=$(printf "$TEST_NAME" | sed 's/\//_/g') + echo "Setting output directory tests-output-test-name-sanitized" + echo "$test_name_sanitized" + test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output-$test_name_sanitized" + else + echo "Setting output directory output" + test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output" + fi + + echo "TEST_OUTPUT_DIRECTORY=$test_output_directory" >> $GITHUB_ENV + echo "TEST_NAME_SANITIZED=$test_name_sanitized" >> $GITHUB_ENV + shell: bash + env: + TEST_NAME: ${{ inputs.test_name }} + OUTPUTS_PATH: ${{ inputs.outputs_path }} + + - name: Uninstall app + run: ios-deploy --id $TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN + shell: bash + env: + TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + + - name: Run end-to-end-tests + run: > + if [ -n "$TEST_NAME" ]; then TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"; else TEST_NAME_ARGUMENT=""; fi + + set -o pipefail && env NSUnbufferedIO=YES xcodebuild + -project MullvadVPN.xcodeproj + -scheme MullvadVPNUITests + -testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT + -resultBundlePath ${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report + -derivedDataPath derived-data + -destination "platform=iOS,id=$TEST_DEVICE_UDID" + test-without-building 2>&1 | xcbeautify --report junit + --report-path ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report + shell: bash + working-directory: ${{ inputs.outputs_path }}/mullvadvpn-app/ios + env: + TEST_NAME: ${{ inputs.test_name }} + TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + + - name: Store test report artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.TEST_NAME_SANITIZED }}-test-results + path: | + ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report/junit.xml + ${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report.xcresult + env: + TEST_NAME: ${{ inputs.test_name }} diff --git a/.github/workflows/ios-end-to-end-tests-api.yml b/.github/workflows/ios-end-to-end-tests-api.yml new file mode 100644 index 000000000000..bdb873a80b93 --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests-api.yml @@ -0,0 +1,9 @@ +--- +name: iOS end-to-end API tests +on: + workflow_dispatch: +jobs: + reuse-e2e-workflow: + uses: ./.github/workflows/ios-end-to-end-tests.yml + with: + arg_tests_json_key: "api-tests" diff --git a/.github/workflows/ios-end-to-end-tests-merge-to-main.yml b/.github/workflows/ios-end-to-end-tests-merge-to-main.yml new file mode 100644 index 000000000000..5cfe49095316 --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests-merge-to-main.yml @@ -0,0 +1,17 @@ +--- +name: iOS end-to-end merge to main tests +on: + workflow_dispatch: + pull_request: + types: + - closed + branches: + - main + paths: + - .github/workflows/ios-end-to-end-tests*.yml + - ios/** +jobs: + reuse-e2e-workflow: + uses: ./.github/workflows/ios-end-to-end-tests.yml + with: + arg_tests_json_key: "pr-merge-to-main" diff --git a/.github/workflows/ios-end-to-end-tests-nightly.yml b/.github/workflows/ios-end-to-end-tests-nightly.yml new file mode 100644 index 000000000000..4539f90df70d --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests-nightly.yml @@ -0,0 +1,16 @@ +--- +name: iOS end-to-end nightly tests +on: + workflow_dispatch: + schedule: + # At midnight every day. + # Notifications for scheduled workflows are sent to the user who last modified the cron + # syntax in the workflow file. If you update this you must have notifications for + # Github Actions enabled, so these don't go unnoticed. + # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs + - cron: '0 0 * * *' +jobs: + reuse-e2e-workflow: + uses: ./.github/workflows/ios-end-to-end-tests.yml + with: + arg_tests_json_key: "nightly" diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml index 2ff8e1a2bad2..9c187dfab6fd 100644 --- a/.github/workflows/ios-end-to-end-tests-settings-migration.yml +++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml @@ -1,5 +1,8 @@ --- name: iOS settings migration tests +concurrency: + group: ios-end-to-end-tests + cancel-in-progress: false permissions: contents: read on: @@ -11,6 +14,8 @@ on: # Github Actions enabled, so these don't go unnoticed. # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs - cron: '0 0 * * *' +env: + TEST_DEVICE_UDID: 00008130-0019181022F3803A jobs: test: name: Settings migration end to end tests @@ -19,14 +24,15 @@ jobs: OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280 steps: - name: Configure Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable override: true target: aarch64-apple-ios - name: Uninstall app - run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN + timeout-minutes: 5 + run: ios-deploy --id $${{ env.TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN - name: Checkout old repository version uses: actions/checkout@v4 @@ -40,7 +46,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings' partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} @@ -62,7 +68,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged' @@ -86,7 +92,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsChangeSettings' @@ -108,7 +114,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged' diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml index 8d1c8178d072..60e63e3bf75d 100644 --- a/.github/workflows/ios-end-to-end-tests.yml +++ b/.github/workflows/ios-end-to-end-tests.yml @@ -1,72 +1,156 @@ --- name: iOS end-to-end tests +env: + TEST_DEVICE_UDID: 00008130-0019181022F3803A permissions: contents: read issues: write pull-requests: write on: - pull_request: - types: - - closed - branches: - - main - paths: - - .github/workflows/ios-end-to-end-tests.yml - - ios/** + workflow_call: + inputs: + arg_tests_json_key: + type: string + required: false workflow_dispatch: inputs: # Optionally specify a test case or suite to run. # Must be in the format MullvadVPNUITest// where test case name is optional. - test_name: + user_supplied_test_name: description: 'Only run test case/suite' required: false - schedule: - # At midnight every day. - # Notifications for scheduled workflows are sent to the user who last modified the cron - # syntax in the workflow file. If you update this you must have notifications for - # Github Actions enabled, so these don't go unnoticed. - # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs - - cron: '0 0 * * *' jobs: - test: - if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' - name: End to end tests + set-up-outputs-directory: + name: Set up outputs directory runs-on: [self-hosted, macOS, ios-test] - timeout-minutes: 60 + outputs: + job_outputs_directory: ${{ steps.set-up-job-outputs-directory.outputs.job_outputs_directory }} + steps: + - name: Set up job outputs directory + id: set-up-job-outputs-directory + run: | + job_outputs_directory="$HOME/workflow-outputs/job-outputs-${{ github.run_id }}" + echo "job_outputs_directory=$job_outputs_directory" >> "$GITHUB_OUTPUT" + mkdir -p "$job_outputs_directory" + + shell: bash + + # Define the set of tests to run based on the event type and input + define-test-suites-matrix: + name: Define test suites matrix + runs-on: [self-hosted, macOS, ios-test] + needs: set-up-outputs-directory steps: + - name: Test runs to JSON + id: test-runs-to-json + run: | + if [ -n "${{ inputs.arg_tests_json_key }}" ]; then + # JSON key supplied by another workflow calling this reusable workflow + echo "Using calling workflow supplied test suites JSON key: ${{ inputs.arg_tests_json_key }}" + test_suites_json=$(jq -r --compact-output '.tests."${{ inputs.arg_tests_json_key }}"' tests.json) + echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV + elif [ -n "${{ inputs.user_supplied_test_name }}" ]; then + # User specified test case/suite when manually triggering run + echo "Using user supplied test name: ${{ inputs.user_supplied_test_name }}" + test_suites_json="['${{ inputs.user_supplied_test_name }}']" >> $GITHUB_ENV + echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV + else + echo "Tests not specified, will fallback to running nightly(all) tests scope" + test_suites_json=$(jq -r --compact-output '.tests.nightly' tests.json) + echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV + fi + + echo "Test suites/cases to run: $test_suites_json" + working-directory: ios/MullvadVPNUITests + outputs: + test_suites_json: ${{ env.test_suites_json }} + + # Build app and tests target + build: + name: Build for end to end testing + runs-on: [self-hosted, macOS, ios-test] + needs: set-up-outputs-directory + timeout-minutes: 20 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + clean: true + - name: Configure Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable override: true target: aarch64-apple-ios - - name: Checkout repository - uses: actions/checkout@v4 - - name: Select test plan to execute - run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "XCODE_TEST_PLAN=MullvadVPNUITestsSmoke" >> $GITHUB_ENV - elif [[ "${{ github.event_name }}" == "schedule" ]]; then - echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV - elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV - fi - - - name: iOS end to end tests action - uses: ./.github/actions/ios-end-to-end-tests + - name: Build iOS end to end tests action + uses: ./.github/actions/build-ios-e2e-tests with: - xcode_test_plan: ${{ env.XCODE_TEST_PLAN }} - test_name: ${{ github.event.inputs.test_name }} + test_name: ${{ github.event.inputs.user_supplied_test_name }} ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} + outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + + - name: Debug print job output directory + run: | + echo "Job output directory: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }}" + shell: bash + + - name: Copy build output and project to output directory + run: | + mkdir -p "$JOB_OUTPUTS_DIRECTORY/derived-data" + cp -R ios/derived-data "$JOB_OUTPUTS_DIRECTORY" + cp -R . $JOB_OUTPUTS_DIRECTORY/mullvadvpn-app + shell: bash + env: + JOB_OUTPUTS_DIRECTORY: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + + - name: Clean up + run: | + rm -rf ios/Configurations/*.xcconfig + rm -rf ios/derived-data + shell: bash + + test: + name: Run tests + runs-on: [self-hosted, macOS, ios-test] + needs: [build, define-test-suites-matrix, set-up-outputs-directory] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + test_suite: ${{fromJson(needs.define-test-suites-matrix.outputs.test_suites_json)}} + steps: + - name: Run iOS end to end tests action + uses: ./.github/actions/run-ios-e2e-tests + with: + test_name: "MullvadVPNUITests/${{ matrix.test_suite }}" + test_device_udid: ${{ env.TEST_DEVICE_UDID }} + outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + + clean-up-outputs-directory: + if: always() + name: Clean up outputs directory + runs-on: [self-hosted, macOS, ios-test] + needs: [test, set-up-outputs-directory] + steps: + - name: Clean up outputs directory + run: rm -rf ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + shell: bash + notify-on-failure: + if: failure() && github.event_name == 'pull_request' + name: Notify team on failure(if PR related) + runs-on: [self-hosted, macOS, ios-test] + needs: test + timeout-minutes: 5 + steps: - name: Comment PR on test failure - if: failure() && github.event_name == 'pull_request' uses: actions/github-script@v7 with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -80,19 +164,3 @@ jobs: issue_number: issue_number, body: `🚨 End to end tests failed. Please check the [failed workflow run](${run_url}).` }); - - - name: Store test report artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results - path: | - ios/junit-test-report/junit.xml - ios/xcode-test-report.xcresult - - - name: Store app log artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: app-logs - path: ios/xcode-test-report/**/app-log-*.log diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index b2f733fea38d..078ccf2d9e89 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -64,6 +64,16 @@ jobs: with: go-version: 1.20.14 + - name: Install xcbeautify + run: | + brew update + brew install xcbeautify + + - name: Install protobuf + run: | + brew update + brew install protobuf + - name: Set up yeetd to workaround XCode being slow in CI run: | wget https://github.com/biscuitehh/yeetd/releases/download/1.0/yeetd-normal.pkg @@ -74,7 +84,15 @@ jobs: with: xcode-version: '15.0.1' - name: Configure Rust + # Since the https://github.com/actions/runner-images/releases/tag/macos-13-arm64%2F20240721.1 release + # Brew does not install tools at the correct location anymore + # This update broke the rust build script which was assuming the cargo binary was located in ~/.cargo/bin/cargo + # The workaround is to fix brew paths by running brew bundle dump, and then brew bundle + # WARNING: This has to be the last brew "upgrade" commands that is ran, + # otherwise the brew path will be broken again. run: | + brew bundle dump + brew bundle rustup default stable rustup update stable rustup target add aarch64-apple-ios-sim @@ -88,16 +106,6 @@ jobs: cp Api.xcconfig.template Api.xcconfig working-directory: ios/Configurations - - name: Install xcbeautify - run: | - brew update - brew install xcbeautify - - - name: Install protobuf - run: | - brew update - brew install protobuf - - name: Run unit tests run: | set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ diff --git a/ios/Configurations/UITests.xcconfig.template b/ios/Configurations/UITests.xcconfig.template index 45688d626f15..af31a89b2664 100644 --- a/ios/Configurations/UITests.xcconfig.template +++ b/ios/Configurations/UITests.xcconfig.template @@ -6,6 +6,12 @@ IOS_DEVICE_PIN_CODE = // UUID to identify test runs. Should be unique per test device. Generate with for example uuidgen on macOS. TEST_DEVICE_IDENTIFIER_UUID = +// Specify whether test device is an iPad or not +TEST_DEVICE_IS_IPAD = 0 + +// Uninstall app after each test suite finish running? +UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN = 1 + // Base64 encoded token for the partner API. Will only be used if account numbers are not configured. // PARTNER_API_TOKEN = diff --git a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme index 3258ef6f4722..48eba40cb235 100644 --- a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme +++ b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme @@ -16,9 +16,6 @@ reference = "container:TestPlans/MullvadVPNUITestsAll.xctestplan" default = "YES"> - - @@ -72,15 +69,6 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - - - - diff --git a/ios/MullvadVPNUITests/Base/BaseUITestCase.swift b/ios/MullvadVPNUITests/Base/BaseUITestCase.swift index a4b3a7d4e200..0df61d39f235 100644 --- a/ios/MullvadVPNUITests/Base/BaseUITestCase.swift +++ b/ios/MullvadVPNUITests/Base/BaseUITestCase.swift @@ -44,6 +44,23 @@ class BaseUITestCase: XCTestCase { .infoDictionary?["AttachAppLogsOnFailure"] as! String == "1" // swiftlint:enable force_cast + static func testDeviceIsIPad() -> Bool { + if let testDeviceIsIPad = Bundle(for: BaseUITestCase.self).infoDictionary?["TestDeviceIsIPad"] as? String { + return testDeviceIsIPad == "1" + } + + return false + } + + static func uninstallAppInTestSuiteTearDown() -> Bool { + if let uninstallAppInTestSuiteTearDown = Bundle(for: BaseUITestCase.self) + .infoDictionary?["UninstallAppInTestSuiteTearDown"] as? String { + return uninstallAppInTestSuiteTearDown == "1" + } + + return false + } + /// Get an account number with time. If an account with time is specified in the configuration file that account will be used, else a temporary account will be created if partner API token has been configured. func getAccountWithTime() -> String { if let configuredAccountWithTime = bundleHasTimeAccountNumber, !configuredAccountWithTime.isEmpty { @@ -134,7 +151,7 @@ class BaseUITestCase: XCTestCase { /// Suite level teardown ran after all tests in suite have been executed override class func tearDown() { - if shouldUninstallAppInTeardown() { + if shouldUninstallAppInTeardown() && uninstallAppInTestSuiteTearDown() { uninstallApp() } } @@ -280,17 +297,35 @@ class BaseUITestCase: XCTestCase { let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") let spotlight = XCUIApplication(bundleIdentifier: "com.apple.Spotlight") - springboard.swipeDown() - spotlight.textFields["SpotlightSearchField"].typeText(searchQuery) + /// iPhone uses spotlight, iPad uses springboard. But the usage is quite similar + let spotlightOrSpringboard = BaseUITestCase.testDeviceIsIPad() ? springboard : spotlight + var mullvadAppIcon: XCUIElement + + // How to navigate to Spotlight search differs between iPhone and iPad + if BaseUITestCase.testDeviceIsIPad() == false { // iPhone + springboard.swipeDown() + spotlight.textFields["SpotlightSearchField"].typeText(searchQuery) + mullvadAppIcon = spotlightOrSpringboard.icons[appName] + } else { // iPad + // Swipe left enough times to reach the last page + for _ in 0 ..< 3 { + springboard.swipeLeft() + Thread.sleep(forTimeInterval: 0.5) + } + + springboard.swipeDown() + springboard.searchFields.firstMatch.typeText(searchQuery) + mullvadAppIcon = spotlightOrSpringboard.icons.matching(identifier: appName).allElementsBoundByIndex[1] + } - let appIcon = spotlight.icons[appName].firstMatch - if appIcon.waitForExistence(timeout: timeout) { - appIcon.press(forDuration: 2) + // The rest of the delete app flow is same for iPhone and iPad with the exception that iPhone uses spotlight and iPad uses springboard + if mullvadAppIcon.waitForExistence(timeout: timeout) { + mullvadAppIcon.press(forDuration: 2) } else { XCTFail("Failed to find app icon named \(appName)") } - let deleteAppButton = spotlight.buttons["Delete App"] + let deleteAppButton = spotlightOrSpringboard.buttons["Delete App"] if deleteAppButton.waitForExistence(timeout: timeout) { deleteAppButton.tap() } else { diff --git a/ios/MullvadVPNUITests/ConnectivityTests.swift b/ios/MullvadVPNUITests/ConnectivityTests.swift index 3b3ac28bdd03..a46042fddd9a 100644 --- a/ios/MullvadVPNUITests/ConnectivityTests.swift +++ b/ios/MullvadVPNUITests/ConnectivityTests.swift @@ -96,7 +96,10 @@ class ConnectivityTests: LoggedOutUITestCase { // Select the first country, its first city and its first relay SelectLocationPage(app) - .tapCountryLocationCellExpandButton(withIndex: 0) + .tapCountryLocationCellExpandButton( + withName: BaseUITestCase + .testsDefaultCountryName + ) // Must be a little specific here in order to avoid using relay services country with experimental relays .tapCityLocationCellExpandButton(withIndex: 0) .tapRelayLocationCell(withIndex: 0) diff --git a/ios/MullvadVPNUITests/Info.plist b/ios/MullvadVPNUITests/Info.plist index a81366999153..d477ab4de8e3 100644 --- a/ios/MullvadVPNUITests/Info.plist +++ b/ios/MullvadVPNUITests/Info.plist @@ -28,5 +28,9 @@ $(SHOULD_BE_REACHABLE_DOMAIN) TestDeviceIdentifier $(TEST_DEVICE_IDENTIFIER_UUID + TestDeviceIsIPad + $(TEST_DEVICE_IS_IPAD) + UninstallAppInTestSuiteTearDown + $(UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN) diff --git a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift index 95b9416c1c59..db401a0305c7 100644 --- a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift +++ b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift @@ -22,6 +22,13 @@ class SelectLocationPage: Page { return self } + @discardableResult func tapCountryLocationCellExpandButton(withName name: String) -> Self { + let cell = app.cells.containing(.any, identifier: name) + let expandButton = cell.buttons[AccessibilityIdentifier.expandButton] + expandButton.tap() + return self + } + @discardableResult func tapCountryLocationCellExpandButton(withIndex: Int) -> Self { let cell = app.cells.containing(.any, identifier: AccessibilityIdentifier.countryLocationCell.rawValue) .element(boundBy: withIndex) diff --git a/ios/MullvadVPNUITests/tests.json b/ios/MullvadVPNUITests/tests.json new file mode 100644 index 000000000000..19f5ec287a82 --- /dev/null +++ b/ios/MullvadVPNUITests/tests.json @@ -0,0 +1,19 @@ +{ + "tests": { + "nightly": [ + "AccountTests", + "ConnectivityTests", + "CustomListsTests", + "RelayTests", + "SettingsTests" + ], + "pr-merge-to-main": [ + "AccountTests/testLogin", + "AccountTests/testCreateAccount" + ], + "api-tests": [ + "AccountTests" + ] + } +} +