diff --git a/.github/actions/ios-end-to-end-tests/action.yml b/.github/actions/build-ios-e2e-tests/action.yml similarity index 72% rename from .github/actions/ios-end-to-end-tests/action.yml rename to .github/actions/build-ios-e2e-tests/action.yml index fa2d198a44a5..4d65cfbe3682 100644 --- a/.github/actions/ios-end-to-end-tests/action.yml +++ b/.github/actions/build-ios-e2e-tests/action.yml @@ -1,5 +1,5 @@ -name: 'iOS end to end tests action' -description: 'Prepares and runs end to end tests on iOS device' +name: 'Build iOS end to end tests action' +description: 'Prepares and builds end to end tests on iOS device' inputs: ios_device_pin_code: description: 'iOS Device Pin Code' @@ -16,25 +16,19 @@ inputs: test_device_udid: description: 'Test Device UDID' required: true - xcode_test_plan: - description: 'Xcode Test Plan to run' - required: true partner_api_token: description: 'Partner API Token' required: true test_name: description: 'Test case/suite name. Will run all tests in the test plan if not provided.' required: false + outputs_path: + description: 'Path to store outputs. This should be unique for each job run in order to avoid concurrency issues.' + required: true runs: using: 'composite' steps: - - name: Make sure app is not installed - run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN - shell: bash - env: - IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} - - name: Configure Xcode project run: | for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done @@ -46,12 +40,21 @@ runs: sed -i "" \ "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \ UITests.xcconfig + sed -i "" \ + "s#^// PARTNER_API_TOKEN =#PARTNER_API_TOKEN =#" \ + UITests.xcconfig sed -i "" \ "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \ UITests.xcconfig sed -i "" \ "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \ UITests.xcconfig + sed -i "" \ + "/TEST_DEVICE_IS_IPAD =/ s#= .*#= 0#" \ + UITests.xcconfig + sed -i "" \ + "/UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN =/ s#= .*#= 0#" \ + UITests.xcconfig shell: bash working-directory: ios/Configurations env: @@ -61,7 +64,7 @@ runs: NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }} PARTNER_API_TOKEN: ${{ inputs.partner_api_token }} - - name: Run end-to-end-tests + - name: Build app and tests for testing run: | if [ -n "$TEST_NAME" ]; then TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME" @@ -71,19 +74,12 @@ runs: set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ -project MullvadVPN.xcodeproj \ -scheme MullvadVPNUITests \ - -testPlan $XCODE_TEST_PLAN $TEST_NAME_ARGUMENT \ - -resultBundlePath xcode-test-report \ + -testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT \ -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ - clean test 2>&1 | xcbeautify --report junit --report-path junit-test-report + -derivedDataPath derived-data \ + clean build-for-testing 2>&1 shell: bash working-directory: ios/ env: - XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }} TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} TEST_NAME: ${{ inputs.test_name }} - - - name: Uninstall app if still installed - run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN - shell: bash - env: - IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} diff --git a/.github/actions/run-ios-e2e-tests/action.yml b/.github/actions/run-ios-e2e-tests/action.yml new file mode 100644 index 000000000000..438112803ed9 --- /dev/null +++ b/.github/actions/run-ios-e2e-tests/action.yml @@ -0,0 +1,74 @@ +name: 'Run iOS end to end tests action' +description: 'Runs end to end tests on iOS device' +inputs: + test_name: + description: 'Test case/suite name. Will run all tests in the test plan if not provided.' + required: false + test_device_udid: + description: 'Test Device UDID' + required: true + outputs_path: + description: 'Path to where outputs are stored - both build outputs and outputs from running tests. This should be unique for each job run in order to avoid concurrency issues.' + required: true + +runs: + using: 'composite' + steps: + # Set up a unique output directory + - name: Set up outputs directory + run: | + if [ -n "$TEST_NAME" ]; then + # Strip slashes to avoid creating subdirectories + test_name_sanitized=$(printf "$TEST_NAME" | sed 's/\//_/g') + echo "Setting output directory tests-output-test-name-sanitized" + echo "$test_name_sanitized" + test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output-$test_name_sanitized" + else + echo "Setting output directory output" + test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output" + fi + + echo "TEST_OUTPUT_DIRECTORY=$test_output_directory" >> $GITHUB_ENV + echo "TEST_NAME_SANITIZED=$test_name_sanitized" >> $GITHUB_ENV + shell: bash + env: + TEST_NAME: ${{ inputs.test_name }} + OUTPUTS_PATH: ${{ inputs.outputs_path }} + + - name: Uninstall app + run: ios-deploy --id $TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN + shell: bash + env: + TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + + - name: Run end-to-end-tests + run: | + if [ -n "$TEST_NAME" ]; then + TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME" + else + TEST_NAME_ARGUMENT="" + fi + set -o pipefail && env NSUnbufferedIO=YES xcodebuild \ + -project MullvadVPN.xcodeproj \ + -scheme MullvadVPNUITests \ + -testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT \ + -resultBundlePath ${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report \ + -derivedDataPath derived-data \ + -destination "platform=iOS,id=$TEST_DEVICE_UDID" \ + test-without-building 2>&1 #| xcbeautify --report junit --report-path ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report + shell: bash + working-directory: ${{ inputs.outputs_path }}/mullvadvpn-app/ios + env: + TEST_NAME: ${{ inputs.test_name }} + TEST_DEVICE_UDID: ${{ inputs.test_device_udid }} + + - name: Store test report artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.TEST_NAME_SANITIZED }}-test-results + path: | + ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report/junit.xml + ${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report.xcresult + env: + TEST_NAME: ${{ inputs.test_name }} diff --git a/.github/workflows/ios-end-to-end-tests-api.yml b/.github/workflows/ios-end-to-end-tests-api.yml new file mode 100644 index 000000000000..bdb873a80b93 --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests-api.yml @@ -0,0 +1,9 @@ +--- +name: iOS end-to-end API tests +on: + workflow_dispatch: +jobs: + reuse-e2e-workflow: + uses: ./.github/workflows/ios-end-to-end-tests.yml + with: + arg_tests_json_key: "api-tests" diff --git a/.github/workflows/ios-end-to-end-tests-merge-to-main.yml b/.github/workflows/ios-end-to-end-tests-merge-to-main.yml new file mode 100644 index 000000000000..5cfe49095316 --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests-merge-to-main.yml @@ -0,0 +1,17 @@ +--- +name: iOS end-to-end merge to main tests +on: + workflow_dispatch: + pull_request: + types: + - closed + branches: + - main + paths: + - .github/workflows/ios-end-to-end-tests*.yml + - ios/** +jobs: + reuse-e2e-workflow: + uses: ./.github/workflows/ios-end-to-end-tests.yml + with: + arg_tests_json_key: "pr-merge-to-main" diff --git a/.github/workflows/ios-end-to-end-tests-nightly.yml b/.github/workflows/ios-end-to-end-tests-nightly.yml new file mode 100644 index 000000000000..4539f90df70d --- /dev/null +++ b/.github/workflows/ios-end-to-end-tests-nightly.yml @@ -0,0 +1,16 @@ +--- +name: iOS end-to-end nightly tests +on: + workflow_dispatch: + schedule: + # At midnight every day. + # Notifications for scheduled workflows are sent to the user who last modified the cron + # syntax in the workflow file. If you update this you must have notifications for + # Github Actions enabled, so these don't go unnoticed. + # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs + - cron: '0 0 * * *' +jobs: + reuse-e2e-workflow: + uses: ./.github/workflows/ios-end-to-end-tests.yml + with: + arg_tests_json_key: "nightly" diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml index 2ff8e1a2bad2..9c187dfab6fd 100644 --- a/.github/workflows/ios-end-to-end-tests-settings-migration.yml +++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml @@ -1,5 +1,8 @@ --- name: iOS settings migration tests +concurrency: + group: ios-end-to-end-tests + cancel-in-progress: false permissions: contents: read on: @@ -11,6 +14,8 @@ on: # Github Actions enabled, so these don't go unnoticed. # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs - cron: '0 0 * * *' +env: + TEST_DEVICE_UDID: 00008130-0019181022F3803A jobs: test: name: Settings migration end to end tests @@ -19,14 +24,15 @@ jobs: OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280 steps: - name: Configure Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable override: true target: aarch64-apple-ios - name: Uninstall app - run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN + timeout-minutes: 5 + run: ios-deploy --id $${{ env.TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN - name: Checkout old repository version uses: actions/checkout@v4 @@ -40,7 +46,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings' partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} @@ -62,7 +68,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged' @@ -86,7 +92,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsChangeSettings' @@ -108,7 +114,7 @@ jobs: test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged' diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml index 8d1c8178d072..60e63e3bf75d 100644 --- a/.github/workflows/ios-end-to-end-tests.yml +++ b/.github/workflows/ios-end-to-end-tests.yml @@ -1,72 +1,156 @@ --- name: iOS end-to-end tests +env: + TEST_DEVICE_UDID: 00008130-0019181022F3803A permissions: contents: read issues: write pull-requests: write on: - pull_request: - types: - - closed - branches: - - main - paths: - - .github/workflows/ios-end-to-end-tests.yml - - ios/** + workflow_call: + inputs: + arg_tests_json_key: + type: string + required: false workflow_dispatch: inputs: # Optionally specify a test case or suite to run. # Must be in the format MullvadVPNUITest// where test case name is optional. - test_name: + user_supplied_test_name: description: 'Only run test case/suite' required: false - schedule: - # At midnight every day. - # Notifications for scheduled workflows are sent to the user who last modified the cron - # syntax in the workflow file. If you update this you must have notifications for - # Github Actions enabled, so these don't go unnoticed. - # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs - - cron: '0 0 * * *' jobs: - test: - if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' - name: End to end tests + set-up-outputs-directory: + name: Set up outputs directory runs-on: [self-hosted, macOS, ios-test] - timeout-minutes: 60 + outputs: + job_outputs_directory: ${{ steps.set-up-job-outputs-directory.outputs.job_outputs_directory }} + steps: + - name: Set up job outputs directory + id: set-up-job-outputs-directory + run: | + job_outputs_directory="$HOME/workflow-outputs/job-outputs-${{ github.run_id }}" + echo "job_outputs_directory=$job_outputs_directory" >> "$GITHUB_OUTPUT" + mkdir -p "$job_outputs_directory" + + shell: bash + + # Define the set of tests to run based on the event type and input + define-test-suites-matrix: + name: Define test suites matrix + runs-on: [self-hosted, macOS, ios-test] + needs: set-up-outputs-directory steps: + - name: Test runs to JSON + id: test-runs-to-json + run: | + if [ -n "${{ inputs.arg_tests_json_key }}" ]; then + # JSON key supplied by another workflow calling this reusable workflow + echo "Using calling workflow supplied test suites JSON key: ${{ inputs.arg_tests_json_key }}" + test_suites_json=$(jq -r --compact-output '.tests."${{ inputs.arg_tests_json_key }}"' tests.json) + echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV + elif [ -n "${{ inputs.user_supplied_test_name }}" ]; then + # User specified test case/suite when manually triggering run + echo "Using user supplied test name: ${{ inputs.user_supplied_test_name }}" + test_suites_json="['${{ inputs.user_supplied_test_name }}']" >> $GITHUB_ENV + echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV + else + echo "Tests not specified, will fallback to running nightly(all) tests scope" + test_suites_json=$(jq -r --compact-output '.tests.nightly' tests.json) + echo "test_suites_json=$test_suites_json" >> $GITHUB_ENV + fi + + echo "Test suites/cases to run: $test_suites_json" + working-directory: ios/MullvadVPNUITests + outputs: + test_suites_json: ${{ env.test_suites_json }} + + # Build app and tests target + build: + name: Build for end to end testing + runs-on: [self-hosted, macOS, ios-test] + needs: set-up-outputs-directory + timeout-minutes: 20 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + clean: true + - name: Configure Rust - uses: actions-rs/toolchain@v1 + uses: actions-rs/toolchain@v1.0.6 with: toolchain: stable override: true target: aarch64-apple-ios - - name: Checkout repository - uses: actions/checkout@v4 - - name: Select test plan to execute - run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - echo "XCODE_TEST_PLAN=MullvadVPNUITestsSmoke" >> $GITHUB_ENV - elif [[ "${{ github.event_name }}" == "schedule" ]]; then - echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV - elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV - fi - - - name: iOS end to end tests action - uses: ./.github/actions/ios-end-to-end-tests + - name: Build iOS end to end tests action + uses: ./.github/actions/build-ios-e2e-tests with: - xcode_test_plan: ${{ env.XCODE_TEST_PLAN }} - test_name: ${{ github.event.inputs.test_name }} + test_name: ${{ github.event.inputs.user_supplied_test_name }} ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }} test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }} has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }} no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }} - test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }} + test_device_udid: ${{ env.TEST_DEVICE_UDID }} partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }} + outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + + - name: Debug print job output directory + run: | + echo "Job output directory: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }}" + shell: bash + + - name: Copy build output and project to output directory + run: | + mkdir -p "$JOB_OUTPUTS_DIRECTORY/derived-data" + cp -R ios/derived-data "$JOB_OUTPUTS_DIRECTORY" + cp -R . $JOB_OUTPUTS_DIRECTORY/mullvadvpn-app + shell: bash + env: + JOB_OUTPUTS_DIRECTORY: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + + - name: Clean up + run: | + rm -rf ios/Configurations/*.xcconfig + rm -rf ios/derived-data + shell: bash + + test: + name: Run tests + runs-on: [self-hosted, macOS, ios-test] + needs: [build, define-test-suites-matrix, set-up-outputs-directory] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + test_suite: ${{fromJson(needs.define-test-suites-matrix.outputs.test_suites_json)}} + steps: + - name: Run iOS end to end tests action + uses: ./.github/actions/run-ios-e2e-tests + with: + test_name: "MullvadVPNUITests/${{ matrix.test_suite }}" + test_device_udid: ${{ env.TEST_DEVICE_UDID }} + outputs_path: ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + + clean-up-outputs-directory: + if: always() + name: Clean up outputs directory + runs-on: [self-hosted, macOS, ios-test] + needs: [test, set-up-outputs-directory] + steps: + - name: Clean up outputs directory + run: rm -rf ${{ needs.set-up-outputs-directory.outputs.job_outputs_directory }} + shell: bash + notify-on-failure: + if: failure() && github.event_name == 'pull_request' + name: Notify team on failure(if PR related) + runs-on: [self-hosted, macOS, ios-test] + needs: test + timeout-minutes: 5 + steps: - name: Comment PR on test failure - if: failure() && github.event_name == 'pull_request' uses: actions/github-script@v7 with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -80,19 +164,3 @@ jobs: issue_number: issue_number, body: `🚨 End to end tests failed. Please check the [failed workflow run](${run_url}).` }); - - - name: Store test report artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-results - path: | - ios/junit-test-report/junit.xml - ios/xcode-test-report.xcresult - - - name: Store app log artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: app-logs - path: ios/xcode-test-report/**/app-log-*.log diff --git a/ios/Configurations/UITests.xcconfig.template b/ios/Configurations/UITests.xcconfig.template index 45688d626f15..af31a89b2664 100644 --- a/ios/Configurations/UITests.xcconfig.template +++ b/ios/Configurations/UITests.xcconfig.template @@ -6,6 +6,12 @@ IOS_DEVICE_PIN_CODE = // UUID to identify test runs. Should be unique per test device. Generate with for example uuidgen on macOS. TEST_DEVICE_IDENTIFIER_UUID = +// Specify whether test device is an iPad or not +TEST_DEVICE_IS_IPAD = 0 + +// Uninstall app after each test suite finish running? +UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN = 1 + // Base64 encoded token for the partner API. Will only be used if account numbers are not configured. // PARTNER_API_TOKEN = diff --git a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme index 3258ef6f4722..48eba40cb235 100644 --- a/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme +++ b/ios/MullvadVPN.xcodeproj/xcshareddata/xcschemes/MullvadVPNUITests.xcscheme @@ -16,9 +16,6 @@ reference = "container:TestPlans/MullvadVPNUITestsAll.xctestplan" default = "YES"> - - @@ -72,15 +69,6 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - - - - diff --git a/ios/MullvadVPNUITests/Base/BaseUITestCase.swift b/ios/MullvadVPNUITests/Base/BaseUITestCase.swift index a4b3a7d4e200..0df61d39f235 100644 --- a/ios/MullvadVPNUITests/Base/BaseUITestCase.swift +++ b/ios/MullvadVPNUITests/Base/BaseUITestCase.swift @@ -44,6 +44,23 @@ class BaseUITestCase: XCTestCase { .infoDictionary?["AttachAppLogsOnFailure"] as! String == "1" // swiftlint:enable force_cast + static func testDeviceIsIPad() -> Bool { + if let testDeviceIsIPad = Bundle(for: BaseUITestCase.self).infoDictionary?["TestDeviceIsIPad"] as? String { + return testDeviceIsIPad == "1" + } + + return false + } + + static func uninstallAppInTestSuiteTearDown() -> Bool { + if let uninstallAppInTestSuiteTearDown = Bundle(for: BaseUITestCase.self) + .infoDictionary?["UninstallAppInTestSuiteTearDown"] as? String { + return uninstallAppInTestSuiteTearDown == "1" + } + + return false + } + /// Get an account number with time. If an account with time is specified in the configuration file that account will be used, else a temporary account will be created if partner API token has been configured. func getAccountWithTime() -> String { if let configuredAccountWithTime = bundleHasTimeAccountNumber, !configuredAccountWithTime.isEmpty { @@ -134,7 +151,7 @@ class BaseUITestCase: XCTestCase { /// Suite level teardown ran after all tests in suite have been executed override class func tearDown() { - if shouldUninstallAppInTeardown() { + if shouldUninstallAppInTeardown() && uninstallAppInTestSuiteTearDown() { uninstallApp() } } @@ -280,17 +297,35 @@ class BaseUITestCase: XCTestCase { let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard") let spotlight = XCUIApplication(bundleIdentifier: "com.apple.Spotlight") - springboard.swipeDown() - spotlight.textFields["SpotlightSearchField"].typeText(searchQuery) + /// iPhone uses spotlight, iPad uses springboard. But the usage is quite similar + let spotlightOrSpringboard = BaseUITestCase.testDeviceIsIPad() ? springboard : spotlight + var mullvadAppIcon: XCUIElement + + // How to navigate to Spotlight search differs between iPhone and iPad + if BaseUITestCase.testDeviceIsIPad() == false { // iPhone + springboard.swipeDown() + spotlight.textFields["SpotlightSearchField"].typeText(searchQuery) + mullvadAppIcon = spotlightOrSpringboard.icons[appName] + } else { // iPad + // Swipe left enough times to reach the last page + for _ in 0 ..< 3 { + springboard.swipeLeft() + Thread.sleep(forTimeInterval: 0.5) + } + + springboard.swipeDown() + springboard.searchFields.firstMatch.typeText(searchQuery) + mullvadAppIcon = spotlightOrSpringboard.icons.matching(identifier: appName).allElementsBoundByIndex[1] + } - let appIcon = spotlight.icons[appName].firstMatch - if appIcon.waitForExistence(timeout: timeout) { - appIcon.press(forDuration: 2) + // The rest of the delete app flow is same for iPhone and iPad with the exception that iPhone uses spotlight and iPad uses springboard + if mullvadAppIcon.waitForExistence(timeout: timeout) { + mullvadAppIcon.press(forDuration: 2) } else { XCTFail("Failed to find app icon named \(appName)") } - let deleteAppButton = spotlight.buttons["Delete App"] + let deleteAppButton = spotlightOrSpringboard.buttons["Delete App"] if deleteAppButton.waitForExistence(timeout: timeout) { deleteAppButton.tap() } else { diff --git a/ios/MullvadVPNUITests/ConnectivityTests.swift b/ios/MullvadVPNUITests/ConnectivityTests.swift index 3b3ac28bdd03..a46042fddd9a 100644 --- a/ios/MullvadVPNUITests/ConnectivityTests.swift +++ b/ios/MullvadVPNUITests/ConnectivityTests.swift @@ -96,7 +96,10 @@ class ConnectivityTests: LoggedOutUITestCase { // Select the first country, its first city and its first relay SelectLocationPage(app) - .tapCountryLocationCellExpandButton(withIndex: 0) + .tapCountryLocationCellExpandButton( + withName: BaseUITestCase + .testsDefaultCountryName + ) // Must be a little specific here in order to avoid using relay services country with experimental relays .tapCityLocationCellExpandButton(withIndex: 0) .tapRelayLocationCell(withIndex: 0) diff --git a/ios/MullvadVPNUITests/Info.plist b/ios/MullvadVPNUITests/Info.plist index a81366999153..d477ab4de8e3 100644 --- a/ios/MullvadVPNUITests/Info.plist +++ b/ios/MullvadVPNUITests/Info.plist @@ -28,5 +28,9 @@ $(SHOULD_BE_REACHABLE_DOMAIN) TestDeviceIdentifier $(TEST_DEVICE_IDENTIFIER_UUID + TestDeviceIsIPad + $(TEST_DEVICE_IS_IPAD) + UninstallAppInTestSuiteTearDown + $(UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN) diff --git a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift index 95b9416c1c59..db401a0305c7 100644 --- a/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift +++ b/ios/MullvadVPNUITests/Pages/SelectLocationPage.swift @@ -22,6 +22,13 @@ class SelectLocationPage: Page { return self } + @discardableResult func tapCountryLocationCellExpandButton(withName name: String) -> Self { + let cell = app.cells.containing(.any, identifier: name) + let expandButton = cell.buttons[AccessibilityIdentifier.expandButton] + expandButton.tap() + return self + } + @discardableResult func tapCountryLocationCellExpandButton(withIndex: Int) -> Self { let cell = app.cells.containing(.any, identifier: AccessibilityIdentifier.countryLocationCell.rawValue) .element(boundBy: withIndex) diff --git a/ios/MullvadVPNUITests/README.md b/ios/MullvadVPNUITests/README.md index 703beee9f25e..1a473d1c4b18 100644 --- a/ios/MullvadVPNUITests/README.md +++ b/ios/MullvadVPNUITests/README.md @@ -1,13 +1,42 @@ -# Integration tests +# iOS end to end tests +## Running tests +### Locally using Xcode +Tests can be triggered locally from Xcode in the Test navigator or by running tests from the diamond in the editor gutter. -## iOS device setup +#### On GitHub +There are five workflows running tests: + - [ios-end-to-end-tests.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests.yml) - super workflow which other workflows reuse. This is also the workflow you can manually trigger to run all tests or optionally specify which tests to run. + - [ios-end-to-end-tests-nightly.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-nightly.yml) - scheduled nightly test run, running all tests. + - [ios-end-to-end-tests-merge-to-main.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-merge-to-main.yml) - automatically tryggered by a PR merge to `main`. + - [ios-end-to-end-tests-api.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-api.yml) - manually triggered tests focusing on making sure the API is functioning as intended on stagemole. + - [ios-end-to-end-tests-settings-migration.yml](https://github.com/mullvad/mullvadvpn-app/actions/workflows/ios-end-to-end-tests-settings-migration.yml) - for now this is still manually triggered. Tests installing older version of the app, changing settings, upgrading the app and verifying that settings were correctly migrated. + +## Adding more tests +When adding more files with test suites they must be added to the `MullvadVPNUITestsAll` test plan and also added to the appropriate node(s) in `ios/MullvadVPNUITests/tests.json` file in order to run in CI. For new test cases in already existing test suite nothing needs to be done. The test case/suite values in `tests.json` translate to input for `xcodebuild -only-testing` which is in the format `//`. The GitHub actions workflow will add the `` part so only `/` is required, where `` is optional. So for example `AccountTests` and `AccountTests/testLogin` are both valid values. + +## Set up local environment +To run tests locally you need to make sure you have copied the configuration template `UITests.xcconfig.template` to `UITests.xcconfig` and set up the configuration attributes. The configuration attributes you're mostly likely to want to set custom values for are at the top: +``` +// Pin code of the iOS device under test +IOS_DEVICE_PIN_CODE = + +// UUID to identify test runs. Should be unique per test device. Generate with for example uuidgen on macOS. +TEST_DEVICE_IDENTIFIER_UUID = +``` + +Look through other configuration attributes as well, but it is likely that their default value should be kept. Default values are set with local test execution in mind. They are changed in CI. + +## CI setup +### iOS device setup 1. Make sure device is added to provisioning profiles -2. Disable passcode in iOS settings - otherwise tests cannot be started without manually entering passcode -3. Make sure device is configured in GitHub secrets(see *GitHub setup* below) -4. Make sure the test device is connected to the WiFi `app-team-ios-tests` -5. Make sure iCloud syncing of keychain is off on the device so that the device isn't getting WiFi passwords from another device causing it to sometimes connect to another WiFi. +2. Enable developer mode +3. Disable passcode in iOS settings - otherwise tests cannot be started without manually entering passcode +4. Set the value of `TEST_DEVICE_UDID` to the UDID of the test device in `ios-end-to-end-tests.yml` and `ios-end-to-end-tests-settings-migration.yml`. +5. Make sure the test device is connected to the WiFi `app-team-ios-tests` +6. Make sure iCloud syncing of keychain is off on the device so that the device isn't getting WiFi passwords from another device causing it to sometimes connect to another WiFi. +7. After the device is set up download updated provisioning profiles on the GitHub runner computer(Download manual profiles in Xcode settings) -## Set up of runner environment +### Set up of runner build environment 1. Install Xcode 2. Sign in with Apple id in Xcode 3. Download manual provisioning profiles in Xcode @@ -15,8 +44,8 @@ 5. Install yeetd - `wget https://github.com/biscuitehh/yeetd/releases/download/1.0/yeetd-normal.pkg` - `sudo installer -pkg yeetd-normal.pkg -target yeetd` -6. Install ios-deploy - - `brew install ios-deploy` +6. Install ios-deploy and jq + - `brew install ios-deploy jq` 7. Install Homebrew and dependencies - `/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"` - `brew install xcbeautify wget swiftlint protobuf` @@ -28,23 +57,17 @@ 10. Install Go 1.19 - `brew install go@1.19` -## GitHub runner setup +### GitHub runner setup 1. Ask GitHub admin for new runner token and setup steps from GitHub. Set it up according to the steps, pass `--labels ios-test` to `config.sh` when running it. By default it will also have the labels `self-hosted` and `macOS` which are required as well. -2. Make sure GitHub actions secrets for the repository are correctly set up: +2. Make sure GitHub actions secrets for the GitHub project are correctly set up: - `IOS_DEVICE_PIN_CODE` - Device passcode if the device require it, otherwise leave blank. Devices used with CI should not require passcode. - `IOS_HAS_TIME_ACCOUNT_NUMBER` - Production server account without time left - `IOS_NO_TIME_ACCOUNT_NUMBER` - Production server account with time added to it - - `IOS_TEST_DEVICE_IDENTIFIER_UUID` - unique identifier for the test device. Create new identifier with `uuidgen`. - - `IOS_TEST_DEVICE_UDID` - the iOS device's UDID. + - `TEST_DEVICE_IDENTIFIER_UUID` - unique identifier for the test device. Create new identifier with `uuidgen`. - `PARTNER_API_TOKEN` - secret token for partner API. Optional and only intended to be used in CI when running tests against staging environment. -## Test plans -There are a few different test plans which are mainly to be triggered by GitHub action workflows but can also be triggered manually with Xcode: -* `MullvadVPNUITestsAll` - All tests except settings migration tests which are in separate test plan and workflow -* `MullvadVPNUITestsSmoke` - A few tests for smoke testing when merge:ing to `main` - -And also the following test plans which are used for testing settings migration(`ios-end-to-end-tests-settings-migration`): - +### Specifying which tests run when in CI +Which tests run when is specified in `tests.json`(See _Adding more tests_). Settings migration is an exception, it uses four different test plans and a separate workflow `ios-end-to-end-tests-settings-migration.yml` which executes the test plans in order, do not reinstall the app in between runs but upgrades the app after changing settings: * `MullvadVPNUITestsChangeDNSSettings` - Change settings for using custom DNS * `MullvadVPNUITestsVerifyDNSSettingsChanged` - Verify custom DNS settings still changed * `MullvadVPNUITestsChangeSettings` - Change all settings except custom DNS setting diff --git a/ios/MullvadVPNUITests/tests.json b/ios/MullvadVPNUITests/tests.json new file mode 100644 index 000000000000..2e8658400619 --- /dev/null +++ b/ios/MullvadVPNUITests/tests.json @@ -0,0 +1,18 @@ +{ + "tests": { + "nightly": [ + "AccountTests", + "ConnectivityTests", + "CustomListsTests", + "RelayTests", + "SettingsTests" + ], + "pr-merge-to-main": [ + "AccountTests/testLogin", + "AccountTests/testCreateAccount" + ], + "api-tests": [ + "AccountTests" + ] + } +} \ No newline at end of file