iOS Klaatu Tests #67
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# This is a basic workflow that is manually triggered | |
name: iOS Klaatu Tests | |
on: | |
workflow_dispatch: | |
inputs: | |
branch: | |
description: 'Experiment Branch' | |
default: 'control' | |
required: true | |
type: string | |
slug: | |
description: 'Experiment Slug' | |
default: 'an-awesome-experiment' | |
required: true | |
type: string | |
firefox-version: | |
description: 'The Firefox Version you want to test. v128 is the minimum.' | |
default: "['128']" | |
required: false | |
type: string | |
feature-name: | |
description: 'The Feature you want to test' | |
default: 'smoke' | |
required: true | |
type: choice | |
options: | |
- smoke | |
- messaging_survey | |
- messaging-new-tab-card | |
extra-arguments: | |
description: 'Additional testing arguments' | |
default: '' | |
required: false | |
type: string | |
jobs: | |
iOS-klaatu-tests: | |
runs-on: macos-14 | |
strategy: | |
fail-fast: false | |
matrix: | |
firefox: ${{ fromJSON(inputs.firefox-version) }} | |
name: ${{ inputs.feature-name }} tests for ${{ inputs.slug }} on firefox iOS v${{ matrix.firefox }} | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
- name: Clone firefox-ios repo | |
run: | | |
git clone --depth 1 https://github.com/mozilla-mobile/firefox-ios.git -b release/v${{ matrix.firefox }} | |
- name: Setup Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: '3.11' | |
- name: Cache | |
uses: actions/[email protected] | |
with: | |
path: /Users/runner/Library/Developer/Xcode/DerivedData/**/ | |
key: xcode-firefox-v${{ matrix.firefox }}-cache | |
- name: Delete Xcode App | |
run: sudo rm -rf /Applications/Xcode.app | |
- name: Set Xcode version | |
run: sudo xcode-select -switch /Applications/Xcode_15.3.app && xcodebuild -version | |
- name: Setup build | |
run: | | |
cd firefox-ios | |
sh ./bootstrap.sh | |
cd firefox-ios | |
brew update | |
pip3 install virtualenv poetry | |
- name: Check if cache loaded correctly | |
id: check_cache | |
run: | | |
if [ -d /Users/runner/Library/Developer/Xcode/DerivedData/Client-*/ ]; then | |
echo "::set-output name=cache_loaded::true" | |
else | |
echo "::set-output name=cache_loaded::false" | |
fi | |
- name: Build app | |
if: steps.check_cache.outputs.cache_loaded == 'false' | |
run: | | |
cd firefox-ios/firefox-ios | |
xcodebuild build-for-testing -project Client.xcodeproj -scheme Fennec -configuration Fennec -sdk iphonesimulator -quiet -destination 'platform=iOS Simulator,name=iPhone 15,OS=17.4' | |
- name: Install Nimbus CLI | |
run: curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/mozilla/application-services/main/install-nimbus-cli.sh | bash | |
- name: Run Tests | |
run: | | |
cp -rf tests/ios/* firefox-ios/firefox-ios/firefox-ios-tests/Tests/ExperimentIntegrationTests | |
cp -f tests/ios/ExperimentIntegrationTests.swift firefox-ios/firefox-ios/firefox-ios-tests/Tests/XCUITests/ExperimentIntegrationTests.swift | |
cd firefox-ios/firefox-ios/firefox-ios-tests/Tests/ExperimentIntegrationTests | |
poetry install | |
xcrun simctl erase all | |
xcrun simctl shutdown all | |
export SIMULATOR_UDID=$(python get_specific_device_udid.py) | |
poetry run python generate_smoke_tests.py | |
poetry run pytest --experiment ${{ inputs.slug }} --experiment-branch ${{ inputs.branch }} --feature ${{ inputs.feature-name }} ${{ inputs.extra-arguments }} | |
- name: Archive Results | |
if: ${{ always() }} | |
run: zip -r results.zip /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/Test/*.xcresult | |
- uses: actions/upload-artifact@v4 | |
if: ${{ always() }} | |
with: | |
name: xctest-results for Firefox v${{ matrix.firefox }} | |
path: results.zip | |
- uses: actions/upload-artifact@v4 | |
if: ${{ always() }} | |
with: | |
name: ${{ inputs.slug }} HTML Test Report Firefox v${{ matrix.firefox }} | |
path: /Users/runner/work/klaatu/klaatu/firefox-ios/firefox-ios-tests/Tests/ExperimentIntegrationTests/results/index.html | |
- uses: actions/upload-artifact@v4 | |
if: ${{ always() }} | |
with: | |
name: xcodebuild.log for Firefox v${{ matrix.firefox }} | |
path: /private/var/folders/3m/**/T/pytest-of-runner/pytest-*/**/xcodebuild.log |