From 88961a009b3f2c55dbab9da1c23f27bcef1ff7d4 Mon Sep 17 00:00:00 2001 From: Sebastian Villena <97059974+ruisebas@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:47:36 -0400 Subject: [PATCH] chore: Caching dependencies when building and testing (#3168) --- .../run_xcodebuild/action.yml | 11 +- .../run_xcodebuild_test/action.yml | 69 +++++++- .github/workflows/build_amplify_swift.yml | 51 ++++++ .github/workflows/deploy_package.yml | 4 + .../run_xcodebuild_test_platforms.yml | 164 ++++++++++++++++++ .github/workflows/upload_coverage_report.yml | 3 +- 6 files changed, 296 insertions(+), 6 deletions(-) diff --git a/.github/composite_actions/run_xcodebuild/action.yml b/.github/composite_actions/run_xcodebuild/action.yml index feae24b433..1a39188604 100644 --- a/.github/composite_actions/run_xcodebuild/action.yml +++ b/.github/composite_actions/run_xcodebuild/action.yml @@ -19,6 +19,10 @@ inputs: required: false type: string default: 'iphonesimulator' + disable_package_resolution: + required: false + type: boolean + default: false other_flags: required: false type: string @@ -39,6 +43,11 @@ runs: if [ ! -z "$XCODE_PATH" ]; then sudo xcode-select -s $XCODE_PATH fi + + otherFlags="${{ inputs.other_flags }}" + if [ "${{ inputs.disable_package_resolution }}" == "true" ]; then + otherFlags+=" -disableAutomaticPackageResolution" + fi xcodebuild -version - xcodebuild build -scheme $SCHEME -sdk '${{ inputs.sdk }}' -destination '${{ inputs.destination }}' ${{ inputs.other_flags }} | xcpretty --simple --color --report junit && exit ${PIPESTATUS[0]} + xcodebuild build -scheme $SCHEME -sdk '${{ inputs.sdk }}' -destination '${{ inputs.destination }}' $otherFlags | xcpretty --simple --color --report junit && exit ${PIPESTATUS[0]} shell: bash \ No newline at end of file diff --git a/.github/composite_actions/run_xcodebuild_test/action.yml b/.github/composite_actions/run_xcodebuild_test/action.yml index 2fdbcf72f2..3f377b7ef5 100644 --- a/.github/composite_actions/run_xcodebuild_test/action.yml +++ b/.github/composite_actions/run_xcodebuild_test/action.yml @@ -27,6 +27,22 @@ inputs: required: false type: boolean default: false + cloned_source_packages_path: + required: false + type: string + default: '' + derived_data_path: + required: false + type: string + default: '' + disable_package_resolution: + required: false + type: boolean + default: false + test_without_building: + required: false + type: boolean + default: false runs: using: "composite" @@ -36,6 +52,8 @@ runs: SCHEME: ${{ inputs.scheme }} PROJECT_PATH: ${{ inputs.project_path }} XCODE_PATH: ${{ inputs.xcode_path }} + CLONED_SOURCE_PACKAGES_PATH: ${{ inputs.cloned_source_packages_path }} + DERIVED_DATA_PATH: ${{ inputs.derived_data_path }} run: | if [ ! -z "$PROJECT_PATH" ]; then cd $PROJECT_PATH @@ -44,23 +62,66 @@ runs: echo "Using Xcode $XCODE_PATH" sudo xcode-select -s $XCODE_PATH fi + + clonedSourcePackagesPath="" + if [ ! -z "$CLONED_SOURCE_PACKAGES_PATH" ]; then + echo "Using custom cloned source packages path" + clonedSourcePackagesPath+="-clonedSourcePackagesDirPath $CLONED_SOURCE_PACKAGES_PATH" + fi + + derivedDataPath="" + if [ ! -z "$DERIVED_DATA_PATH" ]; then + echo "Using custom DerivedData path" + derivedDataPath+="-derivedDataPath $DERIVED_DATA_PATH" + fi + coverageFlags="" if [ "${{ inputs.generate_coverage }}" == "true" ]; then echo "Code Coverage is enabled!" - coverageFlags+="-derivedDataPath Build/ -clonedSourcePackagesDirPath "~/Library/Developer/Xcode/DerivedData/$SCHEME" -enableCodeCoverage YES" + coverageFlags+="-enableCodeCoverage YES" + if [ -z "$clonedSourcePackagesPath" ]; then + clonedSourcePackagesPath+="-clonedSourcePackagesDirPath ~/Library/Developer/Xcode/DerivedData/Amplify" + fi + + if [ -z "$derivedDataPath" ]; then + derivedDataPath+="-derivedDataPath Build/" + fi + fi + + if [ "${{ inputs.disable_package_resolution }}" == "true" ]; then + echo "Disabling Automatic Package Resolution" + clonedSourcePackagesPath+=" -disableAutomaticPackageResolution" + fi + + action="test" + if [ "${{ inputs.test_without_building }}" == "true" ]; then + echo "Testing without building..." + action+="-without-building" fi + xcode-select -p xcodebuild -version - xcodebuild test -scheme $SCHEME -sdk '${{ inputs.sdk }}' -destination '${{ inputs.destination }}' ${{ inputs.other_flags }} $coverageFlags | xcpretty --simple --color --report junit && exit ${PIPESTATUS[0]} + xcodebuild $action -scheme $SCHEME -sdk '${{ inputs.sdk }}' -destination '${{ inputs.destination }}' ${{ inputs.other_flags }} $clonedSourcePackagesPath $derivedDataPath $coverageFlags | xcpretty --simple --color --report junit && exit ${PIPESTATUS[0]} shell: bash - name: Generate Coverage report if: ${{ inputs.generate_coverage == 'true' }} + env: + SCHEME: ${{ inputs.scheme }} + DERIVED_DATA_PATH: ${{ inputs.derived_data_path }} run: | echo "Generating Coverage report..." - cd Build/Build/ProfileData + + derivedDataPath="" + if [ ! -z "$DERIVED_DATA_PATH" ]; then + derivedDataPath="$DERIVED_DATA_PATH" + else + derivedDataPath="Build" + fi + + cd $derivedDataPath/Build/ProfileData cd $(ls -d */|head -n 1) pathCoverage=Build/Build/ProfileData/${PWD##*/}/Coverage.profdata cd ${{ github.workspace }} - xcrun llvm-cov export -format="lcov" -instr-profile $pathCoverage Build/Build/Products/Debug-${{ inputs.sdk }}/$SCHEME.o > $SCHEME-Coverage.lcov + xcrun llvm-cov export -format="lcov" -instr-profile $pathCoverage $derivedDataPath/Build/Products/Debug-${{ inputs.sdk }}/$SCHEME.o > $SCHEME-Coverage.lcov shell: bash diff --git a/.github/workflows/build_amplify_swift.yml b/.github/workflows/build_amplify_swift.yml index d6bda1fadd..d3794a26de 100644 --- a/.github/workflows/build_amplify_swift.yml +++ b/.github/workflows/build_amplify_swift.yml @@ -22,12 +22,30 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Attempt to restore dependencies cache + id: cache-packages + timeout-minutes: 4 + continue-on-error: true + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- - name: Build Amplify Swift for iOS uses: ./.github/composite_actions/run_xcodebuild with: scheme: Amplify-Package destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4' xcode_path: '/Applications/Xcode_14.3.app' + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + other_flags: '-derivedDataPath Build -clonedSourcePackagesDirPath ~/Library/Developer/Xcode/DerivedData/Amplify' + - name: Save the dependencies cache if necessary + if: steps.cache-packages.outputs.cache-hit != 'true' && github.ref_name == 'main' + uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: ${{ steps.cache-packages.outputs.cache-primary-key }} build-amplify-swift-macOS: runs-on: macos-13 @@ -36,6 +54,15 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Cache packages + id: cache-packages + timeout-minutes: 4 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- - name: Build Amplify Swift for macOS uses: ./.github/composite_actions/run_xcodebuild with: @@ -43,6 +70,8 @@ jobs: destination: platform=macOS,arch=x86_64 sdk: macosx xcode_path: '/Applications/Xcode_14.3.app' + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + other_flags: '-derivedDataPath Build -clonedSourcePackagesDirPath ~/Library/Developer/Xcode/DerivedData/Amplify' build-amplify-swift-tvOS: runs-on: macos-13 @@ -51,6 +80,15 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Cache packages + timeout-minutes: 4 + id: cache-packages + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- - name: Build Amplify Swift for tvOS uses: ./.github/composite_actions/run_xcodebuild with: @@ -58,6 +96,8 @@ jobs: destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=16.4 sdk: appletvsimulator xcode_path: '/Applications/Xcode_14.3.app' + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + other_flags: '-derivedDataPath Build -clonedSourcePackagesDirPath ~/Library/Developer/Xcode/DerivedData/Amplify' build-amplify-swift-watchOS: runs-on: macos-13 @@ -66,6 +106,15 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Cache packages + id: cache-packages + timeout-minutes: 4 + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- - name: Build Amplify Swift for watchOS uses: ./.github/composite_actions/run_xcodebuild with: @@ -73,6 +122,8 @@ jobs: destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=9.4 sdk: watchsimulator xcode_path: '/Applications/Xcode_14.3.app' + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + other_flags: '-derivedDataPath Build -clonedSourcePackagesDirPath ~/Library/Developer/Xcode/DerivedData/Amplify' confirm-pass: runs-on: ubuntu-latest diff --git a/.github/workflows/deploy_package.yml b/.github/workflows/deploy_package.yml index b479f45601..2e65d49b81 100644 --- a/.github/workflows/deploy_package.yml +++ b/.github/workflows/deploy_package.yml @@ -13,6 +13,10 @@ permissions: contents: write jobs: + build-amplify-swift: + name: Build Amplify package + uses: ./.github/workflows/build_amplify_swift.yml + unit-tests: name: Run Plugins Unit Tests uses: ./.github/workflows/unit_test.yml diff --git a/.github/workflows/run_xcodebuild_test_platforms.yml b/.github/workflows/run_xcodebuild_test_platforms.yml index 3f329556d2..f1dc97a73c 100644 --- a/.github/workflows/run_xcodebuild_test_platforms.yml +++ b/.github/workflows/run_xcodebuild_test_platforms.yml @@ -32,13 +32,54 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Attempt to restore dependencies cache + id: cache-packages + timeout-minutes: 4 + continue-on-error: true + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- + - name: Attempt to restore the build cache + id: restore-build + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-iOS-build-${{ github.sha }} - name: Run iOS Test Suite + id: run-tests + continue-on-error: true uses: ./.github/composite_actions/run_xcodebuild_test with: scheme: ${{ env.SCHEME }} destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4' xcode_path: '/Applications/Xcode_14.3.app' generate_coverage: ${{ inputs.generate_coverage_report }} + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + test_without_building: ${{ steps.restore-build.outputs.cache-hit }} + - name: Retry iOS Test Suite if needed + if: steps.run-tests.outcome=='failure' + id: retry-tests + uses: ./.github/composite_actions/run_xcodebuild_test + with: + scheme: ${{ env.SCHEME }} + destination: 'platform=iOS Simulator,name=iPhone 14,OS=16.4' + xcode_path: '/Applications/Xcode_14.3.app' + generate_coverage: ${{ inputs.generate_coverage_report }} + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: true + test_without_building: true + - name: Save the build cache for re-runs + if: failure() && steps.retry-tests.outcome=='failure' + uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-iOS-build-${{ github.sha }} - name: Upload Report File if: ${{ inputs.generate_coverage_report == true }} uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce #v3.1.2 @@ -57,13 +98,54 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Attempt to restore dependencies cache + id: cache-packages + timeout-minutes: 4 + continue-on-error: true + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- + - name: Attempt to restore the build cache + id: restore-build + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-macOS-build-${{ github.sha }} - name: Run macOS Test Suite + id: run-tests + continue-on-error: true uses: ./.github/composite_actions/run_xcodebuild_test with: scheme: ${{ env.SCHEME }} destination: platform=macOS,arch=x86_64 sdk: macosx xcode_path: '/Applications/Xcode_14.3.app' + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + test_without_building: ${{ steps.restore-build.outputs.cache-hit }} + - name: Retry macOS Test Suite if needed + if: steps.run-tests.outcome=='failure' + id: retry-tests + uses: ./.github/composite_actions/run_xcodebuild_test + with: + scheme: ${{ env.SCHEME }} + destination: platform=macOS,arch=x86_64 + sdk: macosx + xcode_path: '/Applications/Xcode_14.3.app' + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: true + test_without_building: true + - name: Save the build cache for re-runs + if: failure() && steps.retry-tests.outcome=='failure' + uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-macOS-build-${{ github.sha }} test-tvOS: name: ${{ inputs.scheme }} tvOS Tests @@ -73,13 +155,54 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Attempt to restore dependencies cache + id: cache-packages + timeout-minutes: 4 + continue-on-error: true + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- + - name: Attempt to restore the build cache + id: restore-build + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-tvOS-build-${{ github.sha }} - name: Run tvOS Test Suite + id: run-tests + continue-on-error: true uses: ./.github/composite_actions/run_xcodebuild_test with: scheme: ${{ env.SCHEME }} destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=16.4 sdk: appletvsimulator xcode_path: '/Applications/Xcode_14.3.app' + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + test_without_building: ${{ steps.restore-build.outputs.cache-hit }} + - name: Retry tvOS Test Suite if needed + if: steps.run-tests.outcome=='failure' + id: retry-tests + uses: ./.github/composite_actions/run_xcodebuild_test + with: + scheme: ${{ env.SCHEME }} + destination: platform=tvOS Simulator,name=Apple TV 4K (3rd generation),OS=16.4 + sdk: appletvsimulator + xcode_path: '/Applications/Xcode_14.3.app' + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: true + test_without_building: true + - name: Save the build cache for re-runs + if: failure() && steps.retry-tests.outcome=='failure' + uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-tvOS-build-${{ github.sha }} test-watchOS: name: ${{ inputs.scheme }} watchOS Tests @@ -89,10 +212,51 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: persist-credentials: false + - name: Attempt to restore dependencies cache + id: cache-packages + timeout-minutes: 4 + continue-on-error: true + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ~/Library/Developer/Xcode/DerivedData/Amplify + key: amplify-packages-${{ hashFiles('Package.resolved') }} + restore-keys: | + amplify-packages- + - name: Attempt to restore the build cache + id: restore-build + uses: actions/cache/restore@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-watchOS-build-${{ github.sha }} - name: Run watchOS Test Suite + id: run-tests + continue-on-error: true uses: ./.github/composite_actions/run_xcodebuild_test with: scheme: ${{ env.SCHEME }} destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=9.4 sdk: watchsimulator xcode_path: '/Applications/Xcode_14.3.app' + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: ${{ steps.cache-packages.outputs.cache-hit }} + test_without_building: ${{ steps.restore-build.outputs.cache-hit }} + - name: Retry watchOS Test Suite if needed + if: steps.run-tests.outcome=='failure' + id: retry-tests + uses: ./.github/composite_actions/run_xcodebuild_test + with: + scheme: ${{ env.SCHEME }} + destination: platform=watchOS Simulator,name=Apple Watch Series 8 (45mm),OS=9.4 + sdk: watchsimulator + xcode_path: '/Applications/Xcode_14.3.app' + cloned_source_packages_path: ~/Library/Developer/Xcode/DerivedData/Amplify + derived_data_path: ${{ github.workspace }}/Build + disable_package_resolution: true + test_without_building: true + - name: Save the build cache for re-runs + if: failure() && steps.retry-tests.outcome=='failure' + uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + path: ${{ github.workspace }}/Build + key: ${{ env.SCHEME }}-watchOS-build-${{ github.sha }} diff --git a/.github/workflows/upload_coverage_report.yml b/.github/workflows/upload_coverage_report.yml index 8bbdc77e91..29a76f6720 100644 --- a/.github/workflows/upload_coverage_report.yml +++ b/.github/workflows/upload_coverage_report.yml @@ -18,7 +18,8 @@ permissions: jobs: upload-coverage: name: ${{ inputs.scheme }} Coverage Report - runs-on: macos-13 + timeout-minutes: 2 + runs-on: ubuntu-latest steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 #v3.5.3 with: