AWSNativeSDK: Linux Runtime Fix #113
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This automation builds 3p packages based on a PR | |
| name: Build 3P Packages | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - development | |
| paths: | |
| - 'package_build_list_host_*.json' | |
| jobs: | |
| detect-changes: | |
| name: Detecting changes in PR to build | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.detect-platform.outputs.matrix }} | |
| steps: | |
| - name: Checkout 3P source repo | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get package and platform from JSON changes | |
| id: detect-platform | |
| run: | | |
| CHANGED_FILES=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --name-only) | |
| # Construct the package and os into a json string to be consumed by Github Actions runners | |
| declare -A PACKAGES_JSON | |
| declare -A DOCKERFILE | |
| for FILE in $CHANGED_FILES; do | |
| if [[ $FILE == package_build_list_host_* ]]; then | |
| PLATFORM=$(echo $FILE | sed -n 's/package_build_list_host_\(.*\).json/\1/p') | |
| case $PLATFORM in | |
| linux*) | |
| OS_RUNNER="ubuntu-20.04" | |
| ;; | |
| windows) | |
| OS_RUNNER="windows-latest" # This is bundled with VS2022 | |
| ;; | |
| darwin) | |
| OS_RUNNER="macos-latest" | |
| ;; | |
| *) | |
| OS_RUNNER="windows-latest" # default | |
| ;; | |
| esac | |
| # Only get the changes that can be built | |
| IFS=$'\n' # Process each line in the package build file | |
| for LINE in DIFF=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ | |
| --no-ext-diff --unified=0 \ | |
| --exit-code -a --no-prefix -- $FILE | egrep "^\+[^\+]" | egrep -v "^\+\+\+ "); do | |
| unset IFS # Reset IFS to avoid word splitting | |
| PACKAGE=$(echo $LINE | cut -d'"' -f2) | |
| PACKPATH=$(echo $LINE | egrep -o "package-system/[^/ ]+(?=/| )" | head -n 1) | |
| if [[ -z "${DOCKERFILE["$PACKAGE"]}" && -n "$PACKPATH" ]]; then | |
| DOCKER=$(test -e ${PACKPATH%% }/Dockerfile* && echo 1 || echo 0) # Assume the build scripts will use the Dockerfile if found in the package path | |
| DOCKERFILE["$PACKAGE"]=1 # Mark Dockerfile check as done | |
| fi | |
| # Special cases for certain packages | |
| if [[ $PACKAGE =~ "DirectXShaderCompilerDxc" ]] && [[ $PLATFORM =~ "windows" ]]; then | |
| OS_RUNNER="windows-2019" | |
| fi | |
| PACKAGES_JSON["$PACKAGE"]="{\"package\": \"$PACKAGE\", \"os\": \"$OS_RUNNER\", \"dockerfile\": \"$DOCKER\"}" | |
| done | |
| unset IFS | |
| fi | |
| done | |
| # Construct the complete JSON from the array | |
| JSON="{\"include\":[" | |
| for PKG_JSON in "${PACKAGES_JSON[@]}"; do | |
| JSON="$JSON$PKG_JSON," | |
| done | |
| # Remove last "," and add closing brackets | |
| if [[ $JSON == *, ]]; then | |
| JSON="${JSON%?}" | |
| fi | |
| JSON="$JSON]}" | |
| echo $JSON | |
| # Set output | |
| echo "matrix=$( echo "$JSON" )" >> $GITHUB_OUTPUT | |
| validate-changes: | |
| name: Check changes for issues | |
| needs: detect-changes | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout 3P source repo | |
| uses: actions/checkout@v4 | |
| - name: Check if package already exists in prod | |
| env: | |
| PROD_CDN: ${{ vars.PROD_CDN }} # Change this to compare on your own endpoint | |
| run: | | |
| url="${{ env.PROD_CDN }}/${{ matrix.package }}" | |
| if curl --head --silent --fail ${url}.tar.xz > /dev/null 2>&1; then | |
| echo ${{ matrix.package }} already exists in prod. Check the rev in the json file to ensure it is incremented | |
| exit 1 | |
| else | |
| echo ${{ matrix.package }} does not exist in CDN, continuing... | |
| exit 0 | |
| fi | |
| - name: Malware scan of repo | |
| uses: dell/common-github-actions/malware-scanner@main | |
| with: | |
| directories: . | |
| options: -r | |
| build-on-specific-os: | |
| name: Build on "${{ matrix.os }}" for "${{ matrix.package }}" | |
| needs: [detect-changes, validate-changes] | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Configure | |
| id: configure | |
| run: | | |
| git config --global core.longpaths true | |
| echo "uid_gid=$(id -u):$(id -g)" >> $GITHUB_OUTPUT | |
| - name: Checkout 3P source repo | |
| uses: actions/checkout@v4 | |
| with: | |
| path: source | |
| fetch-depth: 0 | |
| - name: Checkout 3P scripts repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: o3de/3p-package-scripts | |
| path: scripts | |
| - name: Update python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - name: Install python dependancies | |
| run: | | |
| python3 -m pip install boto3 certifi | |
| - name: Update cmake/ninja | |
| uses: lukka/get-cmake@latest | |
| - name: Update msbuild path | |
| if: runner.os == 'Windows' | |
| uses: ilammy/[email protected] | |
| - name: Install clang/gcc | |
| if: runner.os == 'Linux' | |
| env: | |
| CLANG_VER: 12 | |
| GCC_VER: 9 | |
| run: | | |
| sudo apt-get install -y clang-${{ env.CLANG_VER }} gcc-${{ env.GCC_VER }} g++-${{ env.GCC_VER }} | |
| sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.CLANG_VER }} 10 | |
| sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${{ env.CLANG_VER }} 10 | |
| sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_VER }} 10 | |
| sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_VER }} 10 | |
| - name: Use sccache | |
| uses: hendrikmuhs/[email protected] | |
| with: | |
| variant: sccache | |
| max-size: 2048M | |
| key: ${{ matrix.package }}-${{ matrix.os }} | |
| restore-keys: | |
| ${{ matrix.package }}-${{ matrix.os }} | |
| - name: Set up QEMU (aarch64) # Only if the package folder contains a Dockerfile | |
| if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile == '1') }} | |
| run: | | |
| sudo apt-get install -y qemu qemu-user-static | |
| - name: Run build command | |
| if: ${{ (!contains(matrix.package, 'aarch64')) || (matrix.dockerfile == '1') }} | |
| env: | |
| CMAKE_CXX_COMPILER_LAUNCHER: sccache | |
| CMAKE_C_COMPILER_LAUNCHER: sccache | |
| CMAKE_GENERATOR: Ninja # ccache/sccache cannot be used as the compiler launcher under cmake if the generator is MSBuild | |
| run: | | |
| python3 scripts/o3de_package_scripts/build_package.py --search_path source ${{ matrix.package }} | |
| - name: Run build command (aarch64) # Generic build for packages without a Dockerfile | |
| if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile != '1') }} | |
| uses: uraimo/[email protected] | |
| with: | |
| arch: none | |
| distro: none | |
| base_image: ghcr.io/${{ github.repository }}/run-on-arch-${{ github.repository_owner }}-${{ github.event.repository.name }}-build-container-aarch64-ubuntu20-04:latest # built from build-container.yaml | |
| setup: | | |
| grep -q ${{ matrix.package }} ${PWD}/source/package_build_list_host_linux.json || rm ${PWD}/source/package_build_list_host_linux.json | |
| dockerRunArgs: | | |
| --platform=linux/arm64 | |
| --user ${{ steps.configure.outputs.uid_gid }} | |
| --volume "${PWD}:/workspace" | |
| --volume "${PWD}/scripts:/scripts" | |
| --volume "${PWD}/source:/source" | |
| env: | | |
| CMAKE_CXX_COMPILER_LAUNCHER: sccache | |
| CMAKE_C_COMPILER_LAUNCHER: sccache | |
| SCCACHE_IDLE_TIMEOUT: 0 | |
| SCCACHE_DIR: /workspace/.sccache | |
| SCCACHE_CACHE_SIZE: 2048M | |
| shell: /bin/bash | |
| run: | | |
| lsb_release -a | |
| uname -a | |
| sccache --start-server | |
| sccache -z | |
| ls -lah /workspace | |
| python3 /scripts/o3de_package_scripts/build_package.py --search_path /source/ ${{ matrix.package }} | |
| - name: Upload packages | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.package }} | |
| path: source/packages/* | |
| validate-packages: | |
| name: Validating ${{ matrix.package }} | |
| needs: [detect-changes, build-on-specific-os] | |
| runs-on: 'ubuntu-latest' | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}} | |
| steps: | |
| - name: Download packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.package }} | |
| - name: Verify SHA256 | |
| run: | | |
| echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS)" | |
| echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS | cut -d" " -f1) ${{ matrix.package }}.tar.xz" | sha256sum --check | |
| - name: Decompress package | |
| if: ${{ !contains(matrix.package, 'aarch64') }} | |
| run: | | |
| tar -xvf ${{ matrix.package }}.tar.xz | |
| - name: Malware scan | |
| uses: dell/common-github-actions/malware-scanner@main | |
| with: | |
| directories: . | |
| options: -r |