build-package.yaml

# This automation builds 3p packages based on a PR

name: Build 3P Packages

on:
  pull_request:
    branches:
      - main
      - development
    paths:
      - 'package_build_list_host_*.json'

jobs:
  detect-changes:
    name: Detecting changes in PR to build
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.detect-platform.outputs.matrix }}
    
    steps:
    - name: Checkout 3P source repo
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
          
    - name: Get package and platform from JSON changes
      id: detect-platform
      run: |
        CHANGED_FILES=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --name-only)
        # Construct the package and os into a json string to be consumed by Github Actions runners

        declare -A PACKAGES_JSON
        declare -A DOCKERFILE
        
        for FILE in $CHANGED_FILES; do
          if [[ $FILE == package_build_list_host_* ]]; then
            PLATFORM=$(echo $FILE | sed -n 's/package_build_list_host_\(.*\).json/\1/p')
            case $PLATFORM in
            linux*)
              OS_RUNNER="ubuntu-20.04"
              ;;
            windows)
              OS_RUNNER="windows-latest" # This is bundled with VS2022
              ;;
            darwin)
              OS_RUNNER="macos-13" # Use x86 runner image. Latest uses ARM64
              ;;
            *)
              OS_RUNNER="windows-latest" # default
              ;;
            esac

            # Only get the changes that can be built
            IFS=$'\n' # Process each line in the package build file
            for LINE in DIFF=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
                      --no-ext-diff --unified=0 \
                      --exit-code -a --no-prefix -- $FILE | egrep "^\+[^\+]" | egrep -v "^\+\+\+ "); do
              unset IFS # Reset IFS to avoid word splitting
              PACKAGE=$(echo $LINE | cut -d'"' -f2)
              PACKPATH=$(echo $LINE | egrep -o "package-system/[^/ ]+(?=/| )" | head -n 1)
              if [[ -z "${DOCKERFILE["$PACKAGE"]}" && -n "$PACKPATH" ]]; then
                DOCKER=$(test -e ${PACKPATH%% }/Dockerfile* && echo 1 || echo 0) # Assume the build scripts will use the Dockerfile if found in the package path
                DOCKERFILE["$PACKAGE"]=1 # Mark Dockerfile check as done
              fi

              # Special cases for certain packages
              if [[ $PACKAGE =~ "DirectXShaderCompilerDxc" ]] && [[ $PLATFORM =~ "windows" ]]; then
                OS_RUNNER="windows-2019"
              fi
              
              PACKAGES_JSON["$PACKAGE"]="{\"package\": \"$PACKAGE\", \"os\": \"$OS_RUNNER\", \"dockerfile\": \"$DOCKER\"}"
            done
            unset IFS
          fi
        done

        # Construct the complete JSON from the array
        JSON="{\"include\":["
        for PKG_JSON in "${PACKAGES_JSON[@]}"; do
          JSON="$JSON$PKG_JSON,"
        done

        # Remove last "," and add closing brackets
        if [[ $JSON == *, ]]; then
          JSON="${JSON%?}"
        fi
        JSON="$JSON]}"
        echo $JSON

        # Set output
        echo "matrix=$( echo "$JSON" )" >> $GITHUB_OUTPUT

  validate-changes:
    name: Check changes for issues
    needs: detect-changes
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
    runs-on: ubuntu-latest

    steps:
    - name: Checkout 3P source repo
      uses: actions/checkout@v4
        
    - name: Check if package already exists in prod
      env:
        PROD_CDN: ${{ vars.PROD_CDN }} # Change this to compare on your own endpoint
      run: |
        url="${{ env.PROD_CDN }}/${{ matrix.package }}"
        if curl --head --silent --fail ${url}.tar.xz > /dev/null 2>&1; then
          echo ${{ matrix.package }} already exists in prod. Check the rev in the json file to ensure it is incremented
          exit 1
        else
          echo ${{ matrix.package }} does not exist in CDN, continuing...
          exit 0
        fi

    - name: Malware scan of repo
      uses: dell/common-github-actions/malware-scanner@main
      with:
        directories: .
        options: -r

  build-on-specific-os:
    name: Build on "${{ matrix.os }}" for "${{ matrix.package }}"
    needs: [detect-changes, validate-changes]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
    runs-on: ${{ matrix.os }}

    steps:
    - name: Configure
      id: configure
      run: |
        git config --global core.longpaths true
        echo "uid_gid=$(id -u):$(id -g)" >> $GITHUB_OUTPUT
      
    - name: Checkout 3P source repo
      uses: actions/checkout@v4
      with:
        path: source
        fetch-depth: 0
    
    - name: Checkout 3P scripts repo
      uses: actions/checkout@v4
      with:
        repository: o3de/3p-package-scripts
        path: scripts
      
    - name: Update python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
        cache: 'pip'
    
    - name: Install python dependancies
      run: |
        python3 -m pip install boto3 certifi
    
    - name: Update cmake/ninja
      uses: lukka/get-cmake@latest

    - name: Update msbuild path
      if: runner.os == 'Windows'
      uses: ilammy/msvc-dev-cmd@v1.13.0
    
    - name: Install clang/gcc
      if: runner.os == 'Linux'
      env:
        CLANG_VER: 12
        GCC_VER: 9
      run: |
        sudo apt-get install -y clang-${{ env.CLANG_VER }} gcc-${{ env.GCC_VER }} g++-${{ env.GCC_VER }}
        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.CLANG_VER }} 10
        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${{ env.CLANG_VER }} 10
        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_VER }} 10
        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_VER }} 10

    - name: Use sccache
      uses: hendrikmuhs/ccache-action@v1.2.10
      with:
        variant: sccache
        max-size: 2048M
        key: ${{ matrix.package }}-${{ matrix.os }}
        restore-keys:
          ${{ matrix.package }}-${{ matrix.os }}  

    - name: Set up QEMU (aarch64) # Only if the package folder contains a Dockerfile
      if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile == '1') }}
      run: |
        sudo apt-get install -y qemu qemu-user-static

    - name: Run build command
      if: ${{ (!contains(matrix.package, 'aarch64')) || (matrix.dockerfile == '1') }}
      env:
        CMAKE_CXX_COMPILER_LAUNCHER: sccache
        CMAKE_C_COMPILER_LAUNCHER: sccache
        CMAKE_GENERATOR: Ninja # ccache/sccache cannot be used as the compiler launcher under cmake if the generator is MSBuild
      run: |
        python3 scripts/o3de_package_scripts/build_package.py --search_path source ${{ matrix.package }}

    - name: Run build command (aarch64) # Generic build for packages without a Dockerfile
      if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile != '1') }}
      uses: uraimo/run-on-arch-action@v2.7.1
      with:
        arch: none
        distro: none
        base_image: ghcr.io/${{ github.repository }}/run-on-arch-${{ github.repository_owner }}-${{ github.event.repository.name }}-build-container-aarch64-ubuntu20-04:latest # built from build-container.yaml
        setup: |
          grep -q ${{ matrix.package }} ${PWD}/source/package_build_list_host_linux.json || rm ${PWD}/source/package_build_list_host_linux.json
        dockerRunArgs: |
          --platform=linux/arm64 
          --user ${{ steps.configure.outputs.uid_gid }}
          --volume "${PWD}:/workspace"
          --volume "${PWD}/scripts:/scripts"
          --volume "${PWD}/source:/source"
        env: |
          CMAKE_CXX_COMPILER_LAUNCHER: sccache
          CMAKE_C_COMPILER_LAUNCHER: sccache
          SCCACHE_IDLE_TIMEOUT: 0 
          SCCACHE_DIR: /workspace/.sccache
          SCCACHE_CACHE_SIZE: 2048M
        shell: /bin/bash
        run: |
          lsb_release -a
          uname -a
          sccache --start-server
          sccache -z
          ls -lah /workspace
          python3 /scripts/o3de_package_scripts/build_package.py --search_path /source/ ${{ matrix.package }}
          
    - name: Upload packages
      uses: actions/upload-artifact@v4
      with:
        name: ${{ matrix.package }}
        path: source/packages/*

  validate-packages:
    name: Validating ${{ matrix.package }}
    needs: [detect-changes, build-on-specific-os]
    runs-on: 'ubuntu-latest'
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
    steps:
      - name: Download packages
        uses: actions/download-artifact@v4
        with:
          name: ${{ matrix.package }}
          
      - name: Verify SHA256
        run: |
          echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS)"
          echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS | cut -d" " -f1) ${{ matrix.package }}.tar.xz" | sha256sum --check

      - name: Decompress package
        if: ${{ !contains(matrix.package, 'aarch64') }}
        run: |
          tar -xvf ${{ matrix.package }}.tar.xz
    
      - name: Malware scan
        uses: dell/common-github-actions/malware-scanner@main
        with:
          directories: .
          options: -r