Merge pull request #1053 from lifting-bits/ian/bump-liftingbits-llvm-… #338
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Linux CI | |
| env: | |
| # "Source" is set in the vcpkg install step | |
| VCPKG_BINARY_SOURCES: 'clear;nuget,Source,readwrite;nugettimeout,3601' | |
| # Self-hosted runners can hopefully reuse already downloaded packages | |
| VCPKG_USE_NUGET_CACHE: 1 | |
| VCPKG_DEFAULT_HOST_TRIPLET: 'x64-linux-rel' | |
| on: | |
| release: | |
| types: | |
| - published | |
| pull_request: | |
| paths-ignore: | |
| - 'docker/**' | |
| - '.github/**' | |
| - '**.md' | |
| - '!.github/workflows/vcpkg_ci_amd64*' | |
| push: | |
| paths-ignore: | |
| - 'docker/**' | |
| - '.github/**' | |
| - '**.md' | |
| - '!.github/workflows/vcpkg_ci_amd64*' | |
| tags-ignore: | |
| - 'v*' | |
| branches: | |
| - 'master' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| # 'name' is Docker image name whereas 'os' is more generic | |
| - { os: 'ubuntu', name: 'ubuntu-v2', tag: '22.04' } | |
| llvm: [ | |
| 'llvm-16', | |
| 'llvm-17', | |
| 'llvm-17[liftingbits-llvm]' | |
| ] | |
| target_arch: [ 'x64', 'arm64' ] | |
| container: | |
| image: docker.pkg.github.com/lifting-bits/cxx-common/vcpkg-builder-${{ matrix.image.name }}:${{ matrix.image.tag }} | |
| volumes: | |
| - /:/gha-runner | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| name: ${{matrix.image.os}}-${{matrix.image.tag}} ${{matrix.llvm}} ${{matrix.target_arch}} | |
| runs-on: ubuntu-latest # run the job on the newly created runner | |
| steps: | |
| - name: Free Disk Space | |
| run: | | |
| df -h | |
| rm -rf /gha-runner/usr/local/lib/android | |
| rm -rf /gha-runner/usr/local/share/boost | |
| df -h | |
| - name: Set Artifact Name | |
| run: | | |
| # Need to fix because paths with brackets cause issues | |
| NAME="$(echo 'vcpkg_${{ matrix.image.os }}-${{ matrix.image.tag }}_${{ matrix.llvm }}_${{ matrix.target_arch == 'x64' && 'amd64' || matrix.target_arch }}' | tr '[' '-' | tr -d ']')" | |
| echo "ARTIFACT_NAME=${NAME}" >> "$GITHUB_ENV" | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 2 | |
| # Used to get commit message since PRs are on a merge commit | |
| - name: Get commit message | |
| shell: bash | |
| run: | | |
| git config --global --add safe.directory /__w/cxx-common/cxx-common | |
| echo 'COMMIT_MESSAGE<<EOF' >> "$GITHUB_ENV" | |
| if [[ '${{ github.event_name }}' == 'push' ]]; then | |
| echo "$(git log --format=%B -n 1 HEAD)" >> "$GITHUB_ENV" | |
| elif [[ '${{ github.event_name }}' == 'pull_request' ]]; then | |
| echo "$(git log --format=%B -n 1 HEAD^2)" >> "$GITHUB_ENV" | |
| fi | |
| echo "EOF" >> "$GITHUB_ENV" | |
| - name: Clear prior vcpkg directory | |
| run: | | |
| rm -rf "${{ github.workspace }}/vcpkg" | |
| - name: Initialize vcpkg | |
| shell: bash | |
| run: | | |
| { read -r vcpkg_repo_url && read -r vcpkg_commit; } <./vcpkg_info.txt || exit 1 | |
| git clone "${vcpkg_repo_url}" | |
| git -C vcpkg checkout "${vcpkg_commit}" | |
| export VCPKG_DISABLE_METRICS=1 | |
| ./vcpkg/bootstrap-vcpkg.sh | |
| echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV | |
| - name: 'vcpkg install dependencies' | |
| shell: 'bash' | |
| run: | | |
| export VCPKG_DISABLE_METRICS=1 | |
| # Setup NuGet authentication | |
| mono "$(${VCPKG_ROOT}/vcpkg fetch nuget | tail -n 1)" sources add \ | |
| -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \ | |
| -storepasswordincleartext \ | |
| -name "Source" \ | |
| -username "${{ github.repository_owner }}" \ | |
| -password "${{ secrets.GITHUB_TOKEN }}" || true | |
| mono "$(${VCPKG_ROOT}/vcpkg fetch nuget | tail -n 1)" sources update \ | |
| -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \ | |
| -storepasswordincleartext \ | |
| -name "Source" \ | |
| -username "${{ github.repository_owner }}" \ | |
| -password "${{ secrets.GITHUB_TOKEN }}" | |
| mono "$(${VCPKG_ROOT}/vcpkg fetch nuget | tail -n 1)" setapikey \ | |
| -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" \ | |
| "${{ secrets.GITHUB_TOKEN }}" | |
| ./build_dependencies.sh --release --target-arch ${{ matrix.target_arch }} --export-dir ./${{ env.ARTIFACT_NAME }} --clean-after-build --debug | |
| rm -rf vcpkg/downloads | |
| rm -rf ~/.nuget/packages | |
| echo "Check space..." | |
| df -h | |
| echo "DONE." | |
| ./build_dependencies.sh --release --target-arch ${{ matrix.target_arch }} --export-dir ./${{ env.ARTIFACT_NAME }} ${{ matrix.llvm }} --clean-after-build --debug | |
| echo "VCPKG_ROOT=$(pwd)/${{ env.ARTIFACT_NAME }}" >> $GITHUB_ENV | |
| echo "TARGET_TRIPLET=${{ matrix.target_arch }}-linux-rel" >> $GITHUB_ENV | |
| - name: Check space | |
| if: failure() | |
| run: | | |
| echo "## Space left" | |
| df -h | |
| echo "" | |
| echo "## Storage in working directory" | |
| du -hs * | |
| echo "" | |
| echo "## Storage in vcpkg" | |
| du -hs vcpkg/* | |
| echo "" | |
| echo "## Storage in nuget" | |
| du -hs ~/.nuget/* | |
| - name: Upload CMake logs on error | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACT_NAME }}_logs | |
| path: ${{ github.workspace }}/vcpkg/buildtrees/**/*.log | |
| - name: Cleanup | |
| shell: 'bash' | |
| run: | | |
| echo "Space left" | |
| df -h | |
| echo "Cleaning up..." | |
| rm -rf vcpkg || true | |
| rm -rf ~/.nuget || true | |
| echo "Space left" | |
| df -h | |
| - name: 'Export Packages' | |
| if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release | |
| shell: 'bash' | |
| run: | | |
| apt-get update | |
| apt-get install -y pixz | |
| tar --use-compress-program pixz -cf "${{ env.ARTIFACT_NAME }}.tar.xz" ./${{ env.ARTIFACT_NAME }} | |
| - name: Publish Release Assets | |
| if: github.event.release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: ${{ env.ARTIFACT_NAME }}.tar.xz | |
| prerelease: ${{ contains(github.ref, 'pre') || contains(github.ref, 'rc') }} | |
| token: ${{ secrets.RELEASE_TOKEN }} | |
| - uses: actions/upload-artifact@v4 | |
| if: contains(env.COMMIT_MESSAGE, 'debug artifacts') || github.event.release | |
| with: | |
| name: ${{ env.ARTIFACT_NAME }}.tar.xz | |
| path: ${{ env.ARTIFACT_NAME }}.tar.xz | |
| - name: Prepare ccache | |
| id: ccache_prep | |
| shell: bash | |
| run: | | |
| echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV | |
| echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV | |
| echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV | |
| echo "CMAKE_C_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV | |
| echo "CMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)" >> $GITHUB_ENV | |
| echo "timestamp=$(date +"%Y-%m-%d-%H:%M:%S" --utc)" >> ${GITHUB_OUTPUT} | |
| - name: ccache cache files | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ github.workspace }}/.ccache | |
| key: ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.target_arch }}-${{ matrix.llvm }}-${{ steps.ccache_prep.outputs.timestamp }} | |
| restore-keys: | | |
| ccache-${{ matrix.image.name }}-${{ matrix.image.tag }}-${{ matrix.target_arch }}-${{ matrix.llvm }}- | |
| - name: ccache Initial stats | |
| shell: bash | |
| run: | | |
| ccache --show-stats | |
| - name: 'Rellic build' | |
| shell: 'bash' | |
| working-directory: rellic | |
| if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') | |
| run: | | |
| # Does not compile with gcc | |
| export CC="$(which clang)" | |
| export CXX="$(which clang++)" | |
| cmake -G Ninja \ | |
| -DCMAKE_VERBOSE_MAKEFILE=ON \ | |
| "-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ | |
| -DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \ | |
| -DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \ | |
| "-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \ | |
| -S . -B build | |
| cmake --build build | |
| cmake --build build --target install | |
| - name: 'Rellic test' | |
| shell: 'bash' | |
| working-directory: rellic | |
| if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') | |
| run: | | |
| cmake --build build --target test | |
| - name: 'Remill dependencies' | |
| shell: 'bash' | |
| if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') | |
| working-directory: remill | |
| run: | | |
| python3 -m pip install poetry | |
| python3 -m pip install --user ./scripts/diff_tester_export_insns | |
| git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| - name: 'Remill build' | |
| shell: 'bash' | |
| if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') | |
| working-directory: remill | |
| run: | | |
| cmake -G Ninja \ | |
| -DCMAKE_VERBOSE_MAKEFILE=ON \ | |
| "-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ | |
| -DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \ | |
| -DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \ | |
| "-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \ | |
| -S . -B build | |
| cmake --build build | |
| cmake --install build | |
| cmake --build build --target test_dependencies | |
| - name: 'Remill test' | |
| shell: 'bash' | |
| working-directory: remill/build | |
| if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') | |
| run: | | |
| # Some tests fail on ubuntu 22.04 | |
| env CTEST_OUTPUT_ON_FAILURE=1 ctest . || true | |
| - name: 'Anvill build' | |
| shell: 'bash' | |
| working-directory: anvill | |
| if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') | |
| run: | | |
| # TODO: Does not support compilation with gcc | |
| export CC="$(which clang)" | |
| export CXX="$(which clang++)" | |
| cmake -G Ninja \ | |
| -DCMAKE_VERBOSE_MAKEFILE=ON \ | |
| "-DCMAKE_INSTALL_PREFIX=$(pwd)/install" \ | |
| "-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \ | |
| -DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET} \ | |
| -DVCPKG_HOST_TRIPLET=${VCPKG_DEFAULT_HOST_TRIPLET} \ | |
| -DANVILL_ENABLE_TESTS=true \ | |
| -DANVILL_ENABLE_INSTALL=true \ | |
| -DANVILL_ENABLE_PYTHON3_LIBS=OFF \ | |
| "-Dremill_DIR=$(pwd)/../remill/install/lib/cmake/remill" \ | |
| "-Dsleigh_DIR=$(pwd)/../remill/install/lib/cmake/sleigh" \ | |
| -S . -B build | |
| cmake --build build | |
| cmake --install build | |
| - name: 'Anvill test' | |
| shell: 'bash' | |
| working-directory: anvill | |
| if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') | |
| run: | | |
| cmake --build build --target test | |
| - name: Cache cleanup and reporting | |
| shell: 'bash' | |
| run: | | |
| rm -rf vcpkg/{buildtrees,installed,packages} | |
| ccache --show-stats | |
| df -h |