From 3dc23b6ef3858743bc7daaf145e111baef2261b0 Mon Sep 17 00:00:00 2001 From: Elliott Slaughter Date: Thu, 18 Aug 2022 09:21:49 -0700 Subject: [PATCH] Move all Linux CI jobs to Docker (#600) This avoids churn due to the deprecation schedule of GitHub Actions runner images, and also fixes a number of issues that seem to result from conflicts with the newer runner versions. The Docker build is now fully featured and can build all configurations we use for the tests (except for Make, which we relegate to the old build scripts). The scripts are longer, but it's the same set of commands being executed in the basic cases, so these are still minimal builds. --- .cirrus.yml | 6 +- .github/workflows/main.yml | 295 +++++++++++++------- docker/Dockerfile.ubuntu | 13 +- docker/Dockerfile.ubuntu-prebuilt | 24 -- docker/Dockerfile.ubuntu-prebuilt-multiarch | 24 -- docker/Dockerfile.ubuntu-source | 31 -- docker/Dockerfile.ubuntu-upstream | 25 -- docker/build.sh | 13 +- docker/compatibility_test.sh | 11 +- docker/install_script.sh | 122 ++++++++ docker/install_script_multiarch.sh | 18 -- travis.sh | 83 ++---- 12 files changed, 355 insertions(+), 310 deletions(-) delete mode 100644 docker/Dockerfile.ubuntu-prebuilt delete mode 100644 docker/Dockerfile.ubuntu-prebuilt-multiarch delete mode 100644 docker/Dockerfile.ubuntu-source delete mode 100644 docker/Dockerfile.ubuntu-upstream create mode 100755 docker/install_script.sh delete mode 100755 docker/install_script_multiarch.sh diff --git a/.cirrus.yml b/.cirrus.yml index d2e13995f..9ca3e9368 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -27,7 +27,9 @@ arm_task: image: ubuntu:18.04 env: llvm: 11.1.0 - arch: aarch64 + cuda: 0 + variant: prebuilt + test: 1 threads: 2 script: | - ./docker/install_script_multiarch.sh + ./docker/install_script.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3b0d10c04..eef801f88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,34 +4,45 @@ on: [push, pull_request] jobs: check_format: name: Check Format - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v1 - run: ./travis.sh env: CHECK_CLANG_FORMAT: 1 build: - name: Build (${{ matrix.os }}, llvm-${{ matrix.llvm }}, cmake=${{ matrix.cmake }}, cuda=${{ matrix.cuda }}, static=${{ matrix.static }}, slib=${{ matrix.slib }}, ${{ matrix.lua }}) + name: Build (${{ matrix.os }}, llvm-${{ matrix.llvm }}, cmake=${{ matrix.cmake }}, cuda=${{ matrix.cuda }}, ${{ matrix.lua }}) runs-on: ${{ matrix.os }} strategy: matrix: - os: ['ubuntu-18.04', 'macos-11', 'windows-2022'] - llvm: ['5.0', '6.0', '7', '8', '9', '10', '11', '12', '13', '14'] + os: ['ubuntu-22.04', 'macos-11', 'windows-2022'] + llvm: ['6.0', '7', '8', '9', '10', '11', '12', '13', '14'] cmake: ['0', '1'] cuda: ['0', '1'] - static: ['0', '1'] - slib: ['0', '1'] lua: ['luajit', 'moonjit'] exclude: - # Linux: exclude LLVM 13-14 - - os: 'ubuntu-18.04' + # Linux: exclude LLVM 6.0-13, cmake + # Note: CMake configurations are tested in Docker + - os: 'ubuntu-22.04' + llvm: '6.0' + - os: 'ubuntu-22.04' + llvm: '7' + - os: 'ubuntu-22.04' + llvm: '8' + - os: 'ubuntu-22.04' + llvm: '9' + - os: 'ubuntu-22.04' + llvm: '10' + - os: 'ubuntu-22.04' + llvm: '11' + - os: 'ubuntu-22.04' + llvm: '12' + - os: 'ubuntu-22.04' llvm: '13' - - os: 'ubuntu-18.04' - llvm: '14' + - os: 'ubuntu-22.04' + cmake: '1' - # macOS: exclude LLVM 5.0, 8-14 make, cuda/no-static/no-slib - - os: 'macos-11' - llvm: '5.0' + # macOS: exclude LLVM 8-14 make, cuda - os: 'macos-11' llvm: '8' cmake: '0' @@ -55,17 +66,11 @@ jobs: cmake: '0' - os: 'macos-11' cuda: '1' - - os: 'macos-11' - static: '0' - - os: 'macos-11' - slib: '0' # LLVM 10 binaries are broken on recent macOS - os: 'macos-11' llvm: '10' - # Windows: exclude LLVM 5.0-10,12-14, make - - os: 'windows-2022' - llvm: '5.0' + # Windows: exclude LLVM 6.0-10,12-14, make - os: 'windows-2022' llvm: '6.0' - os: 'windows-2022' @@ -86,30 +91,14 @@ jobs: cmake: '0' # CMake - - llvm: '5.0' - cmake: '0' - llvm: '7' cmake: '0' - # no-static/no-slib require CMake, slib requires static - - cmake: '0' - static: '0' - - cmake: '0' - slib: '0' - - static: '0' - slib: '1' - - # FIXME: no-static breaks with LuaJIT - - static: '0' - lua: 'luajit' - # Moonjit requires CMake - cmake: '0' lua: 'moonjit' # CUDA: only LLVM 11 - - llvm: '5.0' - cuda: '1' - llvm: '6.0' cuda: '1' - llvm: '7' @@ -127,49 +116,7 @@ jobs: - llvm: '14' cuda: '1' - # no-static: only LLVM 9 - - llvm: '5.0' - static: '0' - - llvm: '6.0' - static: '0' - - llvm: '7' - static: '0' - - llvm: '8' - static: '0' - - llvm: '10' - static: '0' - - llvm: '11' - static: '0' - - llvm: '12' - static: '0' - - llvm: '13' - static: '0' - - llvm: '14' - static: '0' - - # no-slib: only LLVM 9 - - llvm: '5.0' - slib: '0' - - llvm: '6.0' - slib: '0' - - llvm: '7' - slib: '0' - - llvm: '8' - slib: '0' - - llvm: '10' - slib: '0' - - llvm: '11' - slib: '0' - - llvm: '12' - slib: '0' - - llvm: '13' - slib: '0' - - llvm: '14' - slib: '0' - # Moonjit: only LLVM 9 - - llvm: '5.0' - lua: 'moonjit' - llvm: '6.0' lua: 'moonjit' - llvm: '7' @@ -195,10 +142,10 @@ jobs: CLANG: clang-${{ matrix.llvm }} USE_CMAKE: ${{ matrix.cmake }} USE_CUDA: ${{ matrix.cuda }} - STATIC_LLVM: ${{ matrix.static }} - STATIC_LUAJIT: ${{ matrix.static }} - SLIB_INCLUDE_LLVM: ${{ matrix.slib }} - SLIB_INCLUDE_LUAJIT: ${{ matrix.slib }} + STATIC_LLVM: 1 + STATIC_LUAJIT: 1 + SLIB_INCLUDE_LLVM: 1 + SLIB_INCLUDE_LUAJIT: 1 TERRA_LUA: ${{ matrix.lua }} - uses: actions/upload-artifact@v2 with: @@ -207,47 +154,180 @@ jobs: terra-*.tar.xz terra-*.7z docker: - name: Docker + name: Docker (${{ matrix.distro }}, llvm-${{ matrix.llvm }}, ${{ matrix.lua }}, static=${{ matrix.static }}, slib=${{ matrix.slib }}, cuda=${{ matrix.cuda }}, ${{ matrix.variant }}) runs-on: ubuntu-latest strategy: matrix: - distro: ['ubuntu-16.04', 'ubuntu-18.04', 'ubuntu-20.04', 'ubuntu-22.04'] - llvm: ['3.8', '6.0', '12', '14.0.6'] + distro: ['ubuntu-18.04'] + llvm: ['5.0', '6.0', '7', '8', '9', '10', '11', '12.0.1', '13.0.1', '14.0.6'] + lua: ['luajit', 'moonjit'] + static: ['0', '1'] + slib: ['0', '1'] + cuda: ['0', '1'] + test: ['1'] exclude: - - distro: 'ubuntu-18.04' - llvm: '3.8' - - distro: 'ubuntu-20.04' - llvm: '3.8' - - distro: 'ubuntu-22.04' + # CUDA with LLVM 9 only: + - llvm: '5.0' + cuda: '1' + - llvm: '6.0' + cuda: '1' + - llvm: '7' + cuda: '1' + - llvm: '8' + cuda: '1' + - llvm: '10' + cuda: '1' + - llvm: '11' + cuda: '1' + - llvm: '12.0.1' + cuda: '1' + - llvm: '13.0.1' + cuda: '1' + - llvm: '14.0.6' + cuda: '1' + + # Moonjit with LLVM 10 only: + - llvm: '5.0' + lua: 'moonjit' + - llvm: '6.0' + lua: 'moonjit' + - llvm: '7' + lua: 'moonjit' + - llvm: '8' + lua: 'moonjit' + - llvm: '9' + lua: 'moonjit' + - llvm: '11' + lua: 'moonjit' + - llvm: '12.0.1' + lua: 'moonjit' + - llvm: '13.0.1' + lua: 'moonjit' + - llvm: '14.0.6' + lua: 'moonjit' + + # no-static with LLVM 10 only: + - llvm: '5.0' + static: '0' + - llvm: '6.0' + static: '0' + - llvm: '7' + static: '0' + - llvm: '8' + static: '0' + - llvm: '9' + static: '0' + - llvm: '11' + static: '0' + - llvm: '12.0.1' + static: '0' + - llvm: '13.0.1' + static: '0' + - llvm: '14.0.6' + static: '0' + + # no-slib with LLVM 10 only: + - llvm: '5.0' + slib: '0' + - llvm: '6.0' + slib: '0' + - llvm: '7' + slib: '0' + - llvm: '8' + slib: '0' + - llvm: '9' + slib: '0' + - llvm: '11' + slib: '0' + - llvm: '12.0.1' + slib: '0' + - llvm: '13.0.1' + slib: '0' + - llvm: '14.0.6' + slib: '0' + + # slib requires static + - static: '0' + slib: '1' + + include: + # Assign variants to the basic configurations: + - variant: 'package' + - llvm: '11' + variant: 'upstream' + - llvm: '12.0.1' + variant: 'prebuilt' + - llvm: '13.0.1' + variant: 'prebuilt' + - llvm: '14.0.6' + variant: 'prebuilt' + + # Some additional configurations: + + # LLVM 3.8 on Ubuntu 16.04 + - distro: 'ubuntu-16.04' llvm: '3.8' - - distro: 'ubuntu-22.04' - llvm: '6.0' + lua: 'luajit' + cuda: '0' + variant: 'upstream' + test: '1' + + # LLVM 6.0 on Ubuntu 16.04, 20.04 - distro: 'ubuntu-16.04' - llvm: '12' - - distro: 'ubuntu-18.04' - llvm: '12' + llvm: '6.0' + lua: 'luajit' + cuda: '0' + variant: 'package' + test: '1' - distro: 'ubuntu-20.04' + llvm: '6.0' + lua: 'luajit' + cuda: '0' + variant: 'package' + test: '1' + + # LLVM 12, 13, 14 on Ubuntu 22.04 + - distro: 'ubuntu-22.04' llvm: '12' - - distro: 'ubuntu-16.04' - llvm: '14.0.6' - - distro: 'ubuntu-18.04' - llvm: '14.0.6' - - distro: 'ubuntu-20.04' - llvm: '14.0.6' + lua: 'luajit' + cuda: '0' + variant: 'package' + test: '1' + - distro: 'ubuntu-22.04' + llvm: '13' + lua: 'luajit' + cuda: '0' + variant: 'package' + test: '1' + - distro: 'ubuntu-22.04' + llvm: '14' + lua: 'luajit' + cuda: '0' + variant: 'package' + test: '1' steps: - uses: actions/checkout@v1 - run: ./travis.sh env: - DOCKER_BUILD: ${{ matrix.distro }} + DOCKER_DISTRO: ${{ matrix.distro }} DOCKER_LLVM: ${{ matrix.llvm }} + DOCKER_LUA: ${{ matrix.lua }} + DOCKER_STATIC: ${{ matrix.static }} + DOCKER_SLIB: ${{ matrix.slib }} + DOCKER_CUDA: ${{ matrix.cuda }} + DOCKER_VARIANT: ${{ matrix.variant }} + DOCKER_TEST: ${{ matrix.test }} multiarch: - name: Multiarch + name: Multiarch (${{ matrix.distro }}, ${{ matrix.arch }}, llvm-${{ matrix.llvm }}, cuda=${{ matrix.cuda }}, ${{ matrix.variant }}) runs-on: ubuntu-latest strategy: matrix: distro: ['ubuntu-18.04'] arch: ['arm64', 'ppc64le'] - llvm: ['13.0.0', '11.1.0'] + llvm: ['11.1.0', '13.0.0'] + variant: ['prebuilt'] + cuda: ['0'] + test: ['0'] exclude: - arch: 'arm64' llvm: '13.0.0' @@ -257,9 +337,12 @@ jobs: - uses: actions/checkout@v1 - run: ./travis.sh env: - DOCKER_BUILD: ${{ matrix.distro }} + DOCKER_DISTRO: ${{ matrix.distro }} DOCKER_ARCH: ${{ matrix.arch }} DOCKER_LLVM: ${{ matrix.llvm }} + DOCKER_CUDA: ${{ matrix.cuda }} + DOCKER_VARIANT: ${{ matrix.variant }} + DOCKER_TEST: ${{ matrix.test }} - uses: actions/upload-artifact@v2 with: name: docker-${{ matrix.distro }}-${{ matrix.arch }}-llvm-${{ matrix.llvm }} @@ -271,7 +354,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - run: ./docker/compatibility_test.sh ubuntu 18.04 "18.04 20.04 22.04" "" 13.0.1 prebuilt 2 + - run: ./docker/compatibility_test.sh ubuntu 18.04 "18.04 20.04 22.04" "" 13.0.1 luajit prebuilt 2 - uses: actions/upload-artifact@v2 with: name: docker-ubuntu-18.04-x86_64-llvm-13 diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index a300698b7..15b4e7b38 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -3,6 +3,12 @@ ARG release=18.04 FROM ubuntu:$release ARG llvm=6.0 +ARG lua= +ARG static= +ARG slib= +ARG cuda=0 +ARG variant=package +ARG test=1 ARG threads=4 ENV DEBIAN_FRONTEND noninteractive @@ -10,9 +16,4 @@ ENV CI 1 # skip CUDA tests COPY . /terra -RUN apt-get update -qq && \ - apt-get install -qq build-essential cmake git llvm-$llvm-dev libclang-$llvm-dev clang-$llvm libedit-dev libncurses5-dev zlib1g-dev libpfm4-dev && \ - cd /terra/build && \ - cmake -DCMAKE_INSTALL_PREFIX=/terra_install .. && \ - make install -j$threads && \ - ctest --output-on-failure -j$threads +RUN cd /terra && ./docker/install_script.sh diff --git a/docker/Dockerfile.ubuntu-prebuilt b/docker/Dockerfile.ubuntu-prebuilt deleted file mode 100644 index 1c077c5de..000000000 --- a/docker/Dockerfile.ubuntu-prebuilt +++ /dev/null @@ -1,24 +0,0 @@ -ARG release=18.04 - -FROM ubuntu:$release - -ARG arch=x86_64 -ARG llvm=6.0 -ARG threads=4 - -ENV DEBIAN_FRONTEND noninteractive -ENV CI 1 # skip CUDA tests - -COPY . /terra - -RUN apt-get update -qq && \ - apt-get install -qq build-essential cmake git python3 wget && \ - wget -nv https://github.com/terralang/llvm-build/releases/download/llvm-$llvm/clang+llvm-$llvm-$arch-linux-gnu.tar.xz && \ - tar xf clang+llvm-$llvm-$arch-linux-gnu.tar.xz && \ - mv clang+llvm-$llvm-$arch-linux-gnu /llvm && \ - rm clang+llvm-$llvm-$arch-linux-gnu.tar.xz && \ - /terra/docker/install_cuda.sh && \ - cd /terra/build && \ - cmake -DCMAKE_PREFIX_PATH=/llvm/install -DCMAKE_INSTALL_PREFIX=/terra_install .. && \ - make install -j$threads && \ - ctest --output-on-failure -j$threads diff --git a/docker/Dockerfile.ubuntu-prebuilt-multiarch b/docker/Dockerfile.ubuntu-prebuilt-multiarch deleted file mode 100644 index 67683fd5a..000000000 --- a/docker/Dockerfile.ubuntu-prebuilt-multiarch +++ /dev/null @@ -1,24 +0,0 @@ -ARG release=18.04 - -FROM ubuntu:$release - -ARG arch=x86_64 -ARG llvm=6.0 -ARG threads=4 - -ENV DEBIAN_FRONTEND noninteractive -ENV CI 1 # skip CUDA tests - -COPY . /terra - -RUN apt-get update -qq && \ - apt-get install -qq build-essential cmake git python3 wget && \ - wget -nv https://github.com/terralang/llvm-build/releases/download/llvm-$llvm/clang+llvm-$llvm-$arch-linux-gnu.tar.xz && \ - tar xf clang+llvm-$llvm-$arch-linux-gnu.tar.xz && \ - mv clang+llvm-$llvm-$arch-linux-gnu /llvm && \ - rm clang+llvm-$llvm-$arch-linux-gnu.tar.xz && \ - echo "disabled: /terra/docker/install_cuda.sh" && \ - cd /terra/build && \ - cmake -DCMAKE_PREFIX_PATH=/llvm/install -DCMAKE_INSTALL_PREFIX=/terra_install .. && \ - make install -j$threads && \ - echo "disabled: ctest --output-on-failure -j$threads" diff --git a/docker/Dockerfile.ubuntu-source b/docker/Dockerfile.ubuntu-source deleted file mode 100644 index d6cca80bd..000000000 --- a/docker/Dockerfile.ubuntu-source +++ /dev/null @@ -1,31 +0,0 @@ -ARG release=18.04 - -FROM ubuntu:$release - -ARG llvm=6.0 -ARG threads=4 - -ENV DEBIAN_FRONTEND noninteractive -ENV CI 1 # skip CUDA tests - -COPY . /terra - -RUN apt-get update -qq && \ - apt-get install -qq build-essential cmake git python3 wget && \ - mkdir /llvm && \ - cd /llvm && \ - wget -nv https://releases.llvm.org/$llvm/llvm-$llvm.src.tar.xz && \ - wget -nv https://releases.llvm.org/$llvm/cfe-$llvm.src.tar.xz && \ - tar xf llvm-$llvm.src.tar.xz && \ - tar xf cfe-$llvm.src.tar.xz && \ - mv cfe-$llvm.src llvm-$llvm.src/tools/clang && \ - mkdir build install && \ - cd build && \ - cmake ../llvm-$llvm.src -DCMAKE_INSTALL_PREFIX=$PWD/../install -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_LIBEDIT=OFF -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_ENABLE_ASSERTIONS=OFF && \ - make install -j$threads && \ - cd .. && \ - rm -rf llvm-$llvm.src* cfe-$llvm.src* build && \ - cd /terra/build && \ - cmake -DCMAKE_PREFIX_PATH=/llvm/install -DCMAKE_INSTALL_PREFIX=/terra_install .. && \ - make install -j$threads && \ - ctest --output-on-failure -j$threads diff --git a/docker/Dockerfile.ubuntu-upstream b/docker/Dockerfile.ubuntu-upstream deleted file mode 100644 index 03e3dd23a..000000000 --- a/docker/Dockerfile.ubuntu-upstream +++ /dev/null @@ -1,25 +0,0 @@ -ARG release=18.04 - -FROM ubuntu:$release - -ARG llvm=6.0 -ARG threads=4 - -ENV DEBIAN_FRONTEND noninteractive - -COPY . /terra - -RUN apt-get update -qq && \ - apt-get install -qq wget software-properties-common apt-transport-https ca-certificates && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - . /etc/lsb-release && \ - add-apt-repository -y "deb http://apt.llvm.org/$DISTRIB_CODENAME/ llvm-toolchain-$DISTRIB_CODENAME-$llvm main" && \ - apt-get update -qq && \ - echo 'Package: *' >> /etc/apt/preferences.d/llvm-600 && \ - echo 'Pin: origin apt.llvm.org' >> /etc/apt/preferences.d/llvm-600 && \ - echo 'Pin-Priority: 600' >> /etc/apt/preferences.d/llvm-600 && \ - apt-get install -qq build-essential cmake git llvm-$llvm-dev libclang-$llvm-dev clang-$llvm libedit-dev libncurses5-dev zlib1g-dev && \ - cd /terra/build && \ - cmake -DCMAKE_INSTALL_PREFIX=/terra_install .. && \ - make install -j$threads && \ - ctest --output-on-failure -j$threads diff --git a/docker/build.sh b/docker/build.sh index 8edde4024..0a69b2c9f 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -5,16 +5,19 @@ set -e IFS=- read distro release <<< "$1" arch="$2" llvm="$3" -variant="$4" -threads="$5" +lua="$4" +static="$5" +slib="$6" +cuda="$7" +variant="$8" +test="$9" +threads="${10}" if [[ -n $arch ]]; then export DOCKER_BUILDKIT=1 fi -arch_long=$(echo $arch | sed -e s/arm64/aarch64/ | sed -e s/ppc64le/powerpc64le/) - -docker build ${arch:+--platform=}$arch --build-arg release=$release ${arch:+--build-arg arch=}$arch_long --build-arg llvm=$llvm --build-arg threads=${threads:-4} -t terralang/terra:$distro-$release -f docker/Dockerfile.$distro${variant:+-}$variant . +docker build ${arch:+--platform=}$arch --build-arg release=$release --build-arg llvm=$llvm --build-arg lua=$lua --build-arg static=$static --build-arg slib=$slib --build-arg cuda=$cuda --build-arg variant=$variant --build-arg test=$test --build-arg threads=${threads:-4} -t terralang/terra:$distro-$release -f docker/Dockerfile.$distro . # Copy files out of container and make release. tmp=$(docker create terralang/terra:$distro-$release) diff --git a/docker/compatibility_test.sh b/docker/compatibility_test.sh index 57bd53894..0bc18a7a9 100755 --- a/docker/compatibility_test.sh +++ b/docker/compatibility_test.sh @@ -7,10 +7,15 @@ build_version="$2" test_versions="$3" arch="$4" llvm="$5" -variant="$6" -threads="$7" +lua="$6" +static=1 +slib=1 +cuda=1 +variant="$7" +test=1 +threads="$8" -./docker/build.sh "${distro}-${build_version}" "$arch" "$llvm" "$variant" "$threads" +./docker/build.sh "${distro}-${build_version}" "$arch" "$llvm" "$lua" "$static" "$slib" "$cuda" "$variant" "$test" "$threads" cd terra_install diff --git a/docker/install_script.sh b/docker/install_script.sh new file mode 100755 index 000000000..e175b2249 --- /dev/null +++ b/docker/install_script.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +set -e + +root_dir="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +echo "######################################################################" +echo "### Docker Build Configuration:" +echo "### * LLVM: $llvm" +echo "### * Lua: ${lua:-(auto)}" +echo "### * Static: ${static:-(auto)}" +echo "### * Slib: ${slib:-(auto)}" +echo "### * CUDA: $cuda" +echo "### * Variant: $variant" +echo "### * Test: $test" +echo "### * Threads: $threads" +echo "######################################################################" + +# Check all the variables are set. +[[ -n $llvm ]] +# $lua is optional +# $static is optional +# $slib is optional +[[ -n $cuda ]] +[[ -n $variant ]] +[[ -n $test ]] +[[ -n $threads ]] + +arch=$(uname -m | sed -e s/ppc64le/powerpc64le/) + +packages=( + build-essential cmake git +) +if [[ $variant = "package" || $variant = "upstream" ]]; then + packages+=( + llvm-$llvm-dev libclang-$llvm-dev clang-$llvm + libedit-dev libncurses5-dev zlib1g-dev + ) + if [[ $llvm -ge 13 ]]; then + packages+=( + libmlir-$llvm-dev + libpfm4-dev + ) + fi +elif [[ $variant = "prebuilt" ]]; then + packages+=( + wget + ) +else + echo "Don't know this variant: $variant" + exit 1 +fi +if [[ $cuda -eq 1 ]]; then + packages+=( + wget + ) +fi + +set -x + +apt-get update -qq + +if [[ $variant = "upstream" ]]; then + apt-get install -qq wget software-properties-common apt-transport-https ca-certificates + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + source /etc/lsb-release + add-apt-repository -y "deb http://apt.llvm.org/$DISTRIB_CODENAME/ llvm-toolchain-$DISTRIB_CODENAME-$llvm main" + apt-get update -qq + echo 'Package: *' >> /etc/apt/preferences.d/llvm-600 + echo 'Pin: origin apt.llvm.org' >> /etc/apt/preferences.d/llvm-600 + echo 'Pin-Priority: 600' >> /etc/apt/preferences.d/llvm-600 +fi + +apt-get install -qq "${packages[@]}" + +if [[ $variant = "prebuilt" ]]; then + wget -nv https://github.com/terralang/llvm-build/releases/download/llvm-$llvm/clang+llvm-$llvm-$arch-linux-gnu.tar.xz + tar xf clang+llvm-$llvm-$arch-linux-gnu.tar.xz + mv clang+llvm-$llvm-$arch-linux-gnu /llvm + rm clang+llvm-$llvm-$arch-linux-gnu.tar.xz +fi + +if [[ $cuda -eq 1 ]]; then + "$root_dir/install_cuda.sh" +else + echo "disabled: $root_dir/install_cuda.sh" +fi + +cmake_flags=() +if [[ -n $lua ]]; then + cmake_flags+=( + -DTERRA_LUA="$lua" + ) +fi +if [[ -n $static && $static -eq 0 ]]; then + cmake_flags+=( + -DTERRA_STATIC_LINK_LLVM=OFF + -DTERRA_STATIC_LINK_LUAJIT=OFF + ) +fi +if [[ -n $slib && $slib -eq 0 ]]; then + cmake_flags+=( + -DTERRA_SLIB_INCLUDE_LLVM=OFF + -DTERRA_SLIB_INCLUDE_LUAJIT=OFF + ) +fi +if [[ $cuda -eq 1 ]]; then + # Terra should autodetect, but force an error if it doesn't work. + cmake_flags+=( + -DTERRA_ENABLE_CUDA=ON + ) +fi + +cd build +cmake -DCMAKE_PREFIX_PATH=/llvm/install -DCMAKE_INSTALL_PREFIX=/terra_install "${cmake_flags[@]}" .. +make install -j$threads + +if [[ $test -eq 1 ]]; then + ctest --output-on-failure -j$threads +else + echo "disabled: ctest --output-on-failure -j$threads" +fi diff --git a/docker/install_script_multiarch.sh b/docker/install_script_multiarch.sh deleted file mode 100755 index 978087eaf..000000000 --- a/docker/install_script_multiarch.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# because Cirrus gives us ARM containers and not VMs, we have to -# extract the body of Dockerfile.ubuntu-prebuilt-multiarch here - -set -e - -apt-get update -qq -apt-get install -qq build-essential cmake git python3 wget -wget -nv https://github.com/terralang/llvm-build/releases/download/llvm-$llvm/clang+llvm-$llvm-$arch-linux-gnu.tar.xz -tar xf clang+llvm-$llvm-$arch-linux-gnu.tar.xz -mv clang+llvm-$llvm-$arch-linux-gnu /llvm -rm clang+llvm-$llvm-$arch-linux-gnu.tar.xz -echo "disabled: /terra/docker/install_cuda.sh" -cd build -cmake -DCMAKE_PREFIX_PATH=/llvm/install -DCMAKE_INSTALL_PREFIX=/terra_install .. -make install -j$threads -ctest --output-on-failure -j$threads diff --git a/travis.sh b/travis.sh index 69d50adc2..7bce9c007 100755 --- a/travis.sh +++ b/travis.sh @@ -5,8 +5,8 @@ set -x if [[ $CHECK_CLANG_FORMAT -eq 1 ]]; then if [[ $(uname) = Linux ]]; then - sudo apt-get install -y clang-format-9 - export PATH="/usr/lib/llvm-9/bin:$PATH" + sudo apt-get install -y clang-format-14 + export PATH="/usr/lib/llvm-14/bin:$PATH" else exit 1 fi @@ -19,83 +19,27 @@ if [[ $CHECK_CLANG_FORMAT -eq 1 ]]; then exit 0 fi -if [[ -n $DOCKER_BUILD ]]; then +if [[ -n $DOCKER_DISTRO ]]; then if [[ -n $DOCKER_ARCH ]]; then docker run --rm --privileged multiarch/qemu-user-static --reset -p yes fi - variant= - if [[ $DOCKER_LLVM = "3.8" ]]; then - variant=upstream - elif [[ $DOCKER_LLVM = *"."*"."* ]]; then - variant=prebuilt - fi - if [[ -n $DOCKER_ARCH ]]; then - variant=${variant}${variant:+-}multiarch - fi - ./docker/build.sh $DOCKER_BUILD "$DOCKER_ARCH" $DOCKER_LLVM $variant + ./docker/build.sh "$DOCKER_DISTRO" "$DOCKER_ARCH" "$DOCKER_LLVM" "$DOCKER_LUA" "$DOCKER_STATIC" "$DOCKER_SLIB" "$DOCKER_CUDA" "$DOCKER_VARIANT" "$DOCKER_TEST" exit 0 fi if [[ $(uname) = Linux ]]; then distro_name="$(lsb_release -cs)" sudo apt-get update -qq - if [[ $LLVM_CONFIG = llvm-config-13 ]]; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo add-apt-repository -y "deb http://apt.llvm.org/${distro_name}/ llvm-toolchain-${distro_name}-13 main" - for i in {1..5}; do sudo apt-get update -qq && break || sleep 15; done - sudo apt-get install -y llvm-13-dev clang-13 libclang-13-dev libedit-dev libpfm4-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-13:/usr/share/llvm-13 - if [[ -n $STATIC_LLVM && $STATIC_LLVM -eq 0 ]]; then - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/llvm-13/lib" - fi - elif [[ $LLVM_CONFIG = llvm-config-12 ]]; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo add-apt-repository -y "deb http://apt.llvm.org/${distro_name}/ llvm-toolchain-${distro_name}-12 main" - for i in {1..5}; do sudo apt-get update -qq && break || sleep 15; done - sudo apt-get install -y llvm-12-dev clang-12 libclang-12-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-12:/usr/share/llvm-12 - if [[ -n $STATIC_LLVM && $STATIC_LLVM -eq 0 ]]; then - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/llvm-12/lib" - fi - elif [[ $LLVM_CONFIG = llvm-config-11 ]]; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo add-apt-repository -y "deb http://apt.llvm.org/${distro_name}/ llvm-toolchain-${distro_name}-11 main" - for i in {1..5}; do sudo apt-get update -qq && break || sleep 15; done - sudo apt-get install -y llvm-11-dev clang-11 libclang-11-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-11:/usr/share/llvm-11 - if [[ -n $STATIC_LLVM && $STATIC_LLVM -eq 0 ]]; then - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/llvm-11/lib" - fi - elif [[ $LLVM_CONFIG = llvm-config-10 ]]; then - sudo apt-get install -y llvm-10-dev clang-10 libclang-10-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-10:/usr/share/llvm-10 - elif [[ $LLVM_CONFIG = llvm-config-9 ]]; then - sudo apt-get install -y llvm-9-dev clang-9 libclang-9-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-9:/usr/share/llvm-9 - elif [[ $LLVM_CONFIG = llvm-config-8 ]]; then - sudo apt-get install -y llvm-8-dev clang-8 libclang-8-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-8:/usr/share/llvm-8 - elif [[ $LLVM_CONFIG = llvm-config-7 ]]; then - sudo apt-get install -y llvm-7-dev clang-7 libclang-7-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-7:/usr/share/llvm-7 - elif [[ $LLVM_CONFIG = llvm-config-6.0 ]]; then - sudo apt-get install -qq llvm-6.0-dev clang-6.0 libclang-6.0-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-6.0:/usr/share/llvm-6.0 - elif [[ $LLVM_CONFIG = llvm-config-5.0 ]]; then - sudo apt-get install -qq llvm-5.0-dev clang-5.0 libclang-5.0-dev libedit-dev - export CMAKE_PREFIX_PATH=/usr/lib/llvm-5.0:/usr/share/llvm-5.0 + if [[ $LLVM_CONFIG = llvm-config-14 ]]; then + sudo apt-get install -y llvm-14-dev clang-14 libclang-14-dev libedit-dev libpfm4-dev else echo "Don't know this LLVM version: $LLVM_CONFIG" + echo "(If you're looking for more configurations, see Docker tests.)" exit 1 fi - if [[ $USE_CUDA -eq 1 ]]; then - ./docker/install_cuda.sh sudo - fi -fi - -if [[ $(uname) = Darwin ]]; then +elif [[ $(uname) = Darwin ]]; then if [[ $LLVM_CONFIG = llvm-config-14 ]]; then curl -L -O https://github.com/terralang/llvm-build/releases/download/llvm-14.0.6/clang+llvm-14.0.6-x86_64-apple-darwin.tar.xz tar xf clang+llvm-14.0.6-x86_64-apple-darwin.tar.xz @@ -173,9 +117,8 @@ if [[ $(uname) = Darwin ]]; then fi export PATH=$PWD:$PATH -fi -if [[ $(uname) = MINGW* ]]; then +elif [[ $(uname) = MINGW* ]]; then if [[ $LLVM_CONFIG = llvm-config-14 ]]; then curl -L -O https://github.com/terralang/llvm-build/releases/download/llvm-14.0.0/clang+llvm-14.0.0-x86_64-windows-msvc17.7z 7z x -y clang+llvm-14.0.0-x86_64-windows-msvc17.7z @@ -195,6 +138,14 @@ if [[ $(uname) = MINGW* ]]; then export CMAKE_GENERATOR="Visual Studio 17 2022" export CMAKE_GENERATOR_PLATFORM=x64 export CMAKE_GENERATOR_TOOLSET="host=x64" + +elif [[ $(uname) = FreeBSD ]]; then + # Nothing to do, everything has already been installed + echo + +else + echo "Don't know how to run tests on this OS: $(uname)" + exit 1 fi if [[ $USE_CMAKE -eq 1 ]]; then