From 9862de4d565074cfcdf8b76248ed20b28514efbd Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 28 Feb 2024 20:18:14 -0500 Subject: [PATCH] ci: cache test HIPO files and `hipo` build (#121) --- .github/install-dependency-packages.sh | 5 +- .github/workflows/ci.yml | 131 ++++++++++++++----------- .github/workflows/linux.yml | 1 + .github/workflows/macos.yml | 1 + .github/workflows/minver.yml | 1 + 5 files changed, 83 insertions(+), 56 deletions(-) diff --git a/.github/install-dependency-packages.sh b/.github/install-dependency-packages.sh index fab1ab5d..31b8dffd 100755 --- a/.github/install-dependency-packages.sh +++ b/.github/install-dependency-packages.sh @@ -49,6 +49,9 @@ verset=$2 summary_file=pkg_summary.md > $summary_file +this_script=${BASH_SOURCE[0]:-$0} +this_dir=$(cd $(dirname $this_script)/.. && pwd -P) + case $verset in latest) echo "[+] Prefer latest version for iguana-specific dependencies" ;; minver) echo "[+] Prefer minimum required version for iguana-specific dependencies" ;; @@ -87,7 +90,7 @@ case $runner in echo "[+] INSTALLING PACKAGE $pkg" case $verset in latest) pacman -S --noconfirm $pkg ;; - minver) pacman -U --noconfirm $(meson/minimum-version.sh $pkg ALA) ;; + minver) pacman -U --noconfirm $($this_dir/meson/minimum-version.sh $pkg ALA) ;; esac info_pacman $pkg done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed4758ea..dd58c70e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,10 @@ name: CI on: workflow_call: inputs: + id: + description: 'ID' + required: true + type: string runner: description: 'GitHub runner' required: true @@ -52,21 +56,23 @@ jobs: # download test data ######################################################### - download_validation_files: - name: Download validation files - runs-on: ${{ inputs.runner }} + download_test_data: + runs-on: ubuntu-latest + env: + type: physics steps: - - name: download - run: wget --no-check-certificate http://clasweb.jlab.org/clas12offline/distribution/clas12-timeline/validation_files.tar.gz - - name: untar - run: tar xzvf validation_files.tar.gz - - name: select one file - run: mv -v $(find validation_files -type f -name "*.hipo" | head -n1) test_data.hipo - - uses: actions/upload-artifact@v4 + - uses: actions/cache@v4 + id: cache with: - name: validation_files - retention-days: 1 + key: test_data path: test_data.hipo + lookup-only: true + - name: download + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: | + wget -nv --no-check-certificate http://clasweb.jlab.org/clas12offline/distribution/clas12-timeline/validation_files_${{ env.type }}.tar.zst + tar xvf validation_files_${{ env.type }}.tar.zst + mv -v $(find validation_files -type f -name "*.hipo" | head -n1) test_data.hipo # dependencies ######################################################### @@ -77,26 +83,38 @@ jobs: container: image: ${{ inputs.container }} steps: + - uses: actions/cache/restore@v4 + id: cache + with: + key: hipo-${{ inputs.id }}-${{ env.hipo_version }} + path: hipo.tar.zst + lookup-only: true - name: checkout iguana for dependency installation script + if: ${{ steps.cache.outputs.cache-hit != 'true' }} uses: actions/checkout@v4 - - name: install dependency packages - run: .github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} + with: + path: iguana_src - name: checkout hipo + if: ${{ steps.cache.outputs.cache-hit != 'true' }} uses: actions/checkout@v4 with: repository: gavalian/hipo ref: ${{ env.hipo_version }} + path: hipo_src - name: build + if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | - cmake -S . -B build -G Ninja --install-prefix $(pwd)/hipo -DCMAKE_POSITION_INDEPENDENT_CODE=ON # using PIE build, for sanitizer readibility + iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} + cmake -S hipo_src -B build -G Ninja --install-prefix $(pwd)/hipo -DCMAKE_POSITION_INDEPENDENT_CODE=ON # using PIE build, for sanitizer readibility cmake --build build cmake --install build - tar czvf hipo{.tar.gz,} - - uses: actions/upload-artifact@v4 + tar cavf hipo{.tar.zst,} + - uses: actions/cache/save@v4 + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + id: cache_save with: - name: install_deps_hipo - retention-days: 1 - path: hipo.tar.gz + key: hipo-${{ inputs.id }}-${{ env.hipo_version }} + path: hipo.tar.zst # build ######################################################### @@ -132,13 +150,13 @@ jobs: ### dependencies - name: install dependency packages run: .github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} - - name: get local dependencies - uses: actions/download-artifact@v4 + - name: get local dependency `hipo` + uses: actions/cache/restore@v4 with: - pattern: install_deps_* - merge-multiple: true + key: hipo-${{ inputs.id }}-${{ env.hipo_version }} + path: hipo.tar.zst - name: untar local dependencies - run: ls *.tar.gz | xargs -I{} tar xzvf {} + run: ls *.tar.zst | xargs -I{} tar xvf {} - name: tree local dependencies run: tree hipo - name: summarize dependencies @@ -192,19 +210,19 @@ jobs: ### upload artifacts - name: tar run: | - tar czvf iguana{.tar.gz,} - tar czvf build-iguana{.tar.gz,} + tar cavf iguana{.tar.zst,} + tar cavf build-iguana{.tar.zst,} - uses: actions/upload-artifact@v4 with: name: install_iguana_${{ matrix.id }} retention-days: 1 - path: iguana.tar.gz + path: iguana.tar.zst - uses: actions/upload-artifact@v4 if: ${{ matrix.id == 'cpp-gcc-release' || matrix.id == 'cpp-gcc-debug' || matrix.id == 'cpp-clang-release' || matrix.id == 'cpp-clang-debug' }} with: name: build_iguana_${{ matrix.id }} retention-days: 1 - path: build-iguana.tar.gz + path: build-iguana.tar.zst # run tests ######################################################### @@ -213,7 +231,7 @@ jobs: name: Test Iguana needs: - build_hipo - - download_validation_files + - download_test_data - build_iguana runs-on: ${{ inputs.runner }} container: @@ -233,9 +251,10 @@ jobs: fetch-tags: true fetch-depth: 0 - name: get test data - uses: actions/download-artifact@v4 + uses: actions/cache/restore@v4 with: - name: validation_files + key: test_data + path: test_data.hipo - name: get iguana build dir uses: actions/download-artifact@v4 with: @@ -243,13 +262,13 @@ jobs: ### dependencies - name: install dependency packages run: .github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} - - name: get local dependencies - uses: actions/download-artifact@v4 + - name: get local dependency `hipo` + uses: actions/cache/restore@v4 with: - pattern: install_deps_* - merge-multiple: true + key: hipo-${{ inputs.id }}-${{ env.hipo_version }} + path: hipo.tar.zst - name: untar local dependencies - run: ls *.tar.gz | xargs -I{} tar xzvf {} + run: ls *.tar.zst | xargs -I{} tar xvf {} ### rebuild iguana - name: meson wipe builddir for macOS re-link # macOS Homebrew installation prefix varies among runners, so wipe the builddir to force re-linking if: ${{ inputs.runner == 'macos-latest' }} @@ -318,7 +337,7 @@ jobs: test_examples: name: Test Installed Examples needs: - - download_validation_files + - download_test_data - build_iguana runs-on: ${{ inputs.runner }} container: @@ -337,23 +356,24 @@ jobs: - name: install dependency packages run: .github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} working-directory: iguana_src - - name: get dependency build artifacts - uses: actions/download-artifact@v4 + - name: get local dependency `hipo` + uses: actions/cache/restore@v4 with: - pattern: install_deps_* - merge-multiple: true + key: hipo-${{ inputs.id }}-${{ env.hipo_version }} + path: hipo.tar.zst - name: get iguana build artifacts uses: actions/download-artifact@v4 with: name: install_iguana_${{ matrix.id }} - name: get test data - uses: actions/download-artifact@v4 + uses: actions/cache/restore@v4 with: - name: validation_files + key: test_data + path: test_data.hipo - name: untar artifacts run: | - ls *.tar.gz | xargs -I{} tar xzvf {} - rm -v *.tar.gz + ls *.tar.zst | xargs -I{} tar xvf {} + rm -v *.tar.zst - name: tree artifacts run: tree ### setup python virtaul environment (for python binding tests) @@ -390,7 +410,7 @@ jobs: test_consumer_builds: name: Test consumer builds needs: - - download_validation_files + - download_test_data - build_iguana runs-on: ${{ inputs.runner }} container: @@ -406,23 +426,24 @@ jobs: - uses: actions/checkout@v4 - name: install dependency packages run: .github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} - - name: get dependency build artifacts - uses: actions/download-artifact@v4 + - name: get local dependency `hipo` + uses: actions/cache/restore@v4 with: - pattern: install_deps_* - merge-multiple: true + key: hipo-${{ inputs.id }}-${{ env.hipo_version }} + path: hipo.tar.zst - name: get iguana build artifacts uses: actions/download-artifact@v4 with: name: install_iguana_${{ env.build_id }} - name: get test data - uses: actions/download-artifact@v4 + uses: actions/cache/restore@v4 with: - name: validation_files + key: test_data + path: test_data.hipo - name: untar artifacts run: | - ls *.tar.gz | xargs -I{} tar xzvf {} - rm -v *.tar.gz + ls *.tar.zst | xargs -I{} tar xvf {} + rm -v *.tar.zst ### set env vars - name: set cmake prefix path if: ${{ matrix.tool == 'cmake' }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 823d370d..88546956 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,5 +15,6 @@ jobs: name: Linux (latest) uses: ./.github/workflows/ci.yml with: + id: linux-latest runner: ubuntu-latest container: archlinux/archlinux:latest diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 8d35745d..fb0ee130 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -15,6 +15,7 @@ jobs: name: macOS (latest) uses: ./.github/workflows/ci.yml with: + id: macOS runner: macos-latest test_matrix: >- { diff --git a/.github/workflows/minver.yml b/.github/workflows/minver.yml index 2beabf50..cd5b3b21 100644 --- a/.github/workflows/minver.yml +++ b/.github/workflows/minver.yml @@ -15,6 +15,7 @@ jobs: name: Linux (minver) uses: ./.github/workflows/ci.yml with: + id: linux-minver runner: ubuntu-latest container: archlinux/archlinux:latest verset: minver