From 4dfeb0c99aa6507be26f4bf200810563fedafbf2 Mon Sep 17 00:00:00 2001 From: Kyle Shores Date: Wed, 20 Mar 2024 14:08:58 -0500 Subject: [PATCH] separating build into docker and gcc (#54) * separating build into docker and gcc * fixing pages build * run tests in serial * not running tests in parallel for focker * making sure memcheck happens --- .github/workflows/docker.yml | 50 +++++++++++ .github/workflows/gh-pages.yml | 6 +- .github/workflows/test.yml | 85 ------------------- .github/workflows/ubuntu.yml | 35 ++++++++ CMakeLists.txt | 2 +- README.md | 3 +- Dockerfile => docker/Dockerfile | 1 - .../Dockerfile.coverage | 0 Dockerfile.docs => docker/Dockerfile.docs | 0 docker/Dockerfile.memcheck | 32 +++++++ Dockerfile.mpi => docker/Dockerfile.mpi | 1 - .../Dockerfile.mpi.memcheck | 0 12 files changed, 123 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/ubuntu.yml rename Dockerfile => docker/Dockerfile (93%) rename Dockerfile.memcheck => docker/Dockerfile.coverage (100%) rename Dockerfile.docs => docker/Dockerfile.docs (100%) create mode 100644 docker/Dockerfile.memcheck rename Dockerfile.mpi => docker/Dockerfile.mpi (96%) rename Dockerfile.mpi.memcheck => docker/Dockerfile.mpi.memcheck (100%) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..be54d833 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,50 @@ +name: Docker + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + docker-build-and-test: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + name: Build and Test - ${{ matrix.dockerfile }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + dockerfile: + - Dockerfile + - Dockerfile.coverage + - Dockerfile.memcheck + - Dockerfile.mpi + - Dockerfile.mpi.memcheck + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Build Docker image + run: docker build -t tuvx -f docker/${{ matrix.dockerfile }} . + + - name: Run tests in container + if: matrix.dockerfile != 'Dockerfile.coverage' + run: docker run --name test-container -t tuvx bash -c 'make test ARGS="--rerun-failed --output-on-failure"' + + - name: Run coverage tests in container + if: matrix.dockerfile == 'Dockerfile.coverage' + run: docker run --name test-container -t tuvx bash -c 'make coverage ARGS="--rerun-failed --output-on-failure"' + + - name: Copy coverage from container + if: matrix.dockerfile == 'Dockerfile.coverage' + run: docker cp test-container:build/coverage.info . + + - name: Upload coverage report + if: matrix.dockerfile == 'Dockerfile.coverage' + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.info \ No newline at end of file diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index ed377e74..9ef87037 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -54,7 +54,7 @@ jobs: # create two copies of the documentaiton # 1. the frozen version, represented as vX.X in the version switcher - docker build -t tuvx -f Dockerfile.docs . + docker build -t tuvx -f docker/Dockerfile.docs . id=$(docker create tuvx) docker cp $id:/build/docs/sphinx tmpdocs docker rm -v $id @@ -63,7 +63,7 @@ jobs: # 2. stable, represented as vX.X (stable) in the version switcher # edit conf.py to produce a version string that looks like vX.X (stable) - docker build -t tuvx -f Dockerfile.docs --build-arg SUFFIX=" (stable)" . + docker build -t tuvx -f docker/Dockerfile.docs --build-arg SUFFIX=" (stable)" . id=$(docker create tuvx) docker cp $id:/build/docs/sphinx tmpdocs docker rm -v $id @@ -84,7 +84,7 @@ jobs: !contains(github.ref, env.DEFAULT_BRANCH) run: | set -x - docker build -t tuvx -f Dockerfile.docs . + docker build -t tuvx -f docker/Dockerfile.docs . id=$(docker create tuvx) docker cp $id:/build/docs/sphinx tmpdocs docker rm -v $id diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index ad2c34ba..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: build - -on: [ push, pull_request ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build_test_no_mpi_no_memcheck: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - name: build Docker image - run: docker build -t tuv-x-test . - - name: run tests in container - run: docker run --name test-container -t tuv-x-test bash -c 'make test ARGS="--rerun-failed --output-on-failure"' - build_test_with_mpi_no_memcheck: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - name: build Docker image for MPI tests - run: docker build -t tuv-x-mpi-test . -f Dockerfile.mpi - - name: run MPI tests in container - run: docker run -t tuv-x-mpi-test bash -c 'make test ARGS="--rerun-failed --output-on-failure"' - build_test_no_mpi_with_memcheck: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - name: build Docker image - run: docker build -t tuv-x-test . -f Dockerfile.memcheck - - name: run tests in container - run: docker run --name test-container -t tuv-x-test bash -c 'make coverage ARGS="--rerun-failed --output-on-failure"' - - name: copy coverage from container - run: docker cp test-container:build/coverage.info . - - uses: codecov/codecov-action@v2 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.info - build_test_with_mpi_with_memcheck: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - name: build Docker image for MPI tests - run: docker build -t tuv-x-mpi-test . -f Dockerfile.mpi.memcheck - - name: run MPI tests in container - run: docker run -t tuv-x-mpi-test bash -c 'make test ARGS="--rerun-failed --output-on-failure"' - gcc: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name - strategy: - matrix: - gcc_version: [11, 12, 13] - env: - FC: gfortran-${{ matrix.gcc_version }} - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libnetcdf-dev netcdf-bin libnetcdff-dev - - name: Install python dependencies - run: pip install numpy scipy - - name: Run Cmake - run: cmake -S . -B build - - name: Build - run: cmake --build build --parallel - - name: Run tests - run: | - cd build - ctest --rerun-failed --output-on-failure . --verbose -j \ No newline at end of file diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml new file mode 100644 index 00000000..8610eb18 --- /dev/null +++ b/.github/workflows/ubuntu.yml @@ -0,0 +1,35 @@ +name: Ubuntu + +on: [ push, pull_request ] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + gcc: + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + strategy: + matrix: + gcc_version: [11, 12, 13] + env: + FC: gfortran-${{ matrix.gcc_version }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libnetcdf-dev netcdf-bin libnetcdff-dev + - name: Install python dependencies + run: pip install numpy scipy + - name: Run Cmake + run: cmake -S . -B build + - name: Build + run: cmake --build build --parallel + - name: Run tests + run: | + cd build + ctest --rerun-failed --output-on-failure . --verbose \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d159c87..700484cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ option(ENABLE_MPI "Enable MPI parallel support" OFF) cmake_dependent_option(ENABLE_OPENMP "Enable OpenMP support" OFF "ENABLE_MPI" OFF) option(ENABLE_TESTS "Build tests" ON) option(ENABLE_COVERAGE "Enable code coverage output" OFF) -option(ENABLE_MEMCHECK "Enable memory checking in tests" ON) +option(ENABLE_MEMCHECK "Enable memory checking in tests" OFF) option(ENABLE_NC_CONFIG "Use nc-config to determine NetCDF libraries" OFF) option(BUILD_DOCS "Build the documentation" OFF) diff --git a/README.md b/README.md index a775bd76..4969b2bd 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ Tropospheric ultraviolet-extended (TUV-x): A photolysis rate calculator [![License](https://img.shields.io/github/license/NCAR/tuv-x.svg)](https://github.com/NCAR/tuv-x/blob/main/LICENSE) -[![CI Status](https://github.com/NCAR/tuv-x/actions/workflows/test.yml/badge.svg)](https://github.com/NCAR/tuv-x/actions/workflows/test.yml) +[![Ubuntu](https://github.com/NCAR/tuv-x/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/NCAR/tuv-x/actions/workflows/ubuntu.yml) +[![Docker](https://github.com/NCAR/tuv-x/actions/workflows/docker.yml/badge.svg)](https://github.com/NCAR/tuv-x/actions/workflows/docker.yml) [![codecov](https://codecov.io/gh/NCAR/tuv-x/branch/main/graph/badge.svg?token=H46AAEAQF9)](https://codecov.io/gh/NCAR/tuv-x) [![DOI](https://zenodo.org/badge/396946468.svg)](https://zenodo.org/badge/latestdoi/396946468) [![](https://img.shields.io/badge/Contribute%20with-Gitpod-908a85?logo=gitpod)](https://gitpod.io/#https://github.com/NCAR/tuv-x) diff --git a/Dockerfile b/docker/Dockerfile similarity index 93% rename from Dockerfile rename to docker/Dockerfile index 47cc7390..6baec36b 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -27,7 +27,6 @@ COPY . /tuv-x/ RUN mkdir /build \ && cd /build \ && cmake -D CMAKE_BUILD_TYPE=release \ - -D ENABLE_MEMCHECK=OFF \ /tuv-x \ && make install -j 8 diff --git a/Dockerfile.memcheck b/docker/Dockerfile.coverage similarity index 100% rename from Dockerfile.memcheck rename to docker/Dockerfile.coverage diff --git a/Dockerfile.docs b/docker/Dockerfile.docs similarity index 100% rename from Dockerfile.docs rename to docker/Dockerfile.docs diff --git a/docker/Dockerfile.memcheck b/docker/Dockerfile.memcheck new file mode 100644 index 00000000..ebcbec3e --- /dev/null +++ b/docker/Dockerfile.memcheck @@ -0,0 +1,32 @@ +FROM fedora:37 + +RUN dnf -y update \ + && dnf -y install \ + gcc-fortran \ + gcc-c++ \ + gcc \ + gdb \ + git \ + netcdf-fortran-devel \ + cmake \ + make \ + lcov \ + valgrind \ + python3 \ + python3-pip \ + lapack-devel \ + yaml-cpp-devel \ + && dnf clean all + +RUN pip3 install numpy scipy + +# build the tuv-x tool +COPY . /tuv-x/ +RUN mkdir /build \ + && cd /build \ + && cmake \ + -DENABLE_MEMCHECK:BOOL=TRUE \ + /tuv-x \ + && make -j 8 + +WORKDIR /build diff --git a/Dockerfile.mpi b/docker/Dockerfile.mpi similarity index 96% rename from Dockerfile.mpi rename to docker/Dockerfile.mpi index f2595b32..c04e987c 100644 --- a/Dockerfile.mpi +++ b/docker/Dockerfile.mpi @@ -42,7 +42,6 @@ RUN mkdir build \ -D CMAKE_CXX_COMPILER=/usr/lib64/openmpi/bin/mpicxx \ -D ENABLE_OPENMP:BOOL=TRUE \ -D ENABLE_MPI:BOOL=TRUE \ - -D ENABLE_MEMCHECK:BOOL=FALSE \ ../tuv-x \ && make -j 8 diff --git a/Dockerfile.mpi.memcheck b/docker/Dockerfile.mpi.memcheck similarity index 100% rename from Dockerfile.mpi.memcheck rename to docker/Dockerfile.mpi.memcheck