From db6a42ec3d126bd3a41886842aa46ceeb26e66a5 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Tue, 2 Jan 2024 17:17:42 -0600 Subject: [PATCH 1/6] Update Dockerfile.ngen to add NGEN_WITH_SQLITE: OFF Related to #63 For Testing gpkg support in NGIAB --- docker/Dockerfile.ngen | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.ngen b/docker/Dockerfile.ngen index 823de77b..861aa385 100644 --- a/docker/Dockerfile.ngen +++ b/docker/Dockerfile.ngen @@ -59,6 +59,7 @@ ARG REPO_URL \ NGEN_ACTIVATE_PYTHON="ON" \ NGEN_NETCDF_ACTIVE="ON" \ NGEN_ROUTING_ACTIVE="ON" \ + NGEN_WITH_SQLITE="ON" \ NGEN_UDUNITS_ACTIVE="ON" \ NGEN_UDUNITS_QUIET="ON" \ BUILD_NGEN_SERIAL="true" \ From cc3fb723f3ec663da7533274d47910b420639858 Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Wed, 3 Jan 2024 19:17:11 -0600 Subject: [PATCH 2/6] Update Dockerfile.ngen --- docker/Dockerfile.ngen | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile.ngen b/docker/Dockerfile.ngen index 861aa385..048e273a 100644 --- a/docker/Dockerfile.ngen +++ b/docker/Dockerfile.ngen @@ -127,6 +127,7 @@ RUN cd ${WORKDIR}/ngen \ -DBMI_FORTRAN_ACTIVE:BOOL=${NGEN_ACTIVATE_FORTRAN} \ -DNGEN_ACTIVATE_PYTHON:BOOL=${NGEN_ACTIVATE_PYTHON} \ -DNGEN_ACTIVATE_ROUTING:BOOL=${NGEN_ROUTING_ACTIVE} \ + -DNGEN_WITH_SQLITE:BOOL=${NGEN_WITH_SQLITE} \ -DUDUNITS_ACTIVE:BOOL=${NGEN_UDUNITS_ACTIVE} \ -DUDUNITS_QUIET:BOOL=${NGEN_UDUNITS_QUIET} \ -DCMAKE_INSTALL_PREFIX=${WORKDIR} \ @@ -144,6 +145,7 @@ RUN cd ${WORKDIR}/ngen \ -DBMI_FORTRAN_ACTIVE:BOOL=${NGEN_ACTIVATE_FORTRAN} \ -DNGEN_ACTIVATE_PYTHON:BOOL=${NGEN_ACTIVATE_PYTHON} \ -DNGEN_ACTIVATE_ROUTING:BOOL=${NGEN_ROUTING_ACTIVE} \ + -DNGEN_WITH_SQLITE:BOOL=${NGEN_WITH_SQLITE} \ -DUDUNITS_ACTIVE:BOOL=${NGEN_UDUNITS_ACTIVE} \ -DUDUNITS_QUIET:BOOL=${NGEN_UDUNITS_QUIET} \ -DCMAKE_INSTALL_PREFIX=${WORKDIR} \ From 68fca49c006a141b8fc1c2ea73ef0d1966eb95ce Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Thu, 4 Jan 2024 02:36:45 +0000 Subject: [PATCH 3/6] added sqlite dep --- docker/Dockerfile.ngen-deps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.ngen-deps b/docker/Dockerfile.ngen-deps index 37331530..7730221c 100644 --- a/docker/Dockerfile.ngen-deps +++ b/docker/Dockerfile.ngen-deps @@ -22,7 +22,7 @@ ARG REPO_URL=https://github.com/NOAA-OWP/ngen.git \ # netcdf-cxx4-devel netcdf-cxx4-mpich-devel netcdf-fortran-devel netcdf-fortran-mpich-devel \ # hdf5 hdf5-devel hdf5-mpich hdf5-mpich-devel \ # bzip2 expat expat-devel flex bison udunits2 udunits2-devel" - ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran \ + ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran sqlite \ python3 python3-devel python3-pip gdal gdal-devel\ bzip2 expat expat-devel flex bison udunits2 udunits2-devel zlib-devel" \ # TODO: removed texinfo from list because it couldn't be found; make sure this doesn't lead to issues From 6e49ffa45f16504bf3e74dbb7227b849a83ac97b Mon Sep 17 00:00:00 2001 From: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:19:53 -0600 Subject: [PATCH 4/6] Update Dockerfile.ngen-deps --- docker/Dockerfile.ngen-deps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.ngen-deps b/docker/Dockerfile.ngen-deps index 7730221c..1fbb479f 100644 --- a/docker/Dockerfile.ngen-deps +++ b/docker/Dockerfile.ngen-deps @@ -22,7 +22,7 @@ ARG REPO_URL=https://github.com/NOAA-OWP/ngen.git \ # netcdf-cxx4-devel netcdf-cxx4-mpich-devel netcdf-fortran-devel netcdf-fortran-mpich-devel \ # hdf5 hdf5-devel hdf5-mpich hdf5-mpich-devel \ # bzip2 expat expat-devel flex bison udunits2 udunits2-devel" - ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran sqlite \ + ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran sqlite sqlite-devel \ python3 python3-devel python3-pip gdal gdal-devel\ bzip2 expat expat-devel flex bison udunits2 udunits2-devel zlib-devel" \ # TODO: removed texinfo from list because it couldn't be found; make sure this doesn't lead to issues From a58c60fbc735c6a90025560a1f0a314f7172e4c2 Mon Sep 17 00:00:00 2001 From: Arpita Patel Date: Thu, 4 Jan 2024 14:44:43 -0600 Subject: [PATCH 5/6] Update Dockerfile.ngen --- docker/Dockerfile.ngen | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.ngen b/docker/Dockerfile.ngen index 048e273a..4235831e 100644 --- a/docker/Dockerfile.ngen +++ b/docker/Dockerfile.ngen @@ -179,3 +179,4 @@ RUN cd ${WORKDIR}/ngen \ && mpirun -n 3 cmake_build_parallel/test/test_remote_nexus \ && mpirun -n 4 cmake_build_parallel/test/test_remote_nexus \ && find cmake_build* -type f -name "*" ! \( -name "*.so" -o -name "ngen" -o -name "partitionGenerator" \) -exec rm {} + + From 49e338809ce43522b7b731d08202304dff265853 Mon Sep 17 00:00:00 2001 From: Benjamin Lee <130000039+benlee0423@users.noreply.github.com> Date: Wed, 10 Jan 2024 11:06:12 -0800 Subject: [PATCH 6/6] merge main change to working branch (#77) * Run actions when PR is created and updated. (#74) * refactor to support to run actions on pr * add debug * add docker token * add input * add env * define env * define env * define env * define env * add secrets * add secrets * secrets inherit * fix tag name * fix tag name * fix tag name * set input required * fix * use vars * fix variable * remove debug * Update HelloNGEN.sh (#75) * Use image tag based on branch (#76) * set tag name based on branch * remove unused inputs --------- Co-authored-by: ZacharyWills <60660437+ZacharyWills@users.noreply.github.com> --- .../build-and-push/action.yaml | 16 +++-- .github/workflows/docker_deps_image.yml | 7 +-- .../workflows/docker_image_main_branch.yml | 63 +++++++++++++------ .github/workflows/docker_ngen_image.yml | 16 +---- .github/workflows/docker_troute_image.yml | 16 +---- docker/HelloNGEN.sh | 17 ++++- 6 files changed, 78 insertions(+), 57 deletions(-) diff --git a/.github/action_templates/build-and-push/action.yaml b/.github/action_templates/build-and-push/action.yaml index 25266066..d7601484 100644 --- a/.github/action_templates/build-and-push/action.yaml +++ b/.github/action_templates/build-and-push/action.yaml @@ -5,9 +5,6 @@ inputs: docker-auth-token: description: 'Docker Hub authentication token' required: true - github-sha: - description: 'github sha hash for tagging image' - required: true image-name: description: 'image name to be built' required: true @@ -20,6 +17,17 @@ runs: steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + + - name: Set env vars for main branch + shell: bash + run: | + echo "TAG_NAME=latest" >> "$GITHUB_ENV" + if: startsWith(github.ref, 'refs/heads/main') + - name: Set env vars for other branch + shell: bash + run: | + echo "TAG_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV" + if: ${{ !startsWith(github.ref, 'refs/heads/main') }} - name: Login to Docker Hub shell: bash @@ -39,7 +47,7 @@ runs: push: true platforms: linux/arm64 tags: | - awiciroh/${{ inputs.image-name}}:latest + awiciroh/${{ inputs.image-name}}:${{ env.TAG_NAME}} builder: mybuilder no-cache: true env: diff --git a/.github/workflows/docker_deps_image.yml b/.github/workflows/docker_deps_image.yml index a9f19855..6c5fc896 100644 --- a/.github/workflows/docker_deps_image.yml +++ b/.github/workflows/docker_deps_image.yml @@ -4,12 +4,8 @@ on: branches: [ main ] paths: - docker/** - pull_request: - branches: [ main ] - paths: - - docker/** workflow_dispatch: - + workflow_call: jobs: build-and-push: runs-on: self-hosted @@ -23,6 +19,5 @@ jobs: uses: ./.github/action_templates/build-and-push with: docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }} - github-sha: ${{ github.sha }} image-name: "ngen-deps" dockerfile-name: "Dockerfile.ngen-deps" \ No newline at end of file diff --git a/.github/workflows/docker_image_main_branch.yml b/.github/workflows/docker_image_main_branch.yml index 1f6d4c9c..a5358776 100644 --- a/.github/workflows/docker_image_main_branch.yml +++ b/.github/workflows/docker_image_main_branch.yml @@ -4,16 +4,25 @@ on: branches: [ main ] paths: - docker/** - workflow_run: - workflows: [Build and push ngen image] - types: - - completed + pull_request: + branches: [ main ] workflow_dispatch: - + jobs: - on-success: + deps: + uses: ./.github/workflows/docker_deps_image.yml + secrets: inherit + troute: + needs: deps + uses: ./.github/workflows/docker_troute_image.yml + secrets: inherit + ngen: + needs: troute + uses: ./.github/workflows/docker_ngen_image.yml + secrets: inherit + final: + needs: ngen runs-on: self-hosted - if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: deploy uses: actions/checkout@v2 @@ -21,27 +30,41 @@ jobs: uses: ./.github/action_templates/build-and-push with: docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }} - github-sha: ${{ github.sha }} image-name: "ciroh-ngen-image" dockerfile-name: "Dockerfile" - - name: Publish slack message - if: always() - uses: ./.github/action_templates/slack - with: - slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} - job-status: ${{ job.status }} - pull_request_url: ${{ github.event.pull_request.html_url }} - head_commit_url: ${{ github.event.head_commit.url }} - on-failure: + check_result: + needs: final + runs-on: self-hosted + outputs: + output1: ${{ steps.step1.outputs.result }} + steps: + - id: step1 + run: echo "result=success" >> "$GITHUB_OUTPUT" + + slack: runs-on: self-hosted - if: ${{ github.event.workflow_run.conclusion == 'failure' }} + if: ${{ always() }} + needs: [deps, troute, ngen, final, check_result] + env: + OUTPUT: ${{needs.check_result.outputs.output1}} steps: - - run: echo 'The triggering workflow failed' + - name: Set env JOBSTATUS + run: | + echo "$OUTPUT" + if [[ "success" = "$OUTPUT" ]] + then + echo "JOBSTATUS=success" >> "$GITHUB_ENV" + else + echo "JOBSTATUS=failure" >> "$GITHUB_ENV" + fi + - run: echo "$JOBSTATUS" + - name: deploy + uses: actions/checkout@v2 - name: Publish slack message if: always() uses: ./.github/action_templates/slack with: slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} - job-status: ${{ job.status }} + job-status: ${{ env.JOBSTATUS }} pull_request_url: ${{ github.event.pull_request.html_url }} head_commit_url: ${{ github.event.head_commit.url }} \ No newline at end of file diff --git a/.github/workflows/docker_ngen_image.yml b/.github/workflows/docker_ngen_image.yml index 6aa6405e..57775c35 100644 --- a/.github/workflows/docker_ngen_image.yml +++ b/.github/workflows/docker_ngen_image.yml @@ -4,16 +4,12 @@ on: branches: [ main ] paths: - docker/** - workflow_run: - workflows: [Build and push t-route image] - types: - - completed workflow_dispatch: + workflow_call: jobs: - on-success: + build-and-push: runs-on: self-hosted - if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: deploy uses: actions/checkout@v2 @@ -21,11 +17,5 @@ jobs: uses: ./.github/action_templates/build-and-push with: docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }} - github-sha: ${{ github.sha }} image-name: "ngen" - dockerfile-name: "Dockerfile.ngen" - on-failure: - runs-on: self-hosted - if: ${{ github.event.workflow_run.conclusion == 'failure' }} - steps: - - run: echo 'The triggering workflow failed' \ No newline at end of file + dockerfile-name: "Dockerfile.ngen" \ No newline at end of file diff --git a/.github/workflows/docker_troute_image.yml b/.github/workflows/docker_troute_image.yml index acc0f945..a2ba6b65 100644 --- a/.github/workflows/docker_troute_image.yml +++ b/.github/workflows/docker_troute_image.yml @@ -4,16 +4,12 @@ on: branches: [ main ] paths: - docker/** - workflow_run: - workflows: [Build and push deps image] - types: - - completed workflow_dispatch: + workflow_call: jobs: - on-success: + build-and-push: runs-on: self-hosted - if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: deploy uses: actions/checkout@v2 @@ -21,11 +17,5 @@ jobs: uses: ./.github/action_templates/build-and-push with: docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }} - github-sha: ${{ github.sha }} image-name: "t-route" - dockerfile-name: "Dockerfile.t-route" - on-failure: - runs-on: self-hosted - if: ${{ github.event.workflow_run.conclusion == 'failure' }} - steps: - - run: echo 'The triggering workflow failed' \ No newline at end of file + dockerfile-name: "Dockerfile.t-route" \ No newline at end of file diff --git a/docker/HelloNGEN.sh b/docker/HelloNGEN.sh index deaa9b59..b5ae5ecf 100644 --- a/docker/HelloNGEN.sh +++ b/docker/HelloNGEN.sh @@ -49,6 +49,21 @@ generate_partition() { /dmod/bin/partitionGenerator "$1" "$2" "partitions_$3.json" "$3" '' '' } +if [ "$2" == "auto" ] + then + echo "AUTO MODE ENGAGED" + echo "Running NextGen model framework in parallel mode" + procs=$(nproc) + procs=2 # Temporary fixed value, remove for full utilization + generate_partition "$selected_catchment" "$selected_nexus" "$procs" + mpirun -n $procs /dmod/bin/ngen-parallel $selected_catchment all $selected_nexus all $selected_realization $(pwd)/partitions_$procs.json + echo "Run completed successfully, exiting, have a nice day!" + exit 0 + else + echo "Entering Interactive Mode" + continue +fi + echo -e "${YELLOW}Select an option (type a number): ${RESET}" options=("Run NextGen model framework in serial mode" "Run NextGen model framework in parallel mode" "Run Bash shell" "Exit") select option in "${options[@]}"; do @@ -131,4 +146,4 @@ select option in "${options[@]}"; do ;; esac done -exit \ No newline at end of file +exit