Update Dockerfile.ngen to add NGEN_WITH_SQLITE: ON

.github/action_templates/build-and-push/action.yaml

@@ -5,9 +5,6 @@ inputs:
   docker-auth-token:
     description: 'Docker Hub authentication token'
     required: true
-  github-sha:
-    description: 'github sha hash for tagging image'
-    required: true
   image-name:
     description: 'image name to be built'
     required: true
@@ -20,6 +17,17 @@ runs:
   steps:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
+
+    - name: Set env vars for main branch
+      shell: bash
+      run: |
+        echo "TAG_NAME=latest" >> "$GITHUB_ENV"
+      if: startsWith(github.ref, 'refs/heads/main')
+    - name: Set env vars for other branch
+      shell: bash
+      run: |
+        echo "TAG_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV"
+      if: ${{ !startsWith(github.ref, 'refs/heads/main') }}
 
     - name: Login to Docker Hub
       shell: bash
@@ -39,7 +47,7 @@ runs:
         push: true
         platforms: linux/arm64
         tags: |
-          awiciroh/${{ inputs.image-name}}:latest
+          awiciroh/${{ inputs.image-name}}:${{ env.TAG_NAME}}
         builder: mybuilder
         no-cache: true
       env:

.github/workflows/docker_deps_image.yml

@@ -4,12 +4,8 @@ on:
     branches: [ main ]
     paths:
       - docker/**
-  pull_request:
-    branches: [ main ]
-    paths:
-      - docker/**
   workflow_dispatch:
-    
+  workflow_call:
 jobs:
   build-and-push:
     runs-on: self-hosted
@@ -23,6 +19,5 @@ jobs:
         uses: ./.github/action_templates/build-and-push
         with:
           docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }}
-          github-sha: ${{ github.sha }}
           image-name: "ngen-deps"
           dockerfile-name: "Dockerfile.ngen-deps"

.github/workflows/docker_image_main_branch.yml

@@ -4,44 +4,67 @@ on:
     branches: [ main ]
     paths:
       - docker/**
-  workflow_run:
-    workflows: [Build and push ngen image]
-    types:
-      - completed
+  pull_request:
+    branches: [ main ]
   workflow_dispatch:
-    
+
 jobs:
-  on-success:
+  deps:
+    uses: ./.github/workflows/docker_deps_image.yml
+    secrets: inherit
+  troute:
+    needs: deps
+    uses: ./.github/workflows/docker_troute_image.yml
+    secrets: inherit
+  ngen:
+    needs: troute
+    uses: ./.github/workflows/docker_ngen_image.yml
+    secrets: inherit
+  final:
+    needs: ngen
     runs-on: self-hosted
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: deploy
         uses: actions/checkout@v2
       - name: Build and push Docker image
         uses: ./.github/action_templates/build-and-push
         with:
           docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }}
-          github-sha: ${{ github.sha }}
           image-name: "ciroh-ngen-image"
           dockerfile-name: "Dockerfile"
-      - name: Publish slack message
-        if: always()
-        uses: ./.github/action_templates/slack
-        with:
-          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
-          job-status: ${{ job.status }}
-          pull_request_url: ${{ github.event.pull_request.html_url }}
-          head_commit_url: ${{ github.event.head_commit.url }}
-  on-failure:
+  check_result:
+    needs: final
+    runs-on: self-hosted
+    outputs:
+      output1: ${{ steps.step1.outputs.result }}
+    steps:
+      - id: step1
+        run: echo "result=success" >> "$GITHUB_OUTPUT"
+
+  slack:
     runs-on: self-hosted
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+    if: ${{ always() }}
+    needs: [deps, troute, ngen, final, check_result]
+    env:
+      OUTPUT: ${{needs.check_result.outputs.output1}}
     steps:
-      - run: echo 'The triggering workflow failed'
+      - name: Set env JOBSTATUS
+        run: |
+          echo "$OUTPUT"
+          if [[ "success" = "$OUTPUT" ]]
+          then
+              echo "JOBSTATUS=success" >> "$GITHUB_ENV"
+          else
+              echo "JOBSTATUS=failure" >> "$GITHUB_ENV"
+          fi
+      - run: echo "$JOBSTATUS"
+      - name: deploy
+        uses: actions/checkout@v2
       - name: Publish slack message
         if: always()
         uses: ./.github/action_templates/slack
         with:
           slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
-          job-status: ${{ job.status }}
+          job-status: ${{ env.JOBSTATUS }}
           pull_request_url: ${{ github.event.pull_request.html_url }}
           head_commit_url: ${{ github.event.head_commit.url }}

.github/workflows/docker_ngen_image.yml

@@ -4,28 +4,18 @@ on:
     branches: [ main ]
     paths:
       - docker/**
-  workflow_run:
-    workflows: [Build and push t-route image]
-    types:
-      - completed
   workflow_dispatch:
+  workflow_call:
 
 jobs:
-  on-success:
+  build-and-push:
     runs-on: self-hosted
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: deploy
         uses: actions/checkout@v2
       - name: Build and push Docker image
         uses: ./.github/action_templates/build-and-push
         with:
           docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }}
-          github-sha: ${{ github.sha }}
           image-name: "ngen"
-          dockerfile-name: "Dockerfile.ngen"
-  on-failure:
-    runs-on: self-hosted
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
-    steps:
-      - run: echo 'The triggering workflow failed'
+          dockerfile-name: "Dockerfile.ngen"

.github/workflows/docker_troute_image.yml

@@ -4,28 +4,18 @@ on:
     branches: [ main ]
     paths:
       - docker/**
-  workflow_run:
-    workflows: [Build and push deps image]
-    types:
-      - completed
   workflow_dispatch:
+  workflow_call:
 
 jobs:
-  on-success:
+  build-and-push:
     runs-on: self-hosted
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: deploy
         uses: actions/checkout@v2
       - name: Build and push Docker image
         uses: ./.github/action_templates/build-and-push
         with:
           docker-auth-token: ${{ secrets.DOCKER_AUTH_TOKEN }}
-          github-sha: ${{ github.sha }}
           image-name: "t-route"
-          dockerfile-name: "Dockerfile.t-route"
-  on-failure:
-    runs-on: self-hosted
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
-    steps:
-      - run: echo 'The triggering workflow failed'
+          dockerfile-name: "Dockerfile.t-route"

docker/Dockerfile.ngen

@@ -59,6 +59,7 @@ ARG REPO_URL \
     NGEN_ACTIVATE_PYTHON="ON" \
     NGEN_NETCDF_ACTIVE="ON" \
     NGEN_ROUTING_ACTIVE="ON" \
+    NGEN_WITH_SQLITE="ON" \
     NGEN_UDUNITS_ACTIVE="ON" \
     NGEN_UDUNITS_QUIET="ON" \
     BUILD_NGEN_SERIAL="true" \
@@ -126,6 +127,7 @@ RUN cd ${WORKDIR}/ngen \
         -DBMI_FORTRAN_ACTIVE:BOOL=${NGEN_ACTIVATE_FORTRAN} \
         -DNGEN_ACTIVATE_PYTHON:BOOL=${NGEN_ACTIVATE_PYTHON} \
         -DNGEN_ACTIVATE_ROUTING:BOOL=${NGEN_ROUTING_ACTIVE} \
+        -DNGEN_WITH_SQLITE:BOOL=${NGEN_WITH_SQLITE} \
         -DUDUNITS_ACTIVE:BOOL=${NGEN_UDUNITS_ACTIVE} \
         -DUDUNITS_QUIET:BOOL=${NGEN_UDUNITS_QUIET} \
         -DCMAKE_INSTALL_PREFIX=${WORKDIR} \
@@ -143,6 +145,7 @@ RUN cd ${WORKDIR}/ngen \
         -DBMI_FORTRAN_ACTIVE:BOOL=${NGEN_ACTIVATE_FORTRAN} \
         -DNGEN_ACTIVATE_PYTHON:BOOL=${NGEN_ACTIVATE_PYTHON} \
         -DNGEN_ACTIVATE_ROUTING:BOOL=${NGEN_ROUTING_ACTIVE} \
+        -DNGEN_WITH_SQLITE:BOOL=${NGEN_WITH_SQLITE} \
         -DUDUNITS_ACTIVE:BOOL=${NGEN_UDUNITS_ACTIVE} \
         -DUDUNITS_QUIET:BOOL=${NGEN_UDUNITS_QUIET} \
         -DCMAKE_INSTALL_PREFIX=${WORKDIR} \
@@ -176,3 +179,4 @@ RUN cd ${WORKDIR}/ngen \
     && mpirun -n 3 cmake_build_parallel/test/test_remote_nexus \
     && mpirun -n 4 cmake_build_parallel/test/test_remote_nexus \
     && find cmake_build* -type f -name "*" ! \( -name "*.so" -o -name "ngen" -o -name "partitionGenerator" \) -exec rm {} + 
+

docker/Dockerfile.ngen-deps

@@ -22,7 +22,7 @@ ARG REPO_URL=https://github.com/NOAA-OWP/ngen.git \
 #    netcdf-cxx4-devel netcdf-cxx4-mpich-devel netcdf-fortran-devel netcdf-fortran-mpich-devel \
 #    hdf5 hdf5-devel hdf5-mpich hdf5-mpich-devel \
 #    bzip2 expat expat-devel flex bison udunits2 udunits2-devel"
-    ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran \
+    ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran sqlite sqlite-devel \
     python3 python3-devel python3-pip gdal gdal-devel\
     bzip2 expat expat-devel flex bison udunits2 udunits2-devel zlib-devel" \
 # TODO: removed texinfo from list because it couldn't be found; make sure this doesn't lead to issues

docker/HelloNGEN.sh

@@ -49,6 +49,21 @@ generate_partition() {
   /dmod/bin/partitionGenerator "$1" "$2" "partitions_$3.json" "$3" '' ''
 }
 
+if [ "$2" == "auto" ]
+  then
+    echo "AUTO MODE ENGAGED"
+    echo "Running NextGen model framework in parallel mode"
+    procs=$(nproc)
+    procs=2 # Temporary fixed value, remove for full utilization
+    generate_partition "$selected_catchment" "$selected_nexus" "$procs"
+    mpirun -n $procs /dmod/bin/ngen-parallel $selected_catchment all $selected_nexus all $selected_realization $(pwd)/partitions_$procs.json
+    echo "Run completed successfully, exiting, have a nice day!"
+    exit 0
+  else
+    echo "Entering Interactive Mode"
+    continue
+fi
+
 echo -e "${YELLOW}Select an option (type a number): ${RESET}"
 options=("Run NextGen model framework in serial mode" "Run NextGen model framework in parallel mode" "Run Bash shell" "Exit")
 select option in "${options[@]}"; do
@@ -131,4 +146,4 @@ select option in "${options[@]}"; do
       ;;
   esac
 done
-exit
+exit