From b3f15c52d96507e71aeb1b3250496b4ce083e8bc Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 18 Dec 2023 14:10:34 -0600
Subject: [PATCH] Update Hercules environment to include cycled tasks. #1588

---
 env/HERCULES.env | 236 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 233 insertions(+), 3 deletions(-)

diff --git a/env/HERCULES.env b/env/HERCULES.env
index b4c6c3beb6..60f27faaa1 100755
--- a/env/HERCULES.env
+++ b/env/HERCULES.env
@@ -31,15 +31,157 @@ ulimit -s unlimited
 ulimit -a
 
 case ${step} in
+ "prep" | "prepbufr")
+
+    nth_max=$((npe_node_max / npe_node_prep))
+
+    export POE="NO"
+    export BACK=${BACK:-"YES"}
+    export sys_tp="HERCULES"
+    export launcher_PREP="srun"
+ ;;
+ "preplandobs")
+
+    export APRUN_CALCFIMS="${launcher} -n 1"
+ ;;
  "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")
 
     export CFP_MP="YES"
-    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    [[ "${step}" = "waveprep" ]] && export MP_PULSE=0
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
  ;;
- "fcst" )
+ "atmanlrun")
+
+    nth_max=$((npe_node_max / npe_node_atmanlrun))
+
+    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
+    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
+    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}"
+ ;;
+ "atmensanlrun")
+
+    nth_max=$((npe_node_max / npe_node_atmensanlrun))
+
+    export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}}
+    [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max}
+    export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}"
+ ;;
+ "aeroanlrun")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_aeroanlrun))
+
+    export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
+    [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
+    export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
+ ;;
+ "landanl")
+
+    nth_max=$((npe_node_max / npe_node_landanl))
+
+    export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
+    [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
+    export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+ ;;
+ "ocnanalbmat")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalbmat))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}"
+ ;;
+ "ocnanalrun")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalrun))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
+ ;;
+ "ocnanalchkpt")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalchkpt))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalchkpt:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt} --cpus-per-task=${NTHREADS_OCNANAL}"
+ ;;
+ "anal" | "analcalc")
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_anal))
+
+    export NTHREADS_GSI=${nth_anal:-${nth_max}}
+    [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${nth_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    npe_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${npe_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    npe_gausfcanl=${npe_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+ ;;
+ "sfcanl")
+    nth_max=$((npe_node_max / npe_node_sfcanl))
+
+    export NTHREADS_CYCLE=${nth_sfcanl:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    npe_sfcanl=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${npe_sfcanl} --cpus-per-task=${NTHREADS_CYCLE}"
+ ;;
+ "eobs")
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_eobs))
+
+    export NTHREADS_GSI=${nth_eobs:-${nth_max}}
+    [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_eobs}} --cpus-per-task=${NTHREADS_GSI}"
+ ;;
+ "eupd")
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_eupd))
+
+    export NTHREADS_ENKF=${nth_eupd:-${nth_max}}
+    [[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max}
+    export APRUN_ENKF="${launcher} -n ${npe_enkf:-${npe_eupd}} --cpus-per-task=${NTHREADS_ENKF}"
+ ;;
+ "fcst" | "efcs")
 
     export OMP_STACKSIZE=512M
     if [[ "${CDUMP}" =~ "gfs" ]]; then
@@ -56,7 +198,7 @@ case ${step} in
     unset nprocs ppn nnodes ntasks
  ;;
 
- "upp" )
+ "upp")
 
     nth_max=$((npe_node_max / npe_node_upp))
 
@@ -68,4 +210,92 @@ case ${step} in
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation
  ;;
+ "ecen")
+
+    nth_max=$((npe_node_max / npe_node_ecen))
+
+    export NTHREADS_ECEN=${nth_ecen:-${nth_max}}
+    [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max}
+    export APRUN_ECEN="${launcher} -n ${npe_ecen} --cpus-per-task=${NTHREADS_ECEN}"
+
+    export NTHREADS_CHGRES=${nth_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
+    export APRUN_CALCINC="${launcher} -n ${npe_ecen} --cpus-per-task=${NTHREADS_CALCINC}"
+
+ ;;
+ "esfc")
+
+    nth_max=$((npe_node_max / npe_node_esfc))
+
+    export NTHREADS_ESFC=${nth_esfc:-${nth_max}}
+    [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max}
+    export APRUN_ESFC="${launcher} -n ${npe_esfc} --cpus-per-task=${NTHREADS_ESFC}"
+
+    export NTHREADS_CYCLE=${nth_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    export APRUN_CYCLE="${launcher} -n ${npe_esfc} --cpus-per-task=${NTHREADS_CYCLE}"
+
+ ;;
+ "epos")
+
+    nth_max=$((npe_node_max / npe_node_epos))
+
+    export NTHREADS_EPOS=${nth_epos:-${nth_max}}
+    [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max}
+    export APRUN_EPOS="${launcher} -n ${npe_epos} --cpus-per-task=${NTHREADS_EPOS}"
+
+ ;;
+ "postsnd")
+
+    export CFP_MP="YES"
+
+    nth_max=$((npe_node_max / npe_node_postsnd))
+
+    export NTHREADS_POSTSND=${nth_postsnd:-1}
+    [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max}
+    export APRUN_POSTSND="${launcher} -n ${npe_postsnd} --cpus-per-task=${NTHREADS_POSTSND}"
+
+    export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
+    [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max}
+    export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}"
+
+ ;;
+ "awips")
+
+    nth_max=$((npe_node_max / npe_node_awips))
+
+    export NTHREADS_AWIPS=${nth_awips:-2}
+    [[ ${NTHREADS_AWIPS} -gt ${nth_max} ]] && export NTHREADS_AWIPS=${nth_max}
+    export APRUN_AWIPSCFP="${launcher} -n ${npe_awips} ${mpmd_opt}"
+
+ ;;
+ "gempak")
+
+    export CFP_MP="YES"
+
+    if [[ ${CDUMP} == "gfs" ]]; then
+        npe_gempak=${npe_gempak_gfs}
+        npe_node_gempak=${npe_node_gempak_gfs}
+    fi
+
+    nth_max=$((npe_node_max / npe_node_gempak))
+
+    export NTHREADS_GEMPAK=${nth_gempak:-1}
+    [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
+    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+ ;;
+ "fit2obs")
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs} --cpus-per-task=${NTHREADS_FIT2OBS}"
+
+ ;;
 esac