Enable warm starting of the model for forecast-only mode (NOAA-EMC#2031)

This PR: - adds an option to warm|cold start the ufs-weather-model when setting up an experiment for GFS/GEFS - adds the capability to stage FV3 restarts when the atmosphere is warm started from FV3 restarts. This would be used in the GEFS free-forecast for part of the reforecast capability. - If a coupled model is warm started for the atmosphere, also check if mediator restarts are present. If so, stage mediator restarts to allow restarting the coupled model. Fixes NOAA-EMC#2016
DavidHuber-NOAA · Nov 14, 2023 · 0a0d698 · 0a0d698
1 parent 2cd6a58
commit 0a0d698
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 32 deletions.
diff --git a/ci/cases/pr/C48_S2SA_gefs.yaml b/ci/cases/pr/C48_S2SA_gefs.yaml
@@ -9,9 +9,9 @@ arguments:
   resens: 48
   nens: 2
   gfs_cyc: 1
+  start: cold
   comrot: {{ 'RUNTESTS' | getenv }}/COMROT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48C48mx500
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/platforms/gefs_ci_defaults.yaml
diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base.emc.dyn
@@ -282,11 +282,16 @@ export NMEM_ENS=@NMEM_ENS@
 export ENSMEM="000"
 export MEMDIR="mem${ENSMEM}"
 
+export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
 # Check if cycle is cold starting
 if [[ "${EXP_WARM_START}" = ".false." ]]; then
-    export IAU_FHROT=0
+  export IAU_FHROT=0
 else
+  if [[ "${DOIAU}" = "YES" ]]; then
     export IAU_FHROT=3
+  else
+    export IAU_FHROT=0
+  fi
 fi
 
 # turned on nsst in anal and/or fcst steps, and turn off rtgsst

diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh
@@ -9,9 +9,9 @@ gPDY="${GDATE:0:8}"
 gcyc="${GDATE:8:2}"
 
 MEMDIR_ARRAY=()
-if [[ "${RUN}" == "gefs" ]]; then
+if [[ "${RUN:-}" = "gefs" ]]; then
   # Populate the member_dirs array based on the value of NMEM_ENS
-  for ((ii = 0; ii <= "${NMEM_ENS}"; ii++)); do
+  for ((ii = 0; ii <= "${NMEM_ENS:-0}"; ii++)); do
     MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")")
   done
 else
@@ -27,35 +27,58 @@ error_message() {
 
 ###############################################################
 for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
-  # Stage the FV3 initial conditions to ROTDIR (cold start)
-  YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_INPUT
-  [[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}"
-  src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc"
-  tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc"
-  ${NCP} "${src}" "${tgt}"
-  rc=$?
-  ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
-  err=$((err + rc))
-  for ftype in gfs_data sfc_data; do
-    for ((tt = 1; tt <= 6; tt++)); do
-      src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc"
-      tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
-      ${NCP} "${src}" "${tgt}"
-      rc=$?
-      tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
+
+  # Stage atmosphere initial conditions to ROTDIR
+  if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then
+    # Stage the FV3 restarts to ROTDIR (warm start)
+    RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
+    [[ ! -d "${COM_ATMOS_RESTART_PREV}" ]] && mkdir -p "${COM_ATMOS_RESTART_PREV}"
+    for ftype in coupler.res fv_core.res.nc; do
+      src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}"
+      tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}"
       ${NCP} "${src}" "${tgt}"
       rc=$?
       ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
       err=$((err + rc))
     done
-  done
+    for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do
+      for ((tt = 1; tt <= 6; tt++)); do
+        src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc"
+        tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc"
+        ${NCP} "${src}" "${tgt}"
+        rc=$?
+        ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+        err=$((err + rc))
+      done
+    done
+  else
+    # Stage the FV3 cold-start initial conditions to ROTDIR
+    YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_INPUT
+    [[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}"
+    src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc"
+    tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc"
+    ${NCP} "${src}" "${tgt}"
+    rc=$?
+    ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+    err=$((err + rc))
+    for ftype in gfs_data sfc_data; do
+      for ((tt = 1; tt <= 6; tt++)); do
+        src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc"
+        tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
+        ${NCP} "${src}" "${tgt}"
+        rc=$?
+        ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+        err=$((err + rc))
+      done
+    done
+  fi
 
   # Stage ocean initial conditions to ROTDIR (warm start)
   if [[ "${DO_OCN:-}" = "YES" ]]; then
-    RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_OCEAN_RESTART
-    [[ ! -d "${COM_OCEAN_RESTART}" ]] && mkdir -p "${COM_OCEAN_RESTART}"
-    src="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc"
-    tgt="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res.nc"
+    RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL
+    [[ ! -d "${COM_OCEAN_RESTART_PREV}" ]] && mkdir -p "${COM_OCEAN_RESTART_PREV}"
+    src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc"
+    tgt="${COM_OCEAN_RESTART_PREV}/${PDY}.${cyc}0000.MOM.res.nc"
     ${NCP} "${src}" "${tgt}"
     rc=$?
     ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
@@ -66,7 +89,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
         ;;
       "025" )
         for nn in $(seq 1 3); do
-          src="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
+          src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
           tgt="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
           ${NCP} "${src}" "${tgt}"
           rc=$?
@@ -80,13 +103,33 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
         err=$((err + rc))
         ;;
     esac
+
+    # TODO: Do mediator restarts exists in a ATMW configuration?
+    # TODO: No mediator is presumably involved in an ATMA configuration
+    if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then
+      # Stage the mediator restarts to ROTDIR (warm start/restart the coupled model)
+      RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL
+      [[ ! -d "${COM_MED_RESTART_PREV}" ]] && mkdir -p "${COM_MED_RESTART_PREV}"
+      src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc"
+      tgt="${COM_MED_RESTART_PREV}/${PDY}.${cyc}0000.ufs.cpld.cpl.r.nc"
+      if [[ -f "${src}" ]]; then
+        ${NCP} "${src}" "${tgt}"
+        rc=$?
+        ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+        err=$((err + rc))
+      else
+        echo "WARNING: No mediator restarts available with warm_start=${EXP_WARM_START}"
+      fi
+    fi
+
   fi
+
   # Stage ice initial conditions to ROTDIR (warm start)
   if [[ "${DO_ICE:-}" = "YES" ]]; then
-    RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_ICE_RESTART
-    [[ ! -d "${COM_ICE_RESTART}" ]] && mkdir -p "${COM_ICE_RESTART}"
-    src="${BASE_CPLIC}/${CPL_ICEIC}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc"
-    tgt="${COM_ICE_RESTART}/${PDY}.${cyc}0000.cice_model.res.nc"
+    RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL
+    [[ ! -d "${COM_ICE_RESTART_PREV}" ]] && mkdir -p "${COM_ICE_RESTART_PREV}"
+    src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc"
+    tgt="${COM_ICE_RESTART_PREV}/${PDY}.${cyc}0000.cice_model.res.nc"
     ${NCP} "${src}" "${tgt}"
     rc=$?
     ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
@@ -98,7 +141,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
     YMD=${PDY} HH=${cyc} generate_com COM_WAVE_RESTART
     [[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}"
     for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust
-      src="${BASE_CPLIC}/${CPL_WAVIC}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}"
+      src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}"
       tgt="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}"
       ${NCP} "${src}" "${tgt}"
       rc=$?
@@ -108,6 +151,7 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
   fi
 
 done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
+
 ###############################################################
 # Check for errors and exit if any of the above failed
 if [[ "${err}" -ne 0 ]]; then

diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py
@@ -432,7 +432,8 @@ def _gfs_or_gefs_forecast_args(parser):
         return parser
 
     def _gefs_args(parser):
-        parser.add_argument('--start', help=SUPPRESS, type=str, required=False, default='cold')
+        parser.add_argument('--start', help='restart mode: warm or cold', type=str,
+                            choices=['warm', 'cold'], required=False, default='cold')
         parser.add_argument('--configdir', help=SUPPRESS, type=str, required=False,
                             default=os.path.join(_top, 'parm/config/gefs'))
         parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False,