Merge branch 'feature/spack-stack' of github.com:DavidHuber-NOAA/glob…

…al-workflow into feature/spack-stack

Externals.cfg

@@ -8,7 +8,7 @@ protocol = git
 required = True
 
 [UFS]
-tag = 68050e5
+tag = 63a43d9
 local_path = sorc/ufs_model.fd
 repo_url = https://github.com/ufs-community/ufs-weather-model.git
 protocol = git

ci/platforms/config.hera

@@ -4,5 +4,5 @@ export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
 export STMP="/scratch1/NCEPDEV/stmp2/${USER}"
 export SLURM_ACCOUNT=nems
-export max_concurrent_cases=2
-export max_concurrent_pr=2
+export max_concurrent_cases=5
+export max_concurrent_pr=4

ci/platforms/config.orion

@@ -4,5 +4,5 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
 export STMP="/work/noaa/stmp/${USER}"
 export SLURM_ACCOUNT=nems
-export max_concurrent_cases=2
-export max_concurrent_pr=2
+export max_concurrent_cases=5
+export max_concurrent_pr=4

ci/scripts/check_ci.sh

@@ -70,6 +70,8 @@ fi
 
 for pr in ${pr_list}; do
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')
+  output_ci="${GFS_CI_ROOT}/PR/${pr}/output_runtime_${id}"
+  output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_runtime_single.log"
   echo "Processing Pull Request #${pr} and looking for cases"
   pr_dir="${GFS_CI_ROOT}/PR/${pr}"
 
@@ -83,8 +85,9 @@ for pr in ${pr_list}; do
   # shellcheck disable=SC2312
   if [[ -z $(ls -A "${pr_dir}/RUNTESTS/EXPDIR") ]] ; then
     "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed"
-    sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+    sed -i "1 i\`\`\`" "${output_ci}"
+    sed -i "1 i\All CI Test Cases Passed:" "${output_ci}"
+    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
     "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
     # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms
     weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}"  --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true
@@ -121,22 +124,20 @@ for pr in ${pr_list}; do
     rocoto_stat_output=$("${rocotostat}" -w "${xml}" -d "${db}" -s | grep -v CYCLE) || true
     num_cycles=$(echo "${rocoto_stat_output}" | wc -l) || true
     num_done=$(echo "${rocoto_stat_output}" | grep -c Done) || true
-    num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true
+    # num_succeeded=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c SUCCEEDED) || true
     echo "${pslot} Total Cycles: ${num_cycles} number done: ${num_done}" || true
     num_failed=$("${rocotostat}" -w "${xml}" -d "${db}" -a | grep -c -E 'FAIL|DEAD') || true
     if [[ ${num_failed} -ne 0 ]]; then
-      {
-        echo "Experiment ${pslot} Terminated: *FAILED*"
-        echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true
-      } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
       "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Failed"
+      error_logs=$("${rocotostat}" -d "${db}" -w "${xml}" | grep -E 'FAIL|DEAD' | awk '{print "-c", $1, "-t", $2}' | xargs "${rocotocheck}" -d "${db}" -w "${xml}" | grep join | awk '{print $2}') || true
       {
+       echo "Experiment ${pslot} Terminated: *** FAILED ***"
+       echo "Experiment ${pslot} Terminated with ${num_failed} tasks failed at $(date)" || true
        echo "Error logs:"
        echo "${error_logs}"
-      } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+      } >> "${output_ci}"
+      sed -i "1 i\`\`\`" "${output_ci}"
+      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
       "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       for kill_cases in "${pr_dir}/RUNTESTS/"*; do
          pslot=$(basename "${kill_cases}")
@@ -145,16 +146,16 @@ for pr in ${pr_list}; do
       break
     fi
     if [[ "${num_done}" -eq  "${num_cycles}" ]]; then
-      {
-        echo "Experiment ${pslot} completed: *SUCCESS*"
-        echo "Experiment ${pslot} Completed at $(date)" || true
-        echo "with ${num_succeeded} successfully completed jobs" || true
-      } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
       #Remove Experment cases that completed successfully
       rm -Rf "${pslot_dir}"
       rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}"
+      rm -f "${output_ci_single}"
+      # echo "\`\`\`" > "${output_ci_single}"
+      DATE=$(date)
+      echo "Experiment ${pslot} **SUCCESS** ${DATE}" >> "${output_ci_single}"
+      echo "Experiment ${pslot} **SUCCESS** at ${DATE}" >> "${output_ci}"
+      "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}"
+
     fi
   done
 done

ci/scripts/clone-build_ci.sh

@@ -40,21 +40,13 @@ while getopts "p:d:o:h" opt; do
 done
 
 cd "${repodir}" || exit 1
-# clone copy of repo
 if [[ -d global-workflow ]]; then
   rm -Rf global-workflow
 fi
 
 git clone "${REPO_URL}"
 cd global-workflow || exit 1
 
-pr_state=$("${GH}" pr view "${PR}" --json state --jq '.state')
-if [[ "${pr_state}" != "OPEN" ]]; then
-  title=$("${GH}" pr view "${PR}" --json title --jq '.title')
-  echo "PR ${title} is no longer open, state is ${pr_state} ... quitting"
-  exit 1
-fi
-
 # checkout pull request
 "${GH}" pr checkout "${PR}" --repo "${REPO_URL}"
 HOMEgfs="${PWD}"
@@ -78,19 +70,17 @@ echo "${commit}" > "../commit"
 # run checkout script
 cd sorc || exit 1
 set +e
-# TODO enable -u later when GDASApp tests are added
 ./checkout.sh -c -g -u >> log.checkout 2>&1
 checkout_status=$?
 if [[ ${checkout_status} != 0 ]]; then
   {
-    echo "Checkout:                      *FAILED*"
+    echo "Checkout: *** FAILED ***"
     echo "Checkout: Failed at $(date)" || true
     echo "Checkout: see output at ${PWD}/log.checkout"
   } >> "${outfile}"
   exit "${checkout_status}"
 else
   {
-    echo "Checkout:                      *SUCCESS*"
     echo "Checkout: Completed at $(date)" || true
   } >> "${outfile}"
 fi
@@ -104,19 +94,26 @@ build_status=$?
 
 if [[ ${build_status} != 0 ]]; then
   {
-    echo "Build:                         *FAILED*"
+    echo "Build: *** FAILED ***"
     echo "Build: Failed at $(date)" || true
     echo "Build: see output at ${PWD}/log.build"
   } >> "${outfile}"
   exit "${build_status}"
 else
   {
-    echo "Build:                         *SUCCESS*"
     echo "Build: Completed at $(date)" || true
   } >> "${outfile}"
 fi
 
 ./link_workflow.sh
+link_status=$?
+if [[ ${link_status} != 0 ]]; then
+  {
+    echo "Link: *** FAILED ***"
+    echo "Link: Failed at $(date)" || true
+  } >> "${outfile}"
+  exit "${link_status}"
+fi
 
 echo "check/build/link test completed"
 exit "${build_status}"

ci/scripts/driver.sh

@@ -119,7 +119,10 @@ for pr in ${pr_list}; do
   # call clone-build_ci to clone and build PR
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')
   set +e
-  "${ROOT_DIR}/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${pr_dir}/output_${id}"
+  output_ci="${pr_dir}/output_build_${id}"
+  rm -f "${output_ci}"
+  "${ROOT_DIR}/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${output_ci}"
+  #echo "SKIPPING: ${ROOT_DIR}/ci/scripts/clone-build_ci.sh"
   ci_status=$?
   ##################################################################
   # Checking for special case when Ready label was updated
@@ -138,7 +141,7 @@ for pr in ${pr_list}; do
     #setup space to put an experiment
     # export RUNTESTS for yaml case files to pickup
     export RUNTESTS="${pr_dir}/RUNTESTS"
-    #rm -Rf "${pr_dir:?}/RUNTESTS/"*
+    rm -Rf "${pr_dir:?}/RUNTESTS/"*
 
     #############################################################
     # loop over every yaml file in the PR's ci/cases
@@ -155,39 +158,46 @@ for pr in ${pr_list}; do
       rm -Rf "${STMP}/RUNDIRS/${pslot}"
       set +e
       export LOGFILE_PATH="${HOMEgfs}/ci/scripts/create_experiment.log"
-      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml"
+      rm -f "${LOGFILE_PATH}"
+      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml" 2>&1 "${LOGFILE_PATH}"
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then
+        last_line=$(tail -1 "${LOGFILE_PATH}")
+        if [[ "${last_line}" == *"Skipping creation"* ]]; then
+          action="Skipped"
+        else
+          action="Completed"
+        fi
         {
-          echo "Created experiment:            *SUCCESS*"
-          echo "Case setup: Completed at $(date) for experiment ${pslot}" || true
-        } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-        "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running"
-        "${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running
+          echo "Case setup: ${action} for experiment ${pslot}" || true
+        } >> "${output_ci}"
       else
         {
-          echo "Failed to create experiment:  *FAIL* ${pslot}"
-          echo "Experiment setup: failed at $(date) for experiment ${pslot}" || true
+          echo "*** Failed *** to create experiment: ${pslot}"
           echo ""
           cat "${LOGFILE_PATH}"
-        } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+        } >> "${output_ci}"
         "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
         "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+        "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
+        exit 1
       fi
     done
 
+    "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running"
+    "${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running
+    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
+
   else
     {
-      echo '```'
       echo "Failed on cloning and building global-workflowi PR: ${pr}"
       echo "CI on ${MACHINE_ID^} failed to build on $(date) for repo ${REPO_URL}" || true
-    } >> "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
+    } >> "${output_ci}"
     "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed"
     "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
+    "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
   fi
-  sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
-  "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}"
 
 done # looping over each open and labeled PR
 

jobs/JGDAS_ATMOS_VERFOZN

@@ -4,7 +4,7 @@
 # Set up environment for GDAS Ozone Monitor job
 #############################################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "verfozn" -c "base verfozn"
 
 export OZNMON_SUFFIX=${OZNMON_SUFFIX:-${NET}}
 
@@ -42,15 +42,12 @@ export p_cyc=${pdate:8:2}
 #---------------------------------------------
 # OZN_TANKDIR - WHERE OUTPUT DATA WILL RESIDE
 #
-export OZN_TANKDIR=${OZN_TANKDIR:-$(compath.py ${envir}/${NET}/${gfs_ver})}
-export TANKverf_ozn=${TANKverf_ozn:-${OZN_TANKDIR}/${RUN}.${PDY}/${cyc}/atmos/oznmon}
-export TANKverf_oznM1=${TANKverf_oznM1:-${OZN_TANKDIR}/${RUN}.${P_PDY}/${p_cyc}/atmos/oznmon}
-
 YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS
+YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_OZNMON
 
-if [[ ! -d ${TANKverf_ozn} ]]; then
-   mkdir -p -m 775 ${TANKverf_ozn}
-fi
+export TANKverf_ozn=${TANKverf_ozn:-${COM_ATMOS_OZNMON}}
+
+if [[ ! -d ${TANKverf_ozn} ]]; then mkdir -p -m 775 ${TANKverf_ozn} ; fi
 
 #---------------------------------------
 #  set up validation file
@@ -77,10 +74,6 @@ err=$?
 ################################
 # Remove the Working Directory
 ################################
-KEEPDATA=${KEEPDATA:-NO}
-cd ${DATAROOT}
-if [ ${KEEPDATA} = NO ] ; then
-  rm -rf ${DATA}
-fi
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
 
 exit 0

jobs/JGDAS_ATMOS_VERFRAD

@@ -4,18 +4,13 @@
 # Set up environment for GDAS Radiance Monitor job
 #############################################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "verfrad" -c "base verfrad"
 
 export COMPONENT="atmos"
 
-export RAD_DATA_IN=${DATA}
-
 export RADMON_SUFFIX=${RADMON_SUFFIX:-${RUN}}
 export CYCLE_INTERVAL=${CYCLE_INTERVAL:-6}
 
-mkdir -p ${RAD_DATA_IN}
-cd ${RAD_DATA_IN}
-
 ##############################################
 # Specify Execution Areas
 ##############################################
@@ -52,13 +47,15 @@ export p_cyc=${pdate:8:2}
 # COMOUT - WHERE GSI OUTPUT RESIDES
 # TANKverf - WHERE OUTPUT DATA WILL RESIDE
 #############################################
-export TANKverf=${TANKverf:-$(compath.py ${envir}/${NET}/${gfs_ver})}
-export TANKverf_rad=${TANKverf_rad:-${TANKverf}/${RUN}.${PDY}/${cyc}/atmos/radmon}
-export TANKverf_radM1=${TANKverf_radM1:-${TANKverf}/${RUN}.${P_PDY}/${p_cyc}/atmos/radmon}
-
 YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS
+YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_RADMON
+YMD=${P_PDY} HH=${p_cyc} generate_com -rx COM_ATMOS_RADMON_PREV:COM_ATMOS_RADMON_TMPL
 
-mkdir -p -m 775 ${TANKverf_rad}
+export TANKverf_rad=${TANKverf_rad:-${COM_ATMOS_RADMON}}
+export TANKverf_radM1=${TANKverf_radM1:-${COM_ATMOS_RADMON_PREV}}
+
+if [[ ! -d ${TANKverf_rad} ]]; then mkdir -p -m 775 ${TANKverf_rad} ; fi
+if [[ ! -d ${TANKverf_radM1} ]]; then mkdir -p -m 775 ${TANKverf_radM1} ; fi
 
 ########################################
 # Set necessary environment variables
@@ -89,9 +86,6 @@ fi
 ################################
 # Remove the Working Directory
 ################################
-KEEPDATA=${KEEPDATA:-YES}
-cd ${DATAROOT}
-if [ ${KEEPDATA} = NO ] ; then
-  rm -rf ${RAD_DATA_IN}
-fi
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
 
+exit 0

jobs/JGDAS_ATMOS_VMINMON

@@ -4,7 +4,7 @@
 # GDAS Minimization Monitor (MinMon) job
 ###########################################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "vminmon" -c "base vminmon"
 
 ###########################################################
 # obtain unique process id (pid) and make temp directories
@@ -40,12 +40,15 @@ export p_cyc=${pdate:8:2}
 #############################################
 # TANKverf - WHERE OUTPUT DATA WILL RESIDE
 #############################################
-export M_TANKverf=${M_TANKverf:-${COM_IN}/${RUN}.${PDY}/${cyc}/atmos/minmon}
-export M_TANKverfM1=${M_TANKverfM1:-${COM_IN}/${RUN}.${P_PDY}/${p_cyc}/atmos/minmon}
-
 YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS
+YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_MINMON
+YMD=${P_PDY} HH=${p_cyc} generate_com -rx COM_ATMOS_MINMON_PREV:COM_ATMOS_MINMON_TMPL
+
+export M_TANKverf=${M_TANKverf:-${COM_ATMOS_MINMON}}
+export M_TANKverfM1=${M_TANKverfM1:-${COM_ATMOS_MINMON_PREV}}
 
-mkdir -p -m 775 ${M_TANKverf}
+if [[ ! -d ${M_TANKverf} ]]; then mkdir -p -m 775 ${M_TANKverf} ; fi
+if [[ ! -d ${M_TANKverfM1} ]]; then mkdir -p -m 775 ${M_TANKverfM1} ; fi
 
 
 ########################################
@@ -65,10 +68,6 @@ err=$?
 ################################
 # Remove the Working Directory
 ################################
-KEEPDATA=${KEEPDATA:-NO}
-cd ${DATAROOT}
-if [ ${KEEPDATA} = NO ] ; then
-  rm -rf ${DATA}
-fi
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
 
 exit 0

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT

@@ -20,7 +20,7 @@ export COMOUT=${COMOUT:-${ROTDIR}/${CDUMP}.${PDY}/${cyc}/ocean}
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASPREPPY:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_bmat.sh}
+EXSCRIPT=${GDASOCNBMATSH:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_bmat.sh}
 ${EXSCRIPT}
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"