regression test system bug fixes, eliminate MOM6 warnings (#2197), add xr_cnvcld flag to FV3 (#2185)

tests/atparse.bash

@@ -1,42 +1,106 @@
 #! /usr/bin/env bash
 function atparse {
-    local __set_x
-    [ -o xtrace ] && __set_x='set -x' || __set_x='set +x'
-    set +x
+    # Usage:
+    #     source atparse.bash # defines the "atparse" function; only do this once
+    #     atparse [ var1=value1 [ var2=value2 [...] ] ] < input_file > output_file
+    # This function filters text from stdin to stdout. It scans for text sequences like:
+    #    @[varname]
+    # And replaces them with the value of the corresponding ${varname} variable.
+    # You can provide variables that are not set in bash by providing them on the command line.
+    # If set -u is enabled, it will exit the process when a variable is empty or undefined via set -u.
+
     # Use __ in names to avoid clashing with variables in {var} blocks.
-    local __text __before __after __during
+    local __text # current line of text being parsed, or the current command-line argument being parsed
+    local __before # all text before the next @[...] option
+    local __after # all text after the next @[...] option
+    local __during # the contents of the @[...] option, including the @[ and ]
+    local __set_x=":" # will be "set -x" if the calling script had that option enabled
+    local __set_u=":" # will be "set -u" if the calling script had that option enabled
+    local __set_e=":" # will be "set -e" if the calling script had that option enabled
+    local __abort_on_undefined=NO # YES = script should abort if a variable is undefined, NO otherwise
+
+    # Ensure "set -x -e -u" are all inactive, but remember if they
+    # were active so we can reset them later.
+    if [[ -o xtrace ]] ; then
+        __set_x="set -x"
+    fi
+    if [[ -o errexit ]] ; then
+        __set_e="set -e"
+    fi
+    if [[ -o nounset ]] ; then
+        __set_u="set -u"
+        __abort_on_undefined=YES
+    fi
+    set +eux
+
+    # Allow setting variables on the atparse command line rather than the environment.
+    # They will be local variables in this function.
     for __text in "$@" ; do
         if [[ $__text =~ ^([a-zA-Z][a-zA-Z0-9_]*)=(.*)$ ]] ; then
             eval "local ${BASH_REMATCH[1]}"
             eval "${BASH_REMATCH[1]}="'"${BASH_REMATCH[2]}"'
         else
-            echo "ERROR: Ignoring invalid argument $__text\n" 1>&2
+            echo "ERROR: Ignoring invalid argument $__text" 1>&2
         fi
     done
-    while IFS= read -r __text ; do
+
+    # Loop over all lines of text.
+    while [[ 1 == 1 ]] ; do
+        # Read the next line of text. This will "fail" if no more text
+        # is left OR if the last line lacks an end-of-line character.
+        read -d '' -r __text
+
+        # Stop when "read" reports it is done ($? -ne 0) AND the text is
+        # non-empty (! -n "$__text"). This ensures we read the final line
+        # even if it lacks an end-of-line character.
+        if [[ $? -ne 0 ]] ; then
+            if [[ -n "$__text" ]] ; then
+                # Text remained, but it had no end-of-line.
+                :
+            else
+                break
+            fi
+        fi
+        # Search for strings like @[varname] or @['string'] or @[@]
         while [[ "$__text" =~ ^([^@]*)(@\[[a-zA-Z_][a-zA-Z_0-9]*\]|@\[\'[^\']*\'\]|@\[@\]|@)(.*) ]] ; do
             __before="${BASH_REMATCH[1]}"
             __during="${BASH_REMATCH[2]}"
             __after="${BASH_REMATCH[3]}"
-#            printf 'PARSE[%s|%s|%s]\n' "$__before" "$__during" "$__after"
             printf %s "$__before"
+            # @['string'] inserts string
             if [[ "$__during" =~ ^@\[\'(.*)\'\]$ ]] ; then
                 printf %s "${BASH_REMATCH[1]}"
+            # @[@] inserts @
             elif [[ "$__during" == '@[@]' ]] ; then
                 printf @
+            # @[varname] inserts $varname
             elif [[ "$__during" =~ ^@\[([a-zA-Z_][a-zA-Z_0-9]*)\] ]] ; then
+                # Flag unknown variables at this step only.
+                if [[ ${__abort_on_undefined} == YES ]] ; then
+                    set -u
+                fi
                 eval 'printf %s "$'"${BASH_REMATCH[1]}"'"'
+                if [[ ${__abort_on_undefined} == YES ]] ; then
+                    set +u
+                fi
+            # Unrecognized sequences are inserted verbatim.
             else
                 printf '%s' "$__during"
             fi
+            # Continue until we run out of text in this line.
             if [[ "$__after" == "$__text" ]] ; then
                 break
             fi
             __text="$__after"
         done
+        # Print the corrected text
         printf '%s\n' "$__text"
     done
+
+    # Restore the calling script's shell options.
     eval "$__set_x"
+    eval "$__set_u"
+    eval "$__set_e"
 }
 
 function test_atparse {
@@ -45,14 +109,18 @@ function test_atparse {
     testvar='[testvar]'
     var1='[var1]'
     var2='[var2]'
-    cat<<\EOF | atparse var3='**'
+    var4='[var4]'
+    ( cat<<\EOF ; echo -n "line with no end-of-line character [var4] = @[var4]" ) | atparse var3='**'
 Nothing special here. = @['Nothing special here.']
 [testvar] = @[testvar]
 [var1] [var2] = @[var1] @[var2]
 ** = @[var3]
+[var4] == @[var4]
 @ = @[@] = @['@']
+@[undefined_variable_that_should_exit_script_if_set_minus_u_is_used]
 -n
  eval "export PE$c=\${PE$c:-0}" = @[' eval "export PE$c=\${PE$c:-0}"']
 EOF
+    echo " ... this text should be on the same line as the line with no end-of-line character"
     echo "After block, \$var3 = \"$var3\" should be empty"
 }

tests/logs/OpnReqTests_control_p8_hera.log

@@ -1,9 +1,9 @@
-Fri Mar 15 14:30:32 UTC 2024
+Tue Mar 26 16:22:58 UTC 2024
 Start Operation Requirement Test
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_bit_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_63946/bit_base_bit_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3765727/bit_base_bit_base
 Checking test bit_base results ....
 Moving baseline bit_base files ....
  Moving sfcf000.nc .........OK
@@ -51,14 +51,14 @@ Moving baseline bit_base files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 280.403230
-  0: The maximum resident set size (KB)                   = 1301268
+  0: The total amount of wall time                        = 273.805616
+  0: The maximum resident set size (KB)                   = 1258976
 
 Test bit_base PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_dbg_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_63946/dbg_base_dbg_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3765727/dbg_base_dbg_base
 Checking test dbg_base results ....
 Moving baseline dbg_base files ....
  Moving sfcf000.nc .........OK
@@ -106,14 +106,14 @@ Moving baseline dbg_base files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 914.973019
-  0: The maximum resident set size (KB)                   = 1289712
+  0: The total amount of wall time                        = 952.201032
+  0: The maximum resident set size (KB)                   = 1237956
 
 Test dbg_base PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_63946/dcp_dcp
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3765727/dcp_dcp
 Checking test dcp results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -160,14 +160,14 @@ Checking test dcp results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 254.193285
-  0: The maximum resident set size (KB)                   = 1277804
+  0: The total amount of wall time                        = 242.017304
+  0: The maximum resident set size (KB)                   = 1230120
 
 Test dcp PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_63946/mpi_mpi
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3765727/mpi_mpi
 Checking test mpi results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -214,14 +214,14 @@ Checking test mpi results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 250.421070
-  0: The maximum resident set size (KB)                   = 1278668
+  0: The total amount of wall time                        = 236.990247
+  0: The maximum resident set size (KB)                   = 1230212
 
 Test mpi PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_63946/rst_rst
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3765727/rst_rst
 Checking test rst results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -268,14 +268,14 @@ Checking test rst results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 267.943348
-  0: The maximum resident set size (KB)                   = 1275920
+  0: The total amount of wall time                        = 236.183608
+  0: The maximum resident set size (KB)                   = 1242204
 
 Test rst PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_63946/std_base_std_base
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3765727/std_base_std_base
 Checking test std_base results ....
 Moving baseline std_base files ....
  Moving sfcf000.nc .........OK
@@ -323,14 +323,14 @@ Moving baseline std_base files ....
  Moving RESTART/20210323.060000.sfc_data.tile5.nc .........OK
  Moving RESTART/20210323.060000.sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 249.896889
-  0: The maximum resident set size (KB)                   = 1276116
+  0: The total amount of wall time                        = 239.119350
+  0: The maximum resident set size (KB)                   = 1231424
 
 Test std_base PASS
 
 
 baseline dir = /scratch1/NCEPDEV/stmp4/Zachary.Shrader/FV3_OPNREQ_TEST/OPNREQ_TEST/control_p8_std_base_gnu
-working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_63946/thr_thr
+working dir  = /scratch1/NCEPDEV/stmp2/Zachary.Shrader/FV3_OPNREQ_TEST/opnReqTest_3765727/thr_thr
 Checking test thr results ....
  Comparing sfcf000.nc .....USING NCCMP......OK
  Comparing sfcf021.nc .....USING NCCMP......OK
@@ -377,11 +377,11 @@ Checking test thr results ....
  Comparing RESTART/20210323.060000.sfc_data.tile5.nc .....USING NCCMP......OK
  Comparing RESTART/20210323.060000.sfc_data.tile6.nc .....USING NCCMP......OK
 
-  0: The total amount of wall time                        = 256.272146
-  0: The maximum resident set size (KB)                   = 1278636
+  0: The total amount of wall time                        = 239.894578
+  0: The maximum resident set size (KB)                   = 1240384
 
 Test thr PASS
 
 OPERATION REQUIREMENT TEST WAS SUCCESSFUL
-Fri Mar 15 15:51:09 UTC 2024
-Elapsed time: 01h:20m:37s. Have a nice day!
+Tue Mar 26 17:56:35 UTC 2024
+Elapsed time: 01h:33m:38s. Have a nice day!