Merge branch 'develop' of https://github.com/ufs-community/ufs-srweat…

…her-app into develop

.cicd/scripts/srw_build.sh

@@ -36,6 +36,6 @@ cd -
 # Create combined log file for upload to s3
 build_dir="${workspace}/build_${SRW_COMPILER}"
 cat ${build_dir}/log.cmake ${build_dir}/log.make \
-    >${build_dir}/srw_build-${platform}-${SRW_COMPILER}.txt
+    >${build_dir}/srw_build-${SRW_PLATFORM}-${SRW_COMPILER}.txt
 
 exit $build_exit

.cicd/scripts/srw_metric.sh

@@ -58,7 +58,7 @@ fi
 # Test directories
 we2e_experiment_base_dir="${we2e_experiment_base_dir:=${workspace}/../expt_dirs/metric_test}"
 we2e_test_dir="${we2e_test_dir:=${workspace}/tests/WE2E}"
-we2e_test_name="${test_type:=grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0}"
+we2e_test_name="grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0"
 
 pwd
 

.cicd/scripts/srw_test.sh

@@ -45,7 +45,7 @@ fi
 
 cd ${we2e_test_dir}
 # Progress file
-progress_file="${workspace}/we2e_test_results-${platform}-${SRW_COMPILER}.txt"
+progress_file="${workspace}/we2e_test_results-${SRW_PLATFORM}-${SRW_COMPILER}.txt"
 /usr/bin/time -p -f '{\n  "cpu": "%P"\n, "memMax": "%M"\n, "mem": {"text": "%X", "data": "%D", "swaps": "%W", "context": "%c", "waits": "%w"}\n, "pagefaults": {"major": "%F", "minor": "%R"}\n, "filesystem": {"inputs": "%I", "outputs": "%O"}\n, "time": {"real": "%e", "user": "%U", "sys": "%S"}\n}' -o ${WORKSPACE}/${SRW_PLATFORM}-${SRW_COMPILER}-time-srw_test.json \
     ./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type} \
     --expt_basedir=${we2e_experiment_base_dir} | tee ${progress_file}; \

Externals.cfg

@@ -9,10 +9,10 @@ required = True
 
 [ufs-weather-model]
 protocol = git
-repo_url = https://github.com/noaa-oar-arl/ufs-weather-model
+repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
-branch = develop
-#hash = e051e0e
+#branch = develop
+hash = 1c6b4d4
 local_path = sorc/ufs-weather-model
 required = True
 
@@ -21,7 +21,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/UPP
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 945cb2c
+hash = be0410e
 local_path = sorc/UPP
 required = True
 
@@ -30,7 +30,7 @@ protocol = git
 repo_url = https://github.com/noaa-oar-arl/NEXUS
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 40346b6
+hash = e153072
 local_path = sorc/arl_nexus
 required = True
 
@@ -39,7 +39,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/AQM-utils
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = d953bd1
+hash = e236acd
 local_path = sorc/AQM-utils
 required = True
 

README.md

@@ -1,13 +1,13 @@
 # UFS Short-Range Weather Application
 
-The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. NOAA's operational model suite for numerical weather prediction (NWP) is quickly transitioning to the UFS from a number of legacy modeling systems. The UFS enables research, development, and contribution opportunities within the broader Weather Enterprise (including government, industry, and academia). For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/.
+The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. NOAA's operational model suite for numerical weather prediction (NWP) is quickly transitioning to the UFS from a number of legacy modeling systems. The UFS enables research, development, and contribution opportunities within the broader Weather Enterprise (including government, industry, and academia). For more information about the UFS, visit the UFS Portal at https://ufs.epic.noaa.gov/.
 
-The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.2.0) represents a snapshot of this continuously evolving system. 
+The UFS includes multiple applications (see a complete list at https://ufs.epic.noaa.gov/applications/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.2.0) represents a snapshot of this continuously evolving system. 
 
 The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.2.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2.2.0/. The repository is at: https://github.com/ufs-community/ufs-srweather-app.
 
 For instructions on how to clone the repository, build the code, and run the workflow, see:
-- https://ufs-srweather-app.readthedocs.io/en/develop/BuildingRunningTesting/Quickstart.html
+- https://ufs-srweather-app.readthedocs.io/en/develop/UsersGuide/BuildingRunningTesting/Quickstart.html
 
 For a debugging guide for users and developers in the field of Earth System Modeling, please see:
 https://epic.noaa.gov/wp-content/uploads/2022/12/Debugging-Guide.pdf

doc/README

@@ -20,10 +20,11 @@ Steps to build and use the Sphinx documentation tool:
 
 To build html:
 
-$ cd ufs-srweather-app/docs/UsersGuide
-$ make clean && sphinx-build -b html source build
+$ cd ufs-srweather-app/doc
+$ make clean && sphinx-build -b html . build
 
 The "make html" command can often be used in place of the previous command.
+"make doc" will both build the html and run the linkchecker. 
 
 Sphinx uses Latex to export the documentation as a PDF file.  To build pdf:
 

doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst

@@ -156,8 +156,8 @@ These settings define platform-specific run commands. Users should set run comma
 ``RUN_CMD_SERIAL``: (Default: "")
   The run command for some serial jobs.
 
-``RUN_CMD_AQM``: (Default: "")
-  The run command for some AQM tasks.
+``RUN_CMD_NEXUS``: (Default: "")
+  The run command for the AQM NEXUS tasks.
 
 ``RUN_CMD_AQMLBC``: (Default: "")
   The run command for the ``aqm_lbcs`` task.
@@ -271,6 +271,12 @@ These parameters are associated with the fixed (i.e., static) files. On :srw-wik
 ``FIXshp``: (Default: "")
    System directory containing the graphics shapefiles. On Level 1 systems, these are set within the machine files. Users on other systems will need to provide the path to the directory that contains the *Natural Earth* shapefiles.
 
+``FIXaqm``: (Default: "")
+   Path to system directory containing AQM fixed files.
+
+``FIXemis``: (Default: "")
+   Path to system directory containing AQM emission data files.
+
 ``FIXcrtm``: (Default: "")
    Path to system directory containing CRTM fixed files. 
 
@@ -532,7 +538,7 @@ CCPP Parameter
 ``CCPP_PHYS_SUITE_FP``: (Default: ``'{{ [workflow.EXPTDIR, CCPP_PHYS_SUITE_FN]|path_join }}'``)
    The full path to the suite definition file (SDF) in the experiment directory. 
 
-``CCPP_PHYS_DIR``: (Default: ``'{{ [user.UFS_WTHR_MDL_DIR, "FV3", "ccpp", "physics", "physics"] |path_join }}'``)
+``CCPP_PHYS_DIR``: (Default: ``'{{ [user.UFS_WTHR_MDL_DIR, "FV3", "ccpp", "physics", "physics", "SFC_Models", "Land", "Noahmp"] |path_join }}'``)
    The directory containing the CCPP physics source code. This is needed to link table(s) contained in that repository. 
 
 Field Dictionary Parameters
@@ -701,7 +707,7 @@ A standard set of environment variables has been established for *nco* mode to s
 ``envir_default, NET_default, model_ver_default, RUN_default``:
    Standard environment variables defined in the NCEP Central Operations WCOSS Implementation Standards document. These variables are used in forming the path to various directories containing input, output, and workflow files. The variables are defined in the :nco:`WCOSS Implementation Standards <ImplementationStandards.v11.0.0.pdf>` document (pp. 4-5) as follows: 
 
-   ``envir_default``: (Default: "para")
+   ``envir_default``: (Default: "test")
       Set to "test" during the initial testing phase, "para" when running in parallel (on a schedule), and "prod" in production. 
 
    ``NET_default``: (Default: "srw")
@@ -713,46 +719,28 @@ A standard set of environment variables has been established for *nco* mode to s
    ``RUN_default``: (Default: "srw")
       Name of model run (third level of ``com`` directory structure). In general, same as ``${NET_default}``.
 
-``OPSROOT_default``: (Default: ``'{{ workflow.EXPT_BASEDIR }}/../nco_dirs'``)
-  The operations root directory in *nco* mode.
-
-``COMROOT_default``: (Default: ``'{{ OPSROOT_default }}/com'``)
-   The ``com`` root directory for input/output data that is located on the current system (typically ``$OPSROOT_default/com``). 
-
-``DATAROOT_default``: (Default: ``'{{OPSROOT_default }}/tmp'``)
-   Directory containing the (temporary) working directory for running jobs; typically named ``$OPSROOT_default/tmp`` in production. 
-
-``DCOMROOT_default``: (Default: ``'{{OPSROOT_default }}/dcom'``)
-   ``dcom`` root directory, typically ``$OPSROOT_default/dcom``. This directory contains input/incoming data that is retrieved from outside WCOSS.
-
-``LOGBASEDIR_default``: (Default: ``'{% if user.RUN_ENVIR == "nco" %}{{ [OPSROOT_default, "output"]|path_join }}{% else %}{{ [workflow.EXPTDIR, "log"]|path_join }}{% endif %}'``)
-   Directory in which the log files from the workflow tasks will be placed.
-
-``COMIN_BASEDIR``: (Default: ``'{{ COMROOT_default }}/{{ NET_default }}/{{ model_ver_default }}'``)
-   ``com`` directory for current model's input data, typically ``$COMROOT/$NET/$model_ver/$RUN.$PDY``.
-
-``COMOUT_BASEDIR``: (Default: ``'{{ COMROOT_default }}/{{ NET_default }}/{{ model_ver_default }}'``)
-   ``com`` directory for current model's output data, typically ``$COMROOT/$NET/$model_ver/$RUN.$PDY``.
+``PTMP``: (Default ``'{{ workflow.EXPT_BASEDIR }}/../nco_dirs'``)
+   User-defined path to the com type directories (``OPSROOT=$PTMP/$envir``).
 
 ``DBNROOT_default``: (Default: "")
    Root directory for the data-alerting utilities.
 
-``SENDECF_default``: (Default: false)
+``SENDECF_default``: (Default: "NO")
    Boolean variable used to control ``ecflow_client`` child commands.
 
-``SENDDBN_default``: (Default: false)
+``SENDDBN_default``: (Default: "NO")
    Boolean variable used to control sending products off WCOSS2.
 
-``SENDDBN_NTC_default``: (Default: false)
+``SENDDBN_NTC_default``: (Default: "NO")
    Boolean variable used to control sending products with WMO headers off WCOSS2.
 
-``SENDCOM_default``: (Default: false)
+``SENDCOM_default``: (Default: "YES")
    Boolean variable to control data copies to ``$COMOUT``.
 
-``SENDWEB_default``: (Default: false)
+``SENDWEB_default``: (Default: "NO")
    Boolean variable used to control sending products to a web server, often ``ncorzdm``.
 
-``KEEPDATA_default``: (Default: true)
+``KEEPDATA_default``: (Default: "YES")
    Boolean variable used to specify whether or not the working directory should be kept upon successful job completion.
 
 ``MAILTO_default``: (Default: "")
@@ -1376,6 +1364,9 @@ Non-default parameters for the ``nexus_emission_*`` tasks are set in the ``task_
 ``PPN_NEXUS_EMISSION``: (Default: ``'{{ platform.NCORES_PER_NODE // OMP_NUM_THREADS_NEXUS_EMISSION }}'``)
    Processes per node for the ``nexus_emission_*`` tasks. 
 
+``NNODES_NEXUS_EMISSION``: (Default: 4)
+   The number of nodes to request from the job scheduler for the NEXUS emission task.
+
 ``KMP_AFFINITY_NEXUS_EMISSION``: (Default: "scatter")
    Intel Thread Affinity Interface for the ``nexus_emission_*`` tasks. See :ref:`this note <thread-affinity>` for more information on thread affinity.
 
@@ -1385,12 +1376,20 @@ Non-default parameters for the ``nexus_emission_*`` tasks are set in the ``task_
 ``OMP_STACKSIZE_NEXUS_EMISSION``: (Default: "1024m")
    Controls the size of the stack for threads created by the OpenMP implementation.
 
+POINT_SOURCE Configuration Parameters
+------------------------------------------------
+Non-default parameters for the ``task_point_source`` tasks are set in the ``task_point_source:`` section of the ``config.yaml`` file.
+
+``PT_SRC_SUBDIR``: (Default: ``"NEI2016v1/v2023-01-PT"``)
+   Subdirectory structure of point source data under ``FIXemis``.
+   Full path: ``FIXemis/PT_SRC_SUBDIR``
+
 BIAS_CORRECTION_O3 Configuration Parameters
 -------------------------------------------------
 
 Non-default parameters for the ``bias_correction_o3`` tasks are set in the ``task_bias_correction_o3:`` section of the ``config.yaml`` file. 
 
-``KMP_AFFINITY_BIAS_CORRECTION_O3``: "scatter"
+``KMP_AFFINITY_BIAS_CORRECTION_O3``: (Default: "scatter")
    Intel Thread Affinity Interface for the ``bias_correction_o3`` task. See :ref:`this note <thread-affinity>` for more information on thread affinity.
 
 ``OMP_NUM_THREADS_BIAS_CORRECTION_O3``: (Default: 32)
@@ -1744,38 +1743,14 @@ Non-default parameters for coupled Air Quality Modeling (AQM) tasks are set in t
 ``DO_AQM_SAVE_FIRE``: (Default: false)
    Archive fire emission file to HPSS.
 
-``DCOMINbio_default``: (Default: "")
-   Path to the directory containing AQM bio files.
-
-``DCOMINdust_default``: (Default: "/path/to/dust/dir")
-   Path to the directory containing AQM dust file.
-
-``DCOMINcanopy_default``: (Default: "/path/to/canopy/dir")
-   Path to the directory containing AQM canopy files.
-
-``DCOMINfire_default``: (Default: "")
-   Path to the directory containing AQM fire files.
-
-``DCOMINchem_lbcs_default``: (Default: "")
-   Path to the directory containing chemical LBC files.
-
-``DCOMINgefs_default``: (Default: "")
-   Path to the directory containing GEFS aerosol LBC files.
-
-``DCOMINpt_src_default``: (Default: "/path/to/point/source/base/directory")
-   Parent directory containing point source files.
-
-``DCOMINairnow_default``: (Default: "/path/to/airnow/obaservation/data")
+``COMINairnow_default``: (Default: "/path/to/airnow/observation/data")
    Path to the directory containing AIRNOW observation data.
 
-``COMINbicor``: (Default: "/path/to/historical/airnow/data/dir")
-   Path of reading in historical training data for bias correction. 
-
-``COMOUTbicor``: (Default: "/path/to/historical/airnow/data/dir")
-   Path to save the current cycle's model output and AirNow observations as training data for future use. ``$COMINbicor`` and ``$COMOUTbicor`` can be distinguished by the ``${yyyy}${mm}${dd}`` under the same location.
+``COMINfire_default``: (Default: "")
+   Path to the directory containing AQM fire files.
 
-``AQM_CONFIG_DIR``: (Default: "")
-   Configuration directory for AQM.
+``COMINgefs_default``:(Default: "")
+   Path to the directory containing GEFS aerosol LBC files. 
 
 ``AQM_BIO_FILE``: (Default: "BEIS_SARC401.ncf")
    File name of AQM BIO file.
@@ -1801,9 +1776,6 @@ Non-default parameters for coupled Air Quality Modeling (AQM) tasks are set in t
 ``AQM_FIRE_FILE_OFFSET_HRS``: (Default: 0)
    Time offset when retrieving fire emission data files. In a real-time run, the data files for :term:`ICs/LBCs` are not ready for use until the case starts. To resolve this issue, a real-time run uses the input data files in the previous cycle. For example, if the experiment run cycle starts at 12z, and ``AQM_FIRE_FILE_OFFSET_HRS: 6``, the fire emission data file from the previous cycle (06z) is used.
 
-``AQM_FIRE_ARCHV_DIR``: (Default: "/path/to/archive/dir/for/RAVE/on/HPSS")
-   Path to the archive directory for RAVE emission files on :term:`HPSS`.
-
 ``AQM_RC_FIRE_FREQUENCY``: (Default: "static")
    Fire frequency in ``aqm.rc``.
 
@@ -1822,12 +1794,6 @@ Non-default parameters for coupled Air Quality Modeling (AQM) tasks are set in t
 ``AQM_GEFS_FILE_CYC``: (Default: "")
    Cycle of the GEFS aerosol LBC files only if it is fixed.
 
-``NEXUS_INPUT_DIR``: (Default: "")
-   Same as ``GRID_DIR`` but for the the air quality emission generation task. Should be blank for the default value specified in ``setup.sh``.
-
-``NEXUS_FIX_DIR``: (Default: "")
-   Directory containing ``grid_spec`` files as the input file of NEXUS.
-
 ``NEXUS_GRID_FN``: (Default: "grid_spec_GSD_HRRR_25km.nc")
    File name of the input ``grid_spec`` file of NEXUS.
 

doc/requirements.txt

@@ -2,13 +2,13 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile --strip-extras requirements.in
 #
 alabaster==0.7.16
     # via sphinx
 babel==2.14.0
     # via sphinx
-certifi==2024.2.2
+certifi==2024.7.4
     # via requests
 charset-normalizer==3.3.2
     # via requests
@@ -22,7 +22,7 @@ idna==3.7
     # via requests
 imagesize==1.4.1
     # via sphinx
-jinja2==3.1.3
+jinja2==3.1.4
     # via sphinx
 latexcodec==2.0.1
     # via pybtex
@@ -40,7 +40,7 @@ pygments==2.17.2
     # via sphinx
 pyyaml==6.0.1
     # via pybtex
-requests==2.31.0
+requests==2.32.2
     # via sphinx
 six==1.16.0
     # via
@@ -72,5 +72,5 @@ sphinxcontrib-qthelp==1.0.7
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.10
     # via sphinx
-urllib3==2.2.0
+urllib3==2.2.2
     # via requests

jobs/JREGIONAL_CHECK_POST_OUTPUT

@@ -51,10 +51,11 @@ print_info_msg "
 Entering script:  \"${scrfunc_fn}\"
 In directory:     \"${scrfunc_dir}\"
 
-This is the J-job script for the task that checks that all the post-
-processed output files in fact exist and are at least a certain age.
-These files may have been generated by UPP as part of the current SRW
-App workflow, or they may be user-staged. 
+This is the J-job for the task that checks that no more than 
+NUM_MISSING_FCST_FILES_MAX of each forecast's (ensemble member's) post-
+processed output files are missing.  Note that such files may have been
+generated by UPP as part of the current SRW App workflow, or they may be
+user-staged.
 ========================================================================"
 #
 #-----------------------------------------------------------------------
@@ -70,15 +71,15 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #
 #-----------------------------------------------------------------------
 #
-# Create a flag file to make rocoto aware that the make_grid task has 
-# successfully completed (so that other tasks that depend on it can be
-# launched).  
+# Create a flag file to make rocoto aware that the check_post_output task
+# has successfully completed (so that other tasks that depend on it can
+# be launched).  
 #
 #-----------------------------------------------------------------------
 #
 ensmem_name="mem${ENSMEM_INDX}"
 cycle_dir="$EXPTDIR/$CDATE"
-mkdir_vrfy -p "${cycle_dir}"
+mkdir -p "${cycle_dir}"
 touch "${cycle_dir}/post_files_exist_${ensmem_name}.txt"
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_GET_EXTRN_MDL_FILES

@@ -222,8 +222,8 @@ if [ $RUN_ENVIR = "nco" ]; then
     export EXTRN_MDL_STAGING_DIR="${EXTRN_MDL_STAGING_DIR:-${DATA}}"
 else
     export EXTRN_MDL_STAGING_DIR="${COMIN}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}"
-    mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}"
-    cd_vrfy "${EXTRN_MDL_STAGING_DIR}"
+    mkdir -p "${EXTRN_MDL_STAGING_DIR}"
+    cd "${EXTRN_MDL_STAGING_DIR}"
 fi
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_MAKE_GRID

@@ -153,7 +153,7 @@ This is the J-job script for the task that generates grid files.
 #-----------------------------------------------------------------------
 #
 check_for_preexist_dir_file "${GRID_DIR}" "${PREEXISTING_DIR_METHOD}"
-mkdir_vrfy -p "${GRID_DIR}"
+mkdir -p "${GRID_DIR}"
 #
 #-----------------------------------------------------------------------
 #
@@ -162,7 +162,7 @@ mkdir_vrfy -p "${GRID_DIR}"
 #-----------------------------------------------------------------------
 #
 DATA="${DATA:-${GRID_DIR}/tmp}"
-mkdir_vrfy -p "$DATA"
+mkdir -p "$DATA"
 #
 #-----------------------------------------------------------------------
 #