Merge remote-tracking branch 'emc/develop' into feature/multi-app

DavidHuber-NOAA · Nov 20, 2024 · 252ca72 · 252ca72
2 parents 199274e + 1563594
commit 252ca72
Show file tree

Hide file tree

Showing 38 changed files with 1,060 additions and 610 deletions.
diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -145,14 +145,19 @@ case ${step} in
     export NTHREADS_OCNANALECEN=${threads_per_task_ocnanalecen:-${max_threads_per_task}}
     [[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task}
     export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
-;;
+ ;;
  "marineanlchkpt")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_OCNANAL=${NTHREADSmax}
     export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
  ;;
+ "marineanlletkf")
+
+    export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
+    export APRUN_MARINEANLLETKF="${APRUN_default}"
+ ;;
  "anal" | "analcalc")
 
     export MKL_NUM_THREADS=4

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -107,17 +107,15 @@ elif [[ "${step}" = "marinebmat" ]]; then
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
     export APRUN_MARINEBMAT="${APRUN_default}"
 
-elif [[ "${step}" = "ocnanalrun" ]]; then
+elif [[ "${step}" = "marineanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEANLVAR="${APRUN_default}"
 
-    export APRUN_OCNANAL="${APRUN_default}"
-
-elif [[ "${step}" = "ocnanalchkpt" ]]; then
+elif [[ "${step}" = "marineanlchkpt" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
-
-    export APRUN_OCNANAL="${APRUN_default}"
+    export APRUN_MARINEANLCHKPT="${APRUN_default}"
 
 elif [[ "${step}" = "ocnanalecen" ]]; then
 

diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl
@@ -5,8 +5,7 @@
 
 echo "BEGIN: config.atmanl"
 
-export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2"
-export JCB_ALGO_YAML=@JCB_ALGO_YAML@
+export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@
 
 export STATICB_TYPE=@STATICB_TYPE@
 export LOCALIZATION_TYPE="bump"
@@ -23,6 +22,8 @@ fi
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
+export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmanl_jedi_config.yaml.j2"
 export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2"
 export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2"
 export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2"
@@ -33,6 +34,4 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/gdas.x
-
 echo "END: config.atmanl"
diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc
@@ -8,7 +8,4 @@ echo "BEGIN: config.atmanlfv3inc"
 # Get task specific resources
 . "${EXPDIR}/config.resources" atmanlfv3inc
 
-export JCB_ALGO=fv3jedi_fv3inc_variational
-export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x
-
 echo "END: config.atmanlfv3inc"
diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl
@@ -5,17 +5,16 @@
 
 echo "BEGIN: config.atmensanl"
 
-export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2"
-if [[ ${lobsdiag_forenkf} = ".false." ]] ; then
-    export JCB_ALGO_YAML=@JCB_ALGO_YAML_LETKF@
-else
-    export JCB_ALGO_YAML=@JCB_ALGO_YAML_OBS@
-fi    
+export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@
+export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@
+export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@
 
 export INTERP_METHOD='barycentric'
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
+export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmensanl_jedi_config.yaml.j2"
 export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2"
 
 export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
@@ -24,6 +23,4 @@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
-export JEDIEXE=${EXECgfs}/gdas.x
-
 echo "END: config.atmensanl"
diff --git a/parm/config/gfs/config.atmensanlfv3inc b/parm/config/gfs/config.atmensanlfv3inc
@@ -8,7 +8,4 @@ echo "BEGIN: config.atmensanlfv3inc"
 # Get task specific resources
 . "${EXPDIR}/config.resources" atmensanlfv3inc
 
-export JCB_ALGO=fv3jedi_fv3inc_lgetkf
-export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x
-
 echo "END: config.atmensanlfv3inc"
diff --git a/parm/config/gfs/config.atmensanlobs b/parm/config/gfs/config.atmensanlobs
@@ -8,6 +8,4 @@ echo "BEGIN: config.atmensanlobs"
 # Get task specific resources
 . "${EXPDIR}/config.resources" atmensanlobs
 
-export JCB_ALGO_YAML=@JCB_ALGO_YAML@
-
 echo "END: config.atmensanlobs"
diff --git a/parm/config/gfs/config.atmensanlsol b/parm/config/gfs/config.atmensanlsol
@@ -8,6 +8,4 @@ echo "BEGIN: config.atmensanlsol"
 # Get task specific resources
 . "${EXPDIR}/config.resources" atmensanlsol
 
-export JCB_ALGO_YAML=@JCB_ALGO_YAML@
-
 echo "END: config.atmensanlsol"
diff --git a/parm/config/gfs/config.eupd b/parm/config/gfs/config.eupd
@@ -28,7 +28,7 @@ export lnsigcutoff=2.75          # ignored if modelspace_vloc=.true.
 
 export lobsdiag_forenkf=".true." # use jacobian.  must be .true. if modelspace_vloc=".true."
                                  # need to specify .true. setting since config.anal sets to .false.
-
+export taperanalperts=".true."
 export NAM_ENKF="smoothparm=35,"
 
 echo "END: config.eupd"
diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl
@@ -5,6 +5,8 @@
 
 echo "BEGIN: config.marineanl"
 
+export JEDI_CONFIG_YAML="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2"
+
 export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config"
 export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@
 export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@

diff --git a/parm/config/gfs/config.marineanlletkf b/parm/config/gfs/config.marineanlletkf
@@ -14,7 +14,7 @@ export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml
 export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j2"
 
 export GRIDGEN_EXEC="${EXECgfs}/gdas_soca_gridgen.x"
-export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml"
+export GRIDGEN_YAML="${HOMEgfs}/sorc/gdas.cd/parm/jcb-gdas/algorithm/marine/soca_gridgen.yaml.j2"
 export DIST_HALO_SIZE=500000
 
 echo "END: config.marineanlletkf"
diff --git a/parm/config/gfs/config.marinebmat b/parm/config/gfs/config.marinebmat
@@ -8,12 +8,4 @@ echo "BEGIN: config.marinebmat"
 # Get task specific resources
 . "${EXPDIR}/config.resources" marinebmat
 
-export BERROR_DIAGB_YAML="${PARMgfs}/gdas/soca/berror/soca_diagb.yaml.j2"
-export BERROR_VTSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_vtscales.yaml.j2"
-export BERROR_DIFFV_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_vt.yaml.j2"
-export BERROR_HZSCALES_YAML="${PARMgfs}/gdas/soca/berror/soca_setcorscales.yaml"
-export BERROR_DIFFH_YAML="${PARMgfs}/gdas/soca/berror/soca_parameters_diffusion_hz.yaml.j2"
-export BERROR_ENS_RECENTER_YAML="${PARMgfs}/gdas/soca/berror/soca_ensb.yaml.j2"
-export BERROR_HYB_WEIGHTS_YAML="${PARMgfs}/gdas/soca/berror/soca_ensweights.yaml.j2"
-
 echo "END: config.marinebmat"
diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION
@@ -25,7 +25,8 @@ case ${step} in
   ;;
   "atmanlvar")
     # Run on 8 nodes for memory requirement
-    export tasks_per_node=8
+    export tasks_per_node_gdas=8
+    export tasks_per_node_gfs=8      
     export walltime="00:45:00"
   ;;
   "atmensanlobs")

diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml
@@ -23,7 +23,7 @@ base:
   FHMAX_ENKF_GFS: 12
 
 atmanl:
-  JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"
+  JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2"
   STATICB_TYPE: "gsibec"
   LAYOUT_X_ATMANL: 8
   LAYOUT_Y_ATMANL: 8
@@ -33,16 +33,11 @@ atmanl:
 atmensanl:
   JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2"
   JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2"
+  JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2"
   LAYOUT_X_ATMENSANL: 8
   LAYOUT_Y_ATMENSANL: 8
   IO_LAYOUT_X: 1
   IO_LAYOUT_Y: 1
-
-atmensanlobs:
-  JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2"
-
-atmensanlsol:
-  JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2"
 
 aeroanl:
   IO_LAYOUT_X: 1

diff --git a/parm/gdas/atmanl_jedi_config.yaml.j2 b/parm/gdas/atmanl_jedi_config.yaml.j2
@@ -0,0 +1,13 @@
+atmanlvar:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas.x'
+  mpi_cmd: '{{ APRUN_ATMANLVAR }}'
+  jedi_args: ['fv3jedi', 'variational']
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML_VAR }}'
+atmanlfv3inc:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x'
+  mpi_cmd: '{{ APRUN_ATMANLFV3INC }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
+  jcb_algo: fv3jedi_fv3inc_variational
diff --git a/parm/gdas/atmensanl_jedi_config.yaml.j2 b/parm/gdas/atmensanl_jedi_config.yaml.j2
@@ -0,0 +1,27 @@
+atmensanlobs:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas.x'
+  mpi_cmd: '{{ APRUN_ATMENSANLOBS }}'
+  jedi_args: ['fv3jedi', 'localensembleda']
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML_OBS }}'
+atmensanlsol:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas.x'
+  mpi_cmd: '{{ APRUN_ATMENSANLSOL }}'
+  jedi_args: ['fv3jedi', 'localensembleda']
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML_SOL }}'
+atmensanlfv3inc:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/fv3jedi_fv3inc.x'
+  mpi_cmd: '{{ APRUN_ATMENSANLFV3INC }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
+  jcb_algo: fv3jedi_fv3inc_lgetkf
+atmensanlletkf:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas.x'
+  mpi_cmd: '{{ APRUN_ATMENSANLLETKF }}'
+  jedi_args: ['fv3jedi', 'localensembleda']  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/atm/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML_LETKF }}'
diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2
@@ -0,0 +1,42 @@
+gridgen:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas_soca_gridgen.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
+  jcb_algo: soca_gridgen
+soca_diagb:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_algo: soca_diagb
+soca_parameters_diffusion_vt:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_algo: soca_parameters_diffusion_vt
+soca_setcorscales:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_algo: soca_setcorscales
+soca_parameters_diffusion_hz:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_algo: soca_parameters_diffusion_hz
+soca_ensb:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas_ens_handler.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_algo: soca_ensb
+soca_ensweights:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x'
+  mpi_cmd: '{{ APRUN_MARINEBMAT }}'  
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'  
+  jcb_algo: soca_ensweights