Merge branch 'develop' into feature/jcb-marineobs

.github/workflows/run_jcb_basic_testing.yaml

@@ -52,6 +52,39 @@ jobs:
           echo "JCB_BRANCH=develop" >> $GITHUB_ENV
         fi
 
+    - name: Check for the branch name in the jcb-algorithms repo (if not develop)
+      run: |
+        BRANCH_NAME=${{ env.JCB_APP_BRANCH }}
+
+        # If branch name is develop then we don't need to check the jcb-algorithms repo.
+        if [ "$BRANCH_NAME" == "develop" ]; then
+          echo "JCB_ALGO_BRANCH=develop" >> $GITHUB_ENV
+          exit 0
+        fi
+
+        # If the branch is not develop then check the jcb-algorithms repo.
+        if git ls-remote --heads https://github.com/NOAA-EMC/jcb-algorithms.git $BRANCH_NAME | grep -q "refs/heads/$BRANCH_NAME"; then
+          echo "Branch $BRANCH_NAME exists in jcb-algorithms repo."
+          echo "JCB_ALGO_BRANCH=$BRANCH_NAME" >> $GITHUB_ENV
+
+          # If the branch exists in jcb-algorithms repo but JCB_BRANCH is develop then we need to
+          # throw an error. This is not a safe situation. The developer should create a branch in
+          # the main jcb repo with the same name as the branch in the jcb-application repo.
+          # If there were branches in other apps to account for the changes in the algorithm repo
+          # they could not be tested here since the scipt is not clever enough to check for
+          # the existence of the branch being tested here in all the clients. The safest thing to do
+          # is simply create a branch with the same name (even if empty) in the main jcb repo. This
+          # will ensure the branches of the other applications are checked out by the init script.
+
+          if [ "${{ env.JCB_BRANCH }}" == "develop" ]; then
+            echo "Branch $BRANCH_NAME exists in jcb-algorithms repo but not in the main jcb repo. "
+            echo "Please create a branch with the same name (even if empty with no PR) in the main "
+            echo "jcb repo. This ensures safely checking all the clients that depend on the "
+            echo "jcb and jcb-algorithms repos with the changes being proposed."
+            exit 1
+          fi
+        fi
+
     - name: Clone jcb repository
       run: |
         mkdir -p empty_hooks

algorithm/aero/aero_convert_background.yaml.j2

@@ -0,0 +1,48 @@
+input geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_ges }}
+  npy: {{ aero_npy_ges }}
+  npz: {{ aero_npz_ges }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+output geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+states:
+- input:
+    datetime: '{{ aero_background_error_time_iso }}'
+    filetype: fms restart
+    state variables: [mass_fraction_of_sulfate_in_air,
+                        mass_fraction_of_hydrophobic_black_carbon_in_air,
+                        mass_fraction_of_hydrophilic_black_carbon_in_air,
+                        mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                        mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                        mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                        mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                        mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                        mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                        mass_fraction_of_sea_salt004_in_air]
+    datapath: {{ aero_background_path }}
+    filename_core: '{{ aero_background_error_time_fv3 }}.fv_core.res.nc'
+    filename_trcr: '{{ aero_background_error_time_fv3 }}.fv_tracer.res.nc'
+    filename_cplr: '{{ aero_background_error_time_fv3 }}.coupler.res'
+  output:
+    filetype: fms restart
+    datapath: {{ aero_background_path }}
+    filename_core: 'anlres.fv_core.res.nc'
+    filename_trcr: 'anlres.fv_tracer.res.nc'
+    filename_cplr: 'anlres.coupler.res'

algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2

@@ -0,0 +1,81 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+date: '{{ aero_background_error_time_iso }}'
+background:
+  datetime: '{{ aero_background_error_time_iso }}'
+  filetype: fms restart
+  state variables: [mass_fraction_of_sulfate_in_air,
+                    mass_fraction_of_hydrophobic_black_carbon_in_air,
+                    mass_fraction_of_hydrophilic_black_carbon_in_air,
+                    mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                    mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                    mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                    mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                    mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                    mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                    mass_fraction_of_sea_salt004_in_air]
+  datapath: {{ aero_background_path }}
+  filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc'
+  filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc'
+  filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res'
+background error:
+  filetype: fms restart
+  datapath: {{ aero_standard_deviation_path }}
+  filename_trcr: stddev.fv_tracer.res.nc
+  filename_cplr: stddev.coupler.res
+
+climate background error:
+  geometry:
+    fms initialization:
+        namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+        field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+    akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+    layout:
+    - {{ aero_layout_x }}
+    - {{ aero_layout_y }}
+    npx: {{ aero_npx_clim_b }}
+    npy: {{ aero_npy_clim_b }}
+    npz: {{ aero_npz_clim_b }}
+    field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+  climate background error stddev:
+    filetype: fms restart
+    skip coupler file: true
+    datapath: {{ aero_climatological_b_path }}
+    filename_trcr: stddev.fv_tracer.res.nc
+    filename_cplr: stddev.coupler.res
+  diagb weight: {{ aero_diagb_weight }}
+  staticb rescaling factor: {{ aero_diagb_static_rescale_factor }}
+
+variables:
+  name:
+  - mass_fraction_of_sulfate_in_air
+  - mass_fraction_of_hydrophobic_black_carbon_in_air
+  - mass_fraction_of_hydrophilic_black_carbon_in_air
+  - mass_fraction_of_hydrophobic_organic_carbon_in_air
+  - mass_fraction_of_hydrophilic_organic_carbon_in_air
+  - mass_fraction_of_dust001_in_air
+  - mass_fraction_of_dust002_in_air
+  - mass_fraction_of_dust003_in_air
+  - mass_fraction_of_dust004_in_air
+  - mass_fraction_of_dust005_in_air
+  - mass_fraction_of_sea_salt001_in_air
+  - mass_fraction_of_sea_salt002_in_air
+  - mass_fraction_of_sea_salt003_in_air
+  - mass_fraction_of_sea_salt004_in_air
+
+rescale: {{ aero_diagb_rescale_factor }}    # rescales the filtered std. dev. by "rescale"
+number of halo points: {{ aero_diagb_n_halo }}
+number of neighbors: {{ aero_diagb_n_neighbors }}
+simple smoothing:
+  horizontal iterations: {{ aero_diagb_smooth_horiz_iter }}
+  vertical iterations: {{ aero_diagb_smooth_vert_iter }}

algorithm/aero/aero_gen_bmatrix_diffusion.yaml.j2

@@ -0,0 +1,48 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+date: '{{ aero_background_error_time_iso }}'
+background:
+  datetime: '{{ aero_background_error_time_iso }}'
+  filetype: fms restart
+  state variables: [mass_fraction_of_sulfate_in_air,
+                    mass_fraction_of_hydrophobic_black_carbon_in_air,
+                    mass_fraction_of_hydrophilic_black_carbon_in_air,
+                    mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                    mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                    mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                    mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                    mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                    mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                    mass_fraction_of_sea_salt004_in_air]
+  datapath: {{ aero_background_path }}
+  filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc'
+  filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc'
+  filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res'
+background error:
+  covariance model: SABER
+  saber central block:
+    saber block name: diffusion
+    calibration:
+      normalization:
+        iterations: {{ aero_diffusion_iter }}
+      groups:
+      - horizontal:
+          fixed value: {{ aero_diffusion_horiz_len }}
+        write:
+          filepath: "{{ aero_berror_diffusion_directory }}/diffusion_hz"
+      - vertical:
+          levels: {{ aero_npz_ges }}
+          fixed value: {{ aero_diffusion_fixed_val }}
+          as gaussian: true
+        write:
+          filepath: "{{ aero_berror_diffusion_directory }}/diffusion_vt"

algorithm/atmosphere/atm_bias_staging.yaml.j2

@@ -0,0 +1,10 @@
+mkdir:
+    - '{{atmosphere_obsbiasout_path}}'
+copy:
+    {% for observation_from_jcb in observations %}
+    {% if use_observer(observation_from_jcb) %}
+    {% if observation_from_jcb in bias_files %}
+    - ['{{atmosphere_obsbiasroot_path}}/{{atmosphere_obsbiasin_prefix}}{{bias_files[observation_from_jcb]}}', '{{atmosphere_obsbiasin_path}}']
+    {% endif %}
+    {% endif %}
+    {% endfor %}

algorithm/atmosphere/atm_obs_staging.yaml.j2

@@ -0,0 +1,8 @@
+mkdir:
+    - '{{atmosphere_obsdatain_path}}'
+copy:
+    {% for observation_from_jcb in observations %}
+    {% if use_observer(observation_from_jcb) %}
+    - ['{{atmosphere_obsdataroot_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}', '{{atmosphere_obsdatain_path}}']
+    {% endif %}
+    {% endfor %}

algorithm/atmosphere/fv3jedi_fv3inc_lgetkf.yaml.j2

@@ -1,7 +1,7 @@
 variable change:
   variable change name: Model2GeoVaLs
   input variables: &bkgvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr,surface_geopotential_height]
-  output variables: &fv3incrvars [ua,va,t,sphum,ice_wat,liq_wat,o3mr,delp,hydrostatic_delz]
+  output variables: &fv3incrvars [ua,va,t,sphum,ice_wat,liq_wat,o3mr,delp,delz]
 jedi increment variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr]
 fv3 increment variables: *fv3incrvars
 background geometry:

algorithm/atmosphere/fv3jedi_fv3inc_variational.yaml.j2

@@ -1,7 +1,7 @@
 variable change:
   variable change name: Model2GeoVaLs
   input variables: &bkgvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr,surface_geopotential_height]
-  output variables: &fv3incrvars [ua,va,t,sphum,ice_wat,liq_wat,o3mr,delp,hydrostatic_delz]
+  output variables: &fv3incrvars [ua,va,t,sphum,ice_wat,liq_wat,o3mr,delp,delz]
 jedi increment variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr]
 fv3 increment variables: *fv3incrvars
 background geometry:

algorithm/marine/soca_diagb.yaml.j2

@@ -0,0 +1,37 @@
+geometry:
+  mom6_input_nml: mom_input.nml
+  fields metadata: ./fields_metadata.yaml
+
+date: '{{ marine_window_end_iso }}'
+
+background:
+  date: '{{ marine_window_end_iso }}'
+  basename: ./bkg/
+  ocn_filename: 'ocean.bkg.f009.nc'
+  ice_filename: 'ice.bkg.f009.nc'
+  read_from_file: 1
+
+background error:
+  datadir: ./staticb/
+  date: '{{ marine_window_middle_iso }}'
+  exp: bkgerr_stddev
+  type: incr
+
+variables:
+  name: [tocn, socn, uocn, vocn, hocn, ssh, cicen, hicen, hsnon, mom6_mld]
+
+rescale: 2.0    # rescales the filtered std. dev. by "rescale"
+min sst: 0.0    # Added to sst bkg. err.
+max ssh: 0.0    # Limits the amplitude of the unbalanced bkg err
+min depth: 500.0 # zero out the bkg. error. at less than min depth
+number of halo points: 4
+number of neighbors: 16
+
+simple smoothing:
+  horizontal iterations: 10
+  vertical iterations: 1
+
+# TODO(G): Too slow for the below scale
+#diffusion:
+#  horizontal: 500.0e3
+#  vertical: 3.0

algorithm/marine/soca_ensb.yaml.j2

@@ -0,0 +1,98 @@
+# Configuration for the recentering and re-balancing of the ensemble members
+geometry:
+  mom6_input_nml: mom_input.nml
+  fields metadata: ./fields_metadata.yaml
+
+date: '{{ marine_window_begin_iso }}'
+
+layers variable: [hocn]
+
+increment variables: [tocn, socn, uocn, vocn, ssh, hocn, cicen, hicen, hsnon]
+
+set increment variables to zero: [ssh]
+
+vertical geometry:
+  date: '{{ marine_window_begin_iso }}'
+  basename: ./INPUT/
+  ocn_filename: MOM.res.nc
+  read_from_file: 3
+
+add recentering increment: false
+
+soca increments:  # Could also be states, but they are read as increments
+  number of increments: '{{ marine_number_ensemble_members }}'
+  pattern: '%mem%'
+  template:
+    date: '{{ marine_window_begin_iso }}'
+    basename: '{{ enspert_relpath }}/ens/'
+    ocn_filename: 'ocean.%mem%.nc'
+    ice_filename: 'ice.%mem%.nc'
+    read_from_file: 3
+
+steric height:
+  linear variable changes:
+  - linear variable change name: BalanceSOCA  # Only the steric balance is applied
+
+ensemble mean output:
+  datadir: ./staticb/
+  date: '{{ marine_window_begin_iso }}'
+  exp: ens_mean
+  type: incr
+
+ssh output:
+  unbalanced:
+    datadir: ./staticb/
+    date: '{{ marine_window_begin_iso }}'
+    exp: ssh_unbal_stddev
+    type: incr
+
+  steric:
+    datadir: ./staticb/
+    date: '{{ marine_window_begin_iso }}'
+    exp: ssh_steric_stddev
+    type: incr
+
+  total:
+    datadir: ./staticb/
+    date: '{{ marine_window_begin_iso }}'
+    exp: ssh_total_stddev
+    type: incr
+
+  explained variance:
+    datadir: ./staticb/
+    date: '{{ marine_window_begin_iso }}'
+    exp: steric_explained_variance
+    type: incr
+
+  recentering error:
+    datadir: ./staticb/
+    date: '{{ marine_window_begin_iso }}'
+    exp: ssh_recentering_error
+    type: incr
+
+background error output:
+  datadir: ./staticb/
+  date: '{{ marine_window_begin_iso }}'
+  exp: bkgerr_stddev
+  type: incr
+
+linear variable change:
+  linear variable changes:
+  - linear variable change name: BalanceSOCA
+
+trajectory:
+  state variables: [tocn, socn, uocn, vocn, ssh, hocn, layer_depth, mld, cicen, hicen, hsnon]
+  date: '{{ marine_window_begin_iso }}'
+  basename: ./INPUT/
+  ocn_filename: MOM.res.nc
+  ice_filename: cice.res.nc
+  read_from_file: 1
+
+output increment:
+  # TODO: Revert this when fms can take more than 128 charactres file names
+  datadir: '{{ enspert_relpath }}/enspert/'
+  date: '{{ marine_window_begin_iso }}'
+  exp: trash
+  type: incr
+  output file: 'ocn.pert.steric.%mem%.nc'
+  pattern: '%mem%'