Merge branch 'develop' into feature/calcanl

.github/workflows/run_jcb_basic_testing.yaml

@@ -52,6 +52,39 @@ jobs:
           echo "JCB_BRANCH=develop" >> $GITHUB_ENV
         fi
 
+    - name: Check for the branch name in the jcb-algorithms repo (if not develop)
+      run: |
+        BRANCH_NAME=${{ env.JCB_APP_BRANCH }}
+
+        # If branch name is develop then we don't need to check the jcb-algorithms repo.
+        if [ "$BRANCH_NAME" == "develop" ]; then
+          echo "JCB_ALGO_BRANCH=develop" >> $GITHUB_ENV
+          exit 0
+        fi
+
+        # If the branch is not develop then check the jcb-algorithms repo.
+        if git ls-remote --heads https://github.com/NOAA-EMC/jcb-algorithms.git $BRANCH_NAME | grep -q "refs/heads/$BRANCH_NAME"; then
+          echo "Branch $BRANCH_NAME exists in jcb-algorithms repo."
+          echo "JCB_ALGO_BRANCH=$BRANCH_NAME" >> $GITHUB_ENV
+
+          # If the branch exists in jcb-algorithms repo but JCB_BRANCH is develop then we need to
+          # throw an error. This is not a safe situation. The developer should create a branch in
+          # the main jcb repo with the same name as the branch in the jcb-application repo.
+          # If there were branches in other apps to account for the changes in the algorithm repo
+          # they could not be tested here since the scipt is not clever enough to check for
+          # the existence of the branch being tested here in all the clients. The safest thing to do
+          # is simply create a branch with the same name (even if empty) in the main jcb repo. This
+          # will ensure the branches of the other applications are checked out by the init script.
+
+          if [ "${{ env.JCB_BRANCH }}" == "develop" ]; then
+            echo "Branch $BRANCH_NAME exists in jcb-algorithms repo but not in the main jcb repo. "
+            echo "Please create a branch with the same name (even if empty with no PR) in the main "
+            echo "jcb repo. This ensures safely checking all the clients that depend on the "
+            echo "jcb and jcb-algorithms repos with the changes being proposed."
+            exit 1
+          fi
+        fi
+
     - name: Clone jcb repository
       run: |
         mkdir -p empty_hooks

algorithm/aero/aero_convert_background.yaml.j2

@@ -0,0 +1,48 @@
+input geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_ges }}
+  npy: {{ aero_npy_ges }}
+  npz: {{ aero_npz_ges }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+output geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+states:
+- input:
+    datetime: '{{ aero_background_error_time_iso }}'
+    filetype: fms restart
+    state variables: [mass_fraction_of_sulfate_in_air,
+                        mass_fraction_of_hydrophobic_black_carbon_in_air,
+                        mass_fraction_of_hydrophilic_black_carbon_in_air,
+                        mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                        mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                        mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                        mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                        mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                        mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                        mass_fraction_of_sea_salt004_in_air]
+    datapath: {{ aero_background_path }}
+    filename_core: '{{ aero_background_error_time_fv3 }}.fv_core.res.nc'
+    filename_trcr: '{{ aero_background_error_time_fv3 }}.fv_tracer.res.nc'
+    filename_cplr: '{{ aero_background_error_time_fv3 }}.coupler.res'
+  output:
+    filetype: fms restart
+    datapath: {{ aero_background_path }}
+    filename_core: 'anlres.fv_core.res.nc'
+    filename_trcr: 'anlres.fv_tracer.res.nc'
+    filename_cplr: 'anlres.coupler.res'

algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2

@@ -0,0 +1,81 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+date: '{{ aero_background_error_time_iso }}'
+background:
+  datetime: '{{ aero_background_error_time_iso }}'
+  filetype: fms restart
+  state variables: [mass_fraction_of_sulfate_in_air,
+                    mass_fraction_of_hydrophobic_black_carbon_in_air,
+                    mass_fraction_of_hydrophilic_black_carbon_in_air,
+                    mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                    mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                    mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                    mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                    mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                    mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                    mass_fraction_of_sea_salt004_in_air]
+  datapath: {{ aero_background_path }}
+  filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc'
+  filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc'
+  filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res'
+background error:
+  filetype: fms restart
+  datapath: {{ aero_standard_deviation_path }}
+  filename_trcr: stddev.fv_tracer.res.nc
+  filename_cplr: stddev.coupler.res
+
+climate background error:
+  geometry:
+    fms initialization:
+        namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+        field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+    akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+    layout:
+    - {{ aero_layout_x }}
+    - {{ aero_layout_y }}
+    npx: {{ aero_npx_clim_b }}
+    npy: {{ aero_npy_clim_b }}
+    npz: {{ aero_npz_clim_b }}
+    field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+  climate background error stddev:
+    filetype: fms restart
+    skip coupler file: true
+    datapath: {{ aero_climatological_b_path }}
+    filename_trcr: stddev.fv_tracer.res.nc
+    filename_cplr: stddev.coupler.res
+  diagb weight: {{ aero_diagb_weight }}
+  staticb rescaling factor: {{ aero_diagb_static_rescale_factor }}
+
+variables:
+  name:
+  - mass_fraction_of_sulfate_in_air
+  - mass_fraction_of_hydrophobic_black_carbon_in_air
+  - mass_fraction_of_hydrophilic_black_carbon_in_air
+  - mass_fraction_of_hydrophobic_organic_carbon_in_air
+  - mass_fraction_of_hydrophilic_organic_carbon_in_air
+  - mass_fraction_of_dust001_in_air
+  - mass_fraction_of_dust002_in_air
+  - mass_fraction_of_dust003_in_air
+  - mass_fraction_of_dust004_in_air
+  - mass_fraction_of_dust005_in_air
+  - mass_fraction_of_sea_salt001_in_air
+  - mass_fraction_of_sea_salt002_in_air
+  - mass_fraction_of_sea_salt003_in_air
+  - mass_fraction_of_sea_salt004_in_air
+
+rescale: {{ aero_diagb_rescale_factor }}    # rescales the filtered std. dev. by "rescale"
+number of halo points: {{ aero_diagb_n_halo }}
+number of neighbors: {{ aero_diagb_n_neighbors }}
+simple smoothing:
+  horizontal iterations: {{ aero_diagb_smooth_horiz_iter }}
+  vertical iterations: {{ aero_diagb_smooth_vert_iter }}

algorithm/aero/aero_gen_bmatrix_diffusion.yaml.j2

@@ -0,0 +1,48 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{ aero_layout_x }}
+  - {{ aero_layout_y }}
+  npx: {{ aero_npx_anl }}
+  npy: {{ aero_npy_anl }}
+  npz: {{ aero_npz_anl }}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+date: '{{ aero_background_error_time_iso }}'
+background:
+  datetime: '{{ aero_background_error_time_iso }}'
+  filetype: fms restart
+  state variables: [mass_fraction_of_sulfate_in_air,
+                    mass_fraction_of_hydrophobic_black_carbon_in_air,
+                    mass_fraction_of_hydrophilic_black_carbon_in_air,
+                    mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                    mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                    mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                    mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                    mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                    mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                    mass_fraction_of_sea_salt004_in_air]
+  datapath: {{ aero_background_path }}
+  filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc'
+  filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc'
+  filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res'
+background error:
+  covariance model: SABER
+  saber central block:
+    saber block name: diffusion
+    calibration:
+      normalization:
+        iterations: {{ aero_diffusion_iter }}
+      groups:
+      - horizontal:
+          fixed value: {{ aero_diffusion_horiz_len }}
+        write:
+          filepath: "{{ aero_berror_diffusion_directory }}/diffusion_hz"
+      - vertical:
+          levels: {{ aero_npz_ges }}
+          fixed value: {{ aero_diffusion_fixed_val }}
+          as gaussian: true
+        write:
+          filepath: "{{ aero_berror_diffusion_directory }}/diffusion_vt"

algorithm/atmosphere/atm_bias_staging.yaml.j2

@@ -0,0 +1,10 @@
+mkdir:
+    - '{{atmosphere_obsbiasout_path}}'
+copy:
+    {% for observation_from_jcb in observations %}
+    {% if use_observer(observation_from_jcb) %}
+    {% if observation_from_jcb in bias_files %}
+    - ['{{atmosphere_obsbiasroot_path}}/{{atmosphere_obsbiasin_prefix}}{{bias_files[observation_from_jcb]}}', '{{atmosphere_obsbiasin_path}}']
+    {% endif %}
+    {% endif %}
+    {% endfor %}

algorithm/atmosphere/atm_obs_staging.yaml.j2

@@ -0,0 +1,8 @@
+mkdir:
+    - '{{atmosphere_obsdatain_path}}'
+copy:
+    {% for observation_from_jcb in observations %}
+    {% if use_observer(observation_from_jcb) %}
+    - ['{{atmosphere_obsdataroot_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}', '{{atmosphere_obsdatain_path}}']
+    {% endif %}
+    {% endfor %}

model/aero/aero_3dfgat_outer_loop_1.yaml.j2

@@ -0,0 +1 @@
+aero_3dvar_outer_loop_1.yaml.j2

model/aero/aero_3dfgat_outer_loop_2.yaml.j2

@@ -0,0 +1 @@
+aero_3dvar_outer_loop_2.yaml.j2

model/aero/aero_3dvar_outer_loop_1.yaml.j2

@@ -0,0 +1,17 @@
+- ninner: 35
+  gradient norm reduction: 1e-10
+  test: on
+  geometry:
+    fms initialization:
+      namelist filename: {{aero_fv3jedi_files_path}}/fmsmpp.nml
+      field table filename: {{aero_fv3jedi_files_path}}/field_table
+    akbk: {{aero_fv3jedi_files_path}}/akbk.nc4
+    layout:
+    - {{aero_layout_x}}
+    - {{aero_layout_y}}
+    npx: {{aero_npx_anl}}
+    npy: {{aero_npy_anl}}
+    npz: {{aero_npz_anl}}
+    field metadata override: {{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml
+  diagnostics:
+    departures: bkgmob

model/aero/aero_3dvar_outer_loop_2.yaml.j2

@@ -0,0 +1,17 @@
+- ninner: 35
+  gradient norm reduction: 1e-10
+  test: on
+  geometry:
+    fms initialization:
+      namelist filename: {{aero_fv3jedi_files_path}}/fmsmpp.nml
+      field table filename: {{aero_fv3jedi_files_path}}/field_table
+    akbk: {{aero_fv3jedi_files_path}}/akbk.nc4
+    layout:
+    - {{aero_layout_x}}
+    - {{aero_layout_y}}
+    npx: {{aero_npx_anl}}
+    npy: {{aero_npy_anl}}
+    npz: {{aero_npz_anl}}
+    field metadata override: {{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml
+  diagnostics:
+    departures: bkgmob1

model/aero/aero_background.yaml.j2

@@ -0,0 +1,10 @@
+datapath: {{ aero_background_path }}
+filetype: fms restart
+datetime: '{{ aero_background_time_iso }}'
+filename is datetime templated: true
+filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc'
+filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc'
+filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res'
+state variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2,
+                    dust1,dust2,dust3,dust4,dust5,
+                    seas1,seas2,seas3,seas4]

model/aero/aero_background_error_static_diffusion.yaml.j2

@@ -0,0 +1,33 @@
+covariance model: SABER
+saber central block:
+  saber block name: diffusion 
+  read:
+    groups:
+    - variables: [mass_fraction_of_sulfate_in_air,
+                       mass_fraction_of_hydrophobic_black_carbon_in_air,
+                       mass_fraction_of_hydrophilic_black_carbon_in_air,
+                       mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                       mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                       mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                       mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                       mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                       mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                       mass_fraction_of_sea_salt004_in_air]
+      horizontal:
+        filepath: "{{aero_berror_data_directory}}/diffusion_hz"
+      vertical:
+        levels: {{aero_npz_anl}} 
+        filepath: "{{aero_berror_data_directory}}/diffusion_vt" 
+saber outer blocks:
+- saber block name: StdDev
+  read:
+    model file:
+      datetime: "{{ aero_cycle_time_iso }}"
+      set datetime on read: true
+      filetype: fms restart
+      psinfile: true
+      datapath: "{{aero_berror_data_directory}}"
+      filename_core: '{{ aero_cycle_time_fv3 }}.stddev.fv_core.res.nc'
+      filename_trcr: '{{ aero_cycle_time_fv3 }}.stddev.fv_tracer.res.nc'
+      filename_cplr: '{{ aero_cycle_time_fv3 }}.stddev.coupler.res'
+      date: "{{ aero_cycle_time_iso }}" 

model/aero/aero_final_increment_cubed_sphere.yaml.j2

@@ -0,0 +1,23 @@
+geometry:
+  fms initialization:
+    namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+    field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+  akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+  layout:
+  - {{aero_layout_x}}
+  - {{aero_layout_y}}
+  npx: {{aero_npx_anl}}
+  npy: {{aero_npy_anl}}
+  npz: {{aero_npz_anl}}
+  field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"
+output:
+  state component:
+    datapath: {{ aero_analysis_path }}
+    prefix: aeroinc
+    filetype: fms restart
+    filename_core: '{{ aero_cycle_time_fv3 }}.fv_core.res.nc'
+    filename_trcr: '{{ aero_cycle_time_fv3 }}.fv_tracer.res.nc'
+    filename_cplr: '{{ aero_cycle_time_fv3 }}.coupler.res'
+    state variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2,
+                      dust1,dust2,dust3,dust4,dust5,
+                      seas1,seas2,seas3,seas4]

model/aero/aero_final_increment_gaussian.yaml.j2

@@ -0,0 +1,18 @@
+grid type: regular gaussian
+local interpolator type: atlas interpolator
+interpolation method:
+  type: finite-element
+number of latitude gridpoints: {{ aero_npy_ges - 1 }}
+variables to output: [mass_fraction_of_sulfate_in_air,
+                       mass_fraction_of_hydrophobic_black_carbon_in_air,
+                       mass_fraction_of_hydrophilic_black_carbon_in_air,
+                       mass_fraction_of_hydrophobic_organic_carbon_in_air,
+                       mass_fraction_of_hydrophilic_organic_carbon_in_air,
+                       mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air,
+                       mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air,
+                       mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air,
+                       mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air,
+                       mass_fraction_of_sea_salt004_in_air]
+all model levels: true
+datapath: {{ aero_analysis_path }}
+prefix: aeroinc_gauss

model/aero/aero_geometry_background.yaml.j2

@@ -0,0 +1,11 @@
+fms initialization:
+  namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml"
+  field table filename: "{{aero_fv3jedi_files_path}}/field_table"
+akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4"
+layout:
+- {{aero_layout_x}}
+- {{aero_layout_y}}
+npx: {{aero_npx_ges}}
+npy: {{aero_npy_ges}}
+npz: {{aero_npz_ges}}
+field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml"

model/aero/aero_model_pseudo.yaml.j2

@@ -0,0 +1,13 @@
+name: PSEUDO
+datapath: {{ aero_background_path}}
+filetype: fms restart
+filename is datetime templated: true
+filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc'
+filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc'
+filename_sfcd: '%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc'
+filename_sfcw: '%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc'
+filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res'
+tstep: {{aero_forecast_timestep}}
+model variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2,
+                      dust1,dust2,dust3,dust4,dust5,
+                      seas1,seas2,seas3,seas4]