From 7717c0e7401e344a6bce37a4f8ecc11399256936 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:30:45 -0400 Subject: [PATCH 1/4] JCB-based obs+bias staging (#31) This PR is a companion to Global Workflow PR [#2992](https://github.com/NOAA-EMC/global-workflow/pull/2992) and GDASApp PR [#1312](https://github.com/NOAA-EMC/GDASApp/pull/1312). It add observation and bias files staging templates required for the above Global Workflow PR. --- algorithm/atmosphere/atm_bias_staging.yaml.j2 | 10 ++++++++++ algorithm/atmosphere/atm_obs_staging.yaml.j2 | 8 ++++++++ 2 files changed, 18 insertions(+) create mode 100644 algorithm/atmosphere/atm_bias_staging.yaml.j2 create mode 100644 algorithm/atmosphere/atm_obs_staging.yaml.j2 diff --git a/algorithm/atmosphere/atm_bias_staging.yaml.j2 b/algorithm/atmosphere/atm_bias_staging.yaml.j2 new file mode 100644 index 0000000..c91ea2e --- /dev/null +++ b/algorithm/atmosphere/atm_bias_staging.yaml.j2 @@ -0,0 +1,10 @@ +mkdir: + - '{{atmosphere_obsbiasout_path}}' +copy: + {% for observation_from_jcb in observations %} + {% if use_observer(observation_from_jcb) %} + {% if observation_from_jcb in bias_files %} + - ['{{atmosphere_obsbiasroot_path}}/{{atmosphere_obsbiasin_prefix}}{{bias_files[observation_from_jcb]}}', '{{atmosphere_obsbiasin_path}}'] + {% endif %} + {% endif %} + {% endfor %} diff --git a/algorithm/atmosphere/atm_obs_staging.yaml.j2 b/algorithm/atmosphere/atm_obs_staging.yaml.j2 new file mode 100644 index 0000000..34d0984 --- /dev/null +++ b/algorithm/atmosphere/atm_obs_staging.yaml.j2 @@ -0,0 +1,8 @@ +mkdir: + - '{{atmosphere_obsdatain_path}}' +copy: + {% for observation_from_jcb in observations %} + {% if use_observer(observation_from_jcb) %} + - ['{{atmosphere_obsdataroot_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}', '{{atmosphere_obsdatain_path}}'] + {% endif %} + {% endfor %} From 997d80a7fa500104bb5cf470d042d40bbdec5d0d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 22 Oct 2024 10:17:28 -0400 Subject: [PATCH 2/4] Add support for global aerosol DA in JCB (#33) Add in templates so that global aerosol DA can use JCB to generate YAML configuration for the variational application + B matrix supporting executables. --- .../aero/aero_convert_background.yaml.j2 | 48 +++++++++++ algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2 | 81 +++++++++++++++++++ .../aero/aero_gen_bmatrix_diffusion.yaml.j2 | 48 +++++++++++ model/aero/aero_3dfgat_outer_loop_1.yaml.j2 | 1 + model/aero/aero_3dfgat_outer_loop_2.yaml.j2 | 1 + model/aero/aero_3dvar_outer_loop_1.yaml.j2 | 17 ++++ model/aero/aero_3dvar_outer_loop_2.yaml.j2 | 17 ++++ model/aero/aero_background.yaml.j2 | 10 +++ ..._background_error_static_diffusion.yaml.j2 | 33 ++++++++ .../aero_final_increment_cubed_sphere.yaml.j2 | 23 ++++++ .../aero_final_increment_gaussian.yaml.j2 | 18 +++++ model/aero/aero_geometry.yaml.j2 | 0 model/aero/aero_geometry_background.yaml.j2 | 11 +++ model/aero/aero_model_pseudo.yaml.j2 | 13 +++ observations/aero/viirs_n20_aod.yaml.j2 | 63 +++++++++++++++ observations/aero/viirs_n21_aod.yaml.j2 | 63 +++++++++++++++ observations/aero/viirs_npp_aod.yaml.j2 | 63 +++++++++++++++ 17 files changed, 510 insertions(+) create mode 100644 algorithm/aero/aero_convert_background.yaml.j2 create mode 100644 algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2 create mode 100644 algorithm/aero/aero_gen_bmatrix_diffusion.yaml.j2 create mode 120000 model/aero/aero_3dfgat_outer_loop_1.yaml.j2 create mode 120000 model/aero/aero_3dfgat_outer_loop_2.yaml.j2 create mode 100644 model/aero/aero_3dvar_outer_loop_1.yaml.j2 create mode 100644 model/aero/aero_3dvar_outer_loop_2.yaml.j2 create mode 100644 model/aero/aero_background.yaml.j2 create mode 100644 model/aero/aero_background_error_static_diffusion.yaml.j2 create mode 100644 model/aero/aero_final_increment_cubed_sphere.yaml.j2 create mode 100644 model/aero/aero_final_increment_gaussian.yaml.j2 delete mode 100644 model/aero/aero_geometry.yaml.j2 create mode 100644 model/aero/aero_geometry_background.yaml.j2 create mode 100644 model/aero/aero_model_pseudo.yaml.j2 create mode 100644 observations/aero/viirs_n20_aod.yaml.j2 create mode 100644 observations/aero/viirs_n21_aod.yaml.j2 create mode 100644 observations/aero/viirs_npp_aod.yaml.j2 diff --git a/algorithm/aero/aero_convert_background.yaml.j2 b/algorithm/aero/aero_convert_background.yaml.j2 new file mode 100644 index 0000000..0a1344f --- /dev/null +++ b/algorithm/aero/aero_convert_background.yaml.j2 @@ -0,0 +1,48 @@ +input geometry: + fms initialization: + namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml" + field table filename: "{{aero_fv3jedi_files_path}}/field_table" + akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4" + layout: + - {{ aero_layout_x }} + - {{ aero_layout_y }} + npx: {{ aero_npx_ges }} + npy: {{ aero_npy_ges }} + npz: {{ aero_npz_ges }} + field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml" +output geometry: + fms initialization: + namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml" + field table filename: "{{aero_fv3jedi_files_path}}/field_table" + akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4" + layout: + - {{ aero_layout_x }} + - {{ aero_layout_y }} + npx: {{ aero_npx_anl }} + npy: {{ aero_npy_anl }} + npz: {{ aero_npz_anl }} + field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml" +states: +- input: + datetime: '{{ aero_background_error_time_iso }}' + filetype: fms restart + state variables: [mass_fraction_of_sulfate_in_air, + mass_fraction_of_hydrophobic_black_carbon_in_air, + mass_fraction_of_hydrophilic_black_carbon_in_air, + mass_fraction_of_hydrophobic_organic_carbon_in_air, + mass_fraction_of_hydrophilic_organic_carbon_in_air, + mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air, + mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air, + mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air, + mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air, + mass_fraction_of_sea_salt004_in_air] + datapath: {{ aero_background_path }} + filename_core: '{{ aero_background_error_time_fv3 }}.fv_core.res.nc' + filename_trcr: '{{ aero_background_error_time_fv3 }}.fv_tracer.res.nc' + filename_cplr: '{{ aero_background_error_time_fv3 }}.coupler.res' + output: + filetype: fms restart + datapath: {{ aero_background_path }} + filename_core: 'anlres.fv_core.res.nc' + filename_trcr: 'anlres.fv_tracer.res.nc' + filename_cplr: 'anlres.coupler.res' diff --git a/algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2 b/algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2 new file mode 100644 index 0000000..d1f3eda --- /dev/null +++ b/algorithm/aero/aero_gen_bmatrix_diagb.yaml.j2 @@ -0,0 +1,81 @@ +geometry: + fms initialization: + namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml" + field table filename: "{{aero_fv3jedi_files_path}}/field_table" + akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4" + layout: + - {{ aero_layout_x }} + - {{ aero_layout_y }} + npx: {{ aero_npx_anl }} + npy: {{ aero_npy_anl }} + npz: {{ aero_npz_anl }} + field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml" +date: '{{ aero_background_error_time_iso }}' +background: + datetime: '{{ aero_background_error_time_iso }}' + filetype: fms restart + state variables: [mass_fraction_of_sulfate_in_air, + mass_fraction_of_hydrophobic_black_carbon_in_air, + mass_fraction_of_hydrophilic_black_carbon_in_air, + mass_fraction_of_hydrophobic_organic_carbon_in_air, + mass_fraction_of_hydrophilic_organic_carbon_in_air, + mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air, + mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air, + mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air, + mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air, + mass_fraction_of_sea_salt004_in_air] + datapath: {{ aero_background_path }} + filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc' + filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc' + filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res' +background error: + filetype: fms restart + datapath: {{ aero_standard_deviation_path }} + filename_trcr: stddev.fv_tracer.res.nc + filename_cplr: stddev.coupler.res + +climate background error: + geometry: + fms initialization: + namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml" + field table filename: "{{aero_fv3jedi_files_path}}/field_table" + akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4" + layout: + - {{ aero_layout_x }} + - {{ aero_layout_y }} + npx: {{ aero_npx_clim_b }} + npy: {{ aero_npy_clim_b }} + npz: {{ aero_npz_clim_b }} + field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml" + climate background error stddev: + filetype: fms restart + skip coupler file: true + datapath: {{ aero_climatological_b_path }} + filename_trcr: stddev.fv_tracer.res.nc + filename_cplr: stddev.coupler.res + diagb weight: {{ aero_diagb_weight }} + staticb rescaling factor: {{ aero_diagb_static_rescale_factor }} + +variables: + name: + - mass_fraction_of_sulfate_in_air + - mass_fraction_of_hydrophobic_black_carbon_in_air + - mass_fraction_of_hydrophilic_black_carbon_in_air + - mass_fraction_of_hydrophobic_organic_carbon_in_air + - mass_fraction_of_hydrophilic_organic_carbon_in_air + - mass_fraction_of_dust001_in_air + - mass_fraction_of_dust002_in_air + - mass_fraction_of_dust003_in_air + - mass_fraction_of_dust004_in_air + - mass_fraction_of_dust005_in_air + - mass_fraction_of_sea_salt001_in_air + - mass_fraction_of_sea_salt002_in_air + - mass_fraction_of_sea_salt003_in_air + - mass_fraction_of_sea_salt004_in_air + +rescale: {{ aero_diagb_rescale_factor }} # rescales the filtered std. dev. by "rescale" +number of halo points: {{ aero_diagb_n_halo }} +number of neighbors: {{ aero_diagb_n_neighbors }} +simple smoothing: + horizontal iterations: {{ aero_diagb_smooth_horiz_iter }} + vertical iterations: {{ aero_diagb_smooth_vert_iter }} diff --git a/algorithm/aero/aero_gen_bmatrix_diffusion.yaml.j2 b/algorithm/aero/aero_gen_bmatrix_diffusion.yaml.j2 new file mode 100644 index 0000000..0462daa --- /dev/null +++ b/algorithm/aero/aero_gen_bmatrix_diffusion.yaml.j2 @@ -0,0 +1,48 @@ +geometry: + fms initialization: + namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml" + field table filename: "{{aero_fv3jedi_files_path}}/field_table" + akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4" + layout: + - {{ aero_layout_x }} + - {{ aero_layout_y }} + npx: {{ aero_npx_anl }} + npy: {{ aero_npy_anl }} + npz: {{ aero_npz_anl }} + field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml" +date: '{{ aero_background_error_time_iso }}' +background: + datetime: '{{ aero_background_error_time_iso }}' + filetype: fms restart + state variables: [mass_fraction_of_sulfate_in_air, + mass_fraction_of_hydrophobic_black_carbon_in_air, + mass_fraction_of_hydrophilic_black_carbon_in_air, + mass_fraction_of_hydrophobic_organic_carbon_in_air, + mass_fraction_of_hydrophilic_organic_carbon_in_air, + mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air, + mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air, + mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air, + mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air, + mass_fraction_of_sea_salt004_in_air] + datapath: {{ aero_background_path }} + filename_core: '{{ aero_background_error_time_fv3 }}.anlres.fv_core.res.nc' + filename_trcr: '{{ aero_background_error_time_fv3 }}.anlres.fv_tracer.res.nc' + filename_cplr: '{{ aero_background_error_time_fv3 }}.anlres.coupler.res' +background error: + covariance model: SABER + saber central block: + saber block name: diffusion + calibration: + normalization: + iterations: {{ aero_diffusion_iter }} + groups: + - horizontal: + fixed value: {{ aero_diffusion_horiz_len }} + write: + filepath: "{{ aero_berror_diffusion_directory }}/diffusion_hz" + - vertical: + levels: {{ aero_npz_ges }} + fixed value: {{ aero_diffusion_fixed_val }} + as gaussian: true + write: + filepath: "{{ aero_berror_diffusion_directory }}/diffusion_vt" diff --git a/model/aero/aero_3dfgat_outer_loop_1.yaml.j2 b/model/aero/aero_3dfgat_outer_loop_1.yaml.j2 new file mode 120000 index 0000000..be2ee77 --- /dev/null +++ b/model/aero/aero_3dfgat_outer_loop_1.yaml.j2 @@ -0,0 +1 @@ +aero_3dvar_outer_loop_1.yaml.j2 \ No newline at end of file diff --git a/model/aero/aero_3dfgat_outer_loop_2.yaml.j2 b/model/aero/aero_3dfgat_outer_loop_2.yaml.j2 new file mode 120000 index 0000000..d029b98 --- /dev/null +++ b/model/aero/aero_3dfgat_outer_loop_2.yaml.j2 @@ -0,0 +1 @@ +aero_3dvar_outer_loop_2.yaml.j2 \ No newline at end of file diff --git a/model/aero/aero_3dvar_outer_loop_1.yaml.j2 b/model/aero/aero_3dvar_outer_loop_1.yaml.j2 new file mode 100644 index 0000000..68529a1 --- /dev/null +++ b/model/aero/aero_3dvar_outer_loop_1.yaml.j2 @@ -0,0 +1,17 @@ +- ninner: 35 + gradient norm reduction: 1e-10 + test: on + geometry: + fms initialization: + namelist filename: {{aero_fv3jedi_files_path}}/fmsmpp.nml + field table filename: {{aero_fv3jedi_files_path}}/field_table + akbk: {{aero_fv3jedi_files_path}}/akbk.nc4 + layout: + - {{aero_layout_x}} + - {{aero_layout_y}} + npx: {{aero_npx_anl}} + npy: {{aero_npy_anl}} + npz: {{aero_npz_anl}} + field metadata override: {{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml + diagnostics: + departures: bkgmob diff --git a/model/aero/aero_3dvar_outer_loop_2.yaml.j2 b/model/aero/aero_3dvar_outer_loop_2.yaml.j2 new file mode 100644 index 0000000..7ca1a03 --- /dev/null +++ b/model/aero/aero_3dvar_outer_loop_2.yaml.j2 @@ -0,0 +1,17 @@ +- ninner: 35 + gradient norm reduction: 1e-10 + test: on + geometry: + fms initialization: + namelist filename: {{aero_fv3jedi_files_path}}/fmsmpp.nml + field table filename: {{aero_fv3jedi_files_path}}/field_table + akbk: {{aero_fv3jedi_files_path}}/akbk.nc4 + layout: + - {{aero_layout_x}} + - {{aero_layout_y}} + npx: {{aero_npx_anl}} + npy: {{aero_npy_anl}} + npz: {{aero_npz_anl}} + field metadata override: {{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml + diagnostics: + departures: bkgmob1 diff --git a/model/aero/aero_background.yaml.j2 b/model/aero/aero_background.yaml.j2 new file mode 100644 index 0000000..012c50d --- /dev/null +++ b/model/aero/aero_background.yaml.j2 @@ -0,0 +1,10 @@ +datapath: {{ aero_background_path }} +filetype: fms restart +datetime: '{{ aero_background_time_iso }}' +filename is datetime templated: true +filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc' +filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc' +filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res' +state variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2, + dust1,dust2,dust3,dust4,dust5, + seas1,seas2,seas3,seas4] diff --git a/model/aero/aero_background_error_static_diffusion.yaml.j2 b/model/aero/aero_background_error_static_diffusion.yaml.j2 new file mode 100644 index 0000000..73af01b --- /dev/null +++ b/model/aero/aero_background_error_static_diffusion.yaml.j2 @@ -0,0 +1,33 @@ +covariance model: SABER +saber central block: + saber block name: diffusion + read: + groups: + - variables: [mass_fraction_of_sulfate_in_air, + mass_fraction_of_hydrophobic_black_carbon_in_air, + mass_fraction_of_hydrophilic_black_carbon_in_air, + mass_fraction_of_hydrophobic_organic_carbon_in_air, + mass_fraction_of_hydrophilic_organic_carbon_in_air, + mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air, + mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air, + mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air, + mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air, + mass_fraction_of_sea_salt004_in_air] + horizontal: + filepath: "{{aero_berror_data_directory}}/diffusion_hz" + vertical: + levels: {{aero_npz_anl}} + filepath: "{{aero_berror_data_directory}}/diffusion_vt" +saber outer blocks: +- saber block name: StdDev + read: + model file: + datetime: "{{ aero_cycle_time_iso }}" + set datetime on read: true + filetype: fms restart + psinfile: true + datapath: "{{aero_berror_data_directory}}" + filename_core: '{{ aero_cycle_time_fv3 }}.stddev.fv_core.res.nc' + filename_trcr: '{{ aero_cycle_time_fv3 }}.stddev.fv_tracer.res.nc' + filename_cplr: '{{ aero_cycle_time_fv3 }}.stddev.coupler.res' + date: "{{ aero_cycle_time_iso }}" diff --git a/model/aero/aero_final_increment_cubed_sphere.yaml.j2 b/model/aero/aero_final_increment_cubed_sphere.yaml.j2 new file mode 100644 index 0000000..e34a7fc --- /dev/null +++ b/model/aero/aero_final_increment_cubed_sphere.yaml.j2 @@ -0,0 +1,23 @@ +geometry: + fms initialization: + namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml" + field table filename: "{{aero_fv3jedi_files_path}}/field_table" + akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4" + layout: + - {{aero_layout_x}} + - {{aero_layout_y}} + npx: {{aero_npx_anl}} + npy: {{aero_npy_anl}} + npz: {{aero_npz_anl}} + field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml" +output: + state component: + datapath: {{ aero_analysis_path }} + prefix: aeroinc + filetype: fms restart + filename_core: '{{ aero_cycle_time_fv3 }}.fv_core.res.nc' + filename_trcr: '{{ aero_cycle_time_fv3 }}.fv_tracer.res.nc' + filename_cplr: '{{ aero_cycle_time_fv3 }}.coupler.res' + state variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2, + dust1,dust2,dust3,dust4,dust5, + seas1,seas2,seas3,seas4] diff --git a/model/aero/aero_final_increment_gaussian.yaml.j2 b/model/aero/aero_final_increment_gaussian.yaml.j2 new file mode 100644 index 0000000..97ffc8d --- /dev/null +++ b/model/aero/aero_final_increment_gaussian.yaml.j2 @@ -0,0 +1,18 @@ +grid type: regular gaussian +local interpolator type: atlas interpolator +interpolation method: + type: finite-element +number of latitude gridpoints: {{ aero_npy_ges - 1 }} +variables to output: [mass_fraction_of_sulfate_in_air, + mass_fraction_of_hydrophobic_black_carbon_in_air, + mass_fraction_of_hydrophilic_black_carbon_in_air, + mass_fraction_of_hydrophobic_organic_carbon_in_air, + mass_fraction_of_hydrophilic_organic_carbon_in_air, + mass_fraction_of_dust001_in_air, mass_fraction_of_dust002_in_air, + mass_fraction_of_dust003_in_air, mass_fraction_of_dust004_in_air, + mass_fraction_of_dust005_in_air, mass_fraction_of_sea_salt001_in_air, + mass_fraction_of_sea_salt002_in_air, mass_fraction_of_sea_salt003_in_air, + mass_fraction_of_sea_salt004_in_air] +all model levels: true +datapath: {{ aero_analysis_path }} +prefix: aeroinc_gauss diff --git a/model/aero/aero_geometry.yaml.j2 b/model/aero/aero_geometry.yaml.j2 deleted file mode 100644 index e69de29..0000000 diff --git a/model/aero/aero_geometry_background.yaml.j2 b/model/aero/aero_geometry_background.yaml.j2 new file mode 100644 index 0000000..834e2fc --- /dev/null +++ b/model/aero/aero_geometry_background.yaml.j2 @@ -0,0 +1,11 @@ +fms initialization: + namelist filename: "{{aero_fv3jedi_files_path}}/fmsmpp.nml" + field table filename: "{{aero_fv3jedi_files_path}}/field_table" +akbk: "{{aero_fv3jedi_files_path}}/akbk.nc4" +layout: +- {{aero_layout_x}} +- {{aero_layout_y}} +npx: {{aero_npx_ges}} +npy: {{aero_npy_ges}} +npz: {{aero_npz_ges}} +field metadata override: "{{aero_fv3jedi_files_path}}/fv3jedi_fieldmetadata_restart.yaml" diff --git a/model/aero/aero_model_pseudo.yaml.j2 b/model/aero/aero_model_pseudo.yaml.j2 new file mode 100644 index 0000000..61af8db --- /dev/null +++ b/model/aero/aero_model_pseudo.yaml.j2 @@ -0,0 +1,13 @@ +name: PSEUDO +datapath: {{ aero_background_path}} +filetype: fms restart +filename is datetime templated: true +filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc' +filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc' +filename_sfcd: '%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc' +filename_sfcw: '%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc' +filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res' +tstep: {{aero_forecast_timestep}} +model variables: [t,delp,sphum,so4,bc1,bc2,oc1,oc2, + dust1,dust2,dust3,dust4,dust5, + seas1,seas2,seas3,seas4] diff --git a/observations/aero/viirs_n20_aod.yaml.j2 b/observations/aero/viirs_n20_aod.yaml.j2 new file mode 100644 index 0000000..5ef6aca --- /dev/null +++ b/observations/aero/viirs_n20_aod.yaml.j2 @@ -0,0 +1,63 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: viirs_n20_aod + obsdatain: + engine: + type: H5File + obsfile: "{{aero_obsdatain_path}}/{{aero_obsdatain_prefix}}{{observation_from_jcb}}{{aero_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{aero_obsdataout_path}}/{{aero_obsdataout_prefix}}{{observation_from_jcb}}{{aero_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [aerosolOpticalDepth] + channels: 4 + + # Observation Operator + # -------------------- + get values: + interpolation method: barycentric + time interpolation: linear + obs operator: + name: AodCRTM + Absorbers: [H2O,O3] + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id v.viirs-m_j1 + EndianType: little_endian + AerosolOption: aerosols_gocart_default + CoefficientPath: "{{crtm_coefficient_path}}" + obs error: + covariance model: diagonal + + # Observation Bias Correction (VarBC) + # ----------------------------------- + + # Observation Filters (QC) + # ------------------------ + obs filters: + - filter: PreQC + maxvalue: 1 + - filter: Domain Check + where: + - variable: + name: latitude@MetaData + minvalue: -60 + maxvalue: 60 + - filter: Bounds Check + filter variables: + - name: aerosolOpticalDepth + channels: 4 + minvalue: 0 + maxvalue: 4.9 + action: + name: reject + - filter: Background Check + channels: 4 + threshold: 3.0 + action: + name: inflate error + inflation factor: 3.0 diff --git a/observations/aero/viirs_n21_aod.yaml.j2 b/observations/aero/viirs_n21_aod.yaml.j2 new file mode 100644 index 0000000..9ea4140 --- /dev/null +++ b/observations/aero/viirs_n21_aod.yaml.j2 @@ -0,0 +1,63 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: viirs_n21_aod + obsdatain: + engine: + type: H5File + obsfile: "{{aero_obsdatain_path}}/{{aero_obsdatain_prefix}}{{observation_from_jcb}}{{aero_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{aero_obsdataout_path}}/{{aero_obsdataout_prefix}}{{observation_from_jcb}}{{aero_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [aerosolOpticalDepth] + channels: 4 + + # Observation Operator + # -------------------- + get values: + interpolation method: barycentric + time interpolation: linear + obs operator: + name: AodCRTM + Absorbers: [H2O,O3] + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id v.viirs-m_j2 + EndianType: little_endian + AerosolOption: aerosols_gocart_default + CoefficientPath: "{{crtm_coefficient_path}}" + obs error: + covariance model: diagonal + + # Observation Bias Correction (VarBC) + # ----------------------------------- + + # Observation Filters (QC) + # ------------------------ + obs filters: + - filter: PreQC + maxvalue: 1 + - filter: Domain Check + where: + - variable: + name: latitude@MetaData + minvalue: -60 + maxvalue: 60 + - filter: Bounds Check + filter variables: + - name: aerosolOpticalDepth + channels: 4 + minvalue: 0 + maxvalue: 4.9 + action: + name: reject + - filter: Background Check + channels: 4 + threshold: 3.0 + action: + name: inflate error + inflation factor: 3.0 diff --git a/observations/aero/viirs_npp_aod.yaml.j2 b/observations/aero/viirs_npp_aod.yaml.j2 new file mode 100644 index 0000000..57cad00 --- /dev/null +++ b/observations/aero/viirs_npp_aod.yaml.j2 @@ -0,0 +1,63 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: viirs_npp_aod + obsdatain: + engine: + type: H5File + obsfile: "{{aero_obsdatain_path}}/{{aero_obsdatain_prefix}}{{observation_from_jcb}}{{aero_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{aero_obsdataout_path}}/{{aero_obsdataout_prefix}}{{observation_from_jcb}}{{aero_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [aerosolOpticalDepth] + channels: 4 + + # Observation Operator + # -------------------- + get values: + interpolation method: barycentric + time interpolation: linear + obs operator: + name: AodCRTM + Absorbers: [H2O,O3] + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id v.viirs-m_npp + EndianType: little_endian + AerosolOption: aerosols_gocart_default + CoefficientPath: "{{crtm_coefficient_path}}" + obs error: + covariance model: diagonal + + # Observation Bias Correction (VarBC) + # ----------------------------------- + + # Observation Filters (QC) + # ------------------------ + obs filters: + - filter: PreQC + maxvalue: 1 + - filter: Domain Check + where: + - variable: + name: latitude@MetaData + minvalue: -60 + maxvalue: 60 + - filter: Bounds Check + filter variables: + - name: aerosolOpticalDepth + channels: 4 + minvalue: 0 + maxvalue: 4.9 + action: + name: reject + - filter: Background Check + channels: 4 + threshold: 3.0 + action: + name: inflate error + inflation factor: 3.0 From 3cd697f4e5e8f20cbd5577c4ffaff8e4bf557e59 Mon Sep 17 00:00:00 2001 From: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:53:27 -0400 Subject: [PATCH 3/4] Add safety feature for testing algo changes (#35) When the JCB tests are being run (from anywhere) the code will clone the main JCB repo and when doing so it will look for the branch name of the PR. Then it will run a script to clone all the clients that are registered for testing as well as the algorithms repo. This script will use the branch name of the main JCB repo and if found for any clients clone that branch. Now imagine this scenario: someone is making changes to their client (say gdas) and simultaneously making changes to jcb-algorithms. They might realize that they need to change another client to accommodate the algorithm changes so they make a branch with the same name in (say) rrfs. However if they do not have that branch in the main jcb repo there wouldn't exist a mechanism to check out the correct branch of jcb-algorithms and the other clients during the integration tests. We could put that kind of logic in the GitHub script but that would be complicated and have to propagate to every jcb repo. The change in this code forces the user to make a branch of the main jcb repo if they have created the same branch in the jcb-algorithms branch and thus provide all the mechanisms for the scripts to check everything out properly. The algorithms should really be part of JCB since they are generic and it would avoid the need to do this. But because of EE2 compliance we can't have YAMLs with source code, which is why they are in a separate repo. Co-authored-by: danholdaway --- .github/workflows/run_jcb_basic_testing.yaml | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/run_jcb_basic_testing.yaml b/.github/workflows/run_jcb_basic_testing.yaml index 97714eb..4457e67 100644 --- a/.github/workflows/run_jcb_basic_testing.yaml +++ b/.github/workflows/run_jcb_basic_testing.yaml @@ -52,6 +52,34 @@ jobs: echo "JCB_BRANCH=develop" >> $GITHUB_ENV fi + - name: Check for the branch name in the jcb-algorithms repo + run: | + BRANCH_NAME=${{ env.JCB_APP_BRANCH }} + if git ls-remote --heads https://github.com/NOAA-EMC/jcb-algorithms.git $BRANCH_NAME | grep -q "refs/heads/$BRANCH_NAME"; then + echo "Branch $BRANCH_NAME exists in jcb-algorithms repo." + echo "JCB_ALGO_BRANCH=$BRANCH_NAME" >> $GITHUB_ENV + + # If the branch exists in jcb-algorithms repo but JCB_BRANCH is develop then we need to + # throw an error. This is not a safe situation. The developer should create a branch in + # the main jcb repo with the same name as the branch in the jcb-application repo. + # If there were branches in other apps to account for the changes in the algorithm repo + # they could not be tested here since the scipt is not clever enough to check for + # the existence of the branch being tested here in all the clients. The safest thing to do + # is simply create a branch with the same name (even if empty) in the main jcb repo. This + # will ensure the branches of the other applications are checked out by the init script. + + if [ "${{ env.JCB_BRANCH }}" == "develop" ]; then + echo "Branch $BRANCH_NAME exists in jcb-algorithms repo but not in the main jcb repo. " + echo "Please create a branch with the same name (even if empty with no PR) in the main " + echo "jcb repo. This ensures safely checking all the clients that depend on the " + echo "jcb and jcb-algorithms repos with the changes being proposed." + exit 1 + fi + else + echo "Branch $BRANCH_NAME does not exist in jcb repo. Using develop branch." + echo "JCB_ALGO_BRANCH=develop" >> $GITHUB_ENV + fi + - name: Clone jcb repository run: | mkdir -p empty_hooks From b39f108c58601bb082bd75c9519fb5dcb6f04920 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 23 Oct 2024 12:00:22 -0400 Subject: [PATCH 4/4] Logic fails post merge --- .github/workflows/run_jcb_basic_testing.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_jcb_basic_testing.yaml b/.github/workflows/run_jcb_basic_testing.yaml index 4457e67..102146c 100644 --- a/.github/workflows/run_jcb_basic_testing.yaml +++ b/.github/workflows/run_jcb_basic_testing.yaml @@ -52,9 +52,17 @@ jobs: echo "JCB_BRANCH=develop" >> $GITHUB_ENV fi - - name: Check for the branch name in the jcb-algorithms repo + - name: Check for the branch name in the jcb-algorithms repo (if not develop) run: | BRANCH_NAME=${{ env.JCB_APP_BRANCH }} + + # If branch name is develop then we don't need to check the jcb-algorithms repo. + if [ "$BRANCH_NAME" == "develop" ]; then + echo "JCB_ALGO_BRANCH=develop" >> $GITHUB_ENV + exit 0 + fi + + # If the branch is not develop then check the jcb-algorithms repo. if git ls-remote --heads https://github.com/NOAA-EMC/jcb-algorithms.git $BRANCH_NAME | grep -q "refs/heads/$BRANCH_NAME"; then echo "Branch $BRANCH_NAME exists in jcb-algorithms repo." echo "JCB_ALGO_BRANCH=$BRANCH_NAME" >> $GITHUB_ENV @@ -75,9 +83,6 @@ jobs: echo "jcb and jcb-algorithms repos with the changes being proposed." exit 1 fi - else - echo "Branch $BRANCH_NAME does not exist in jcb repo. Using develop branch." - echo "JCB_ALGO_BRANCH=develop" >> $GITHUB_ENV fi - name: Clone jcb repository