From d95144d446cf369c803be9e0f93bf7891821a086 Mon Sep 17 00:00:00 2001 From: oelbert Date: Wed, 13 Jan 2021 12:46:32 -0500 Subject: [PATCH] Add namelists for c96 baroclinic setup for benchmarking [DSL-315] (#117) We've added namelists for benchmarking runs at C96 resolution. This also switches fill_dp to false for all of our runs, which should be the default going forward. This also increments the Fortran version to 7.2.4 because of the namelist changes. --- .jenkins/actions/serialize_data.sh | 3 +- .../serialized_test_data_generation/Makefile | 2 +- .../configs/c12_54ranks_standard.yml | 2 +- .../configs/c12_6ranks_baroclinic.yml | 2 +- .../configs/c12_6ranks_standard.yml | 2 +- .../configs/c48_6ranks_baroclinic.yml | 2 +- .../configs/c48_6ranks_standard.yml | 2 +- .../configs/c96_54ranks_baroclinic.yml | 329 ++++++++++++++++++ .../configs/c96_6ranks_baroclinic.yml | 329 ++++++++++++++++++ .../make_all_datasets.sh | 2 +- 10 files changed, 666 insertions(+), 9 deletions(-) create mode 100644 tests/serialized_test_data_generation/configs/c96_54ranks_baroclinic.yml create mode 100644 tests/serialized_test_data_generation/configs/c96_6ranks_baroclinic.yml diff --git a/.jenkins/actions/serialize_data.sh b/.jenkins/actions/serialize_data.sh index 6f8834793..fb5f2009e 100755 --- a/.jenkins/actions/serialize_data.sh +++ b/.jenkins/actions/serialize_data.sh @@ -63,14 +63,13 @@ else fi echo "FORCE_PUSH: ${FORCE_PUSH}" echo "=== the following setup is being used ===" - +export CUDA=y # set up virtual env, if not already set up echo ">> Running pip install -r requirements.txt in venv" python3 -m venv venv . ./venv/bin/activate pip install --upgrade pip setuptools wheel pip install -r requirements.txt -export CUDA=y # configure Docker builds export DOCKER_BUILDKIT=1 export BUILD_ARGS="-q" diff --git a/tests/serialized_test_data_generation/Makefile b/tests/serialized_test_data_generation/Makefile index 464152da5..7460e7561 100644 --- a/tests/serialized_test_data_generation/Makefile +++ b/tests/serialized_test_data_generation/Makefile @@ -10,7 +10,7 @@ SER_ENV ?= ALL # -convention: -.. # -version number should be increased when serialization data is expected to change # -omit tag (e.g. 7.1.1) for "operational" serialization data, use tag for experimental serialization data -FORTRAN_VERSION ?= 7.2.3 +FORTRAN_VERSION ?= 7.2.4 # docker container image setup (used for running model, see ../../README.md) GCR_URL = us.gcr.io/vcm-ml COMPILED_IMAGE = $(GCR_URL)/fv3gfs-compiled:$(FORTRAN_VERSION)-serialize-gt4pydev diff --git a/tests/serialized_test_data_generation/configs/c12_54ranks_standard.yml b/tests/serialized_test_data_generation/configs/c12_54ranks_standard.yml index b01935ce6..26f5b96a8 100644 --- a/tests/serialized_test_data_generation/configs/c12_54ranks_standard.yml +++ b/tests/serialized_test_data_generation/configs/c12_54ranks_standard.yml @@ -120,7 +120,7 @@ namelist: dwind_2d: false external_ic: true fill: true - fill_dp: true + fill_dp: false fv_debug: false fv_sg_adj: 600 gfs_phil: false diff --git a/tests/serialized_test_data_generation/configs/c12_6ranks_baroclinic.yml b/tests/serialized_test_data_generation/configs/c12_6ranks_baroclinic.yml index 92dafb1fb..e36be7a88 100644 --- a/tests/serialized_test_data_generation/configs/c12_6ranks_baroclinic.yml +++ b/tests/serialized_test_data_generation/configs/c12_6ranks_baroclinic.yml @@ -121,7 +121,7 @@ namelist: dwind_2d: false external_ic: false fill: true - fill_dp: true + fill_dp: false fv_debug: false fv_sg_adj: 0 gfs_phil: false diff --git a/tests/serialized_test_data_generation/configs/c12_6ranks_standard.yml b/tests/serialized_test_data_generation/configs/c12_6ranks_standard.yml index 997952170..c44795734 100644 --- a/tests/serialized_test_data_generation/configs/c12_6ranks_standard.yml +++ b/tests/serialized_test_data_generation/configs/c12_6ranks_standard.yml @@ -120,7 +120,7 @@ namelist: dwind_2d: false external_ic: true fill: true - fill_dp: true + fill_dp: false fv_debug: false fv_sg_adj: 600 gfs_phil: false diff --git a/tests/serialized_test_data_generation/configs/c48_6ranks_baroclinic.yml b/tests/serialized_test_data_generation/configs/c48_6ranks_baroclinic.yml index 8a1781ee6..91fac68e8 100644 --- a/tests/serialized_test_data_generation/configs/c48_6ranks_baroclinic.yml +++ b/tests/serialized_test_data_generation/configs/c48_6ranks_baroclinic.yml @@ -121,7 +121,7 @@ namelist: dwind_2d: false external_ic: false fill: true - fill_dp: true + fill_dp: false fv_debug: false fv_sg_adj: 0 gfs_phil: false diff --git a/tests/serialized_test_data_generation/configs/c48_6ranks_standard.yml b/tests/serialized_test_data_generation/configs/c48_6ranks_standard.yml index 7ceb9d442..f76f10fd5 100644 --- a/tests/serialized_test_data_generation/configs/c48_6ranks_standard.yml +++ b/tests/serialized_test_data_generation/configs/c48_6ranks_standard.yml @@ -120,7 +120,7 @@ namelist: dwind_2d: false external_ic: true fill: true - fill_dp: true + fill_dp: false fv_debug: false fv_sg_adj: 600 gfs_phil: false diff --git a/tests/serialized_test_data_generation/configs/c96_54ranks_baroclinic.yml b/tests/serialized_test_data_generation/configs/c96_54ranks_baroclinic.yml new file mode 100644 index 000000000..85095379e --- /dev/null +++ b/tests/serialized_test_data_generation/configs/c96_54ranks_baroclinic.yml @@ -0,0 +1,329 @@ +data_table: default +diag_table: gs://vcm-fv3config/config/diag_table/no_output/v1.0/diag_table +field_table: gs://vcm-fv3config/config/field_table/baroclinic_test/v1.0/field_table +experiment_name: c96_54ranks_baroclinic +forcing: gs://vcm-fv3config/data/base_forcing/v1.1/ +orographic_forcing: gs://vcm-fv3config/data/orographic_data/v1.0 +# To use internal test cases, point to an empty ICs directory +initial_conditions: "" +namelist: + amip_interp_nml: + data_set: hurrell + date_out_of_range: climo + do_sst_pert: false + interp_oi_sst: true + no_anom_sst: true + sst_pert: 0.0 + sst_pert_type: random + use_daily: false + use_ncep_ice: true + use_ncep_sst: true + atmos_model_nml: + blocksize: -1 + chksum_debug: false + disable_phys_restart_write: true + dycore_only: true + fdiag: 1.0 + cires_ugwp_nml: + knob_ugwp_azdir: + - 2 + - 4 + - 4 + - 4 + knob_ugwp_doaxyz: 1 + knob_ugwp_doheat: 1 + knob_ugwp_dokdis: 0 + knob_ugwp_effac: + - 1 + - 1 + - 1 + - 1 + knob_ugwp_ndx4lh: 4 + knob_ugwp_solver: 2 + knob_ugwp_source: + - 1 + - 1 + - 1 + - 0 + knob_ugwp_stoch: + - 0 + - 0 + - 0 + - 0 + knob_ugwp_version: 0 + knob_ugwp_wvspec: + - 1 + - 32 + - 32 + - 32 + launch_level: 55 + cloud_diagnosis_nml: + reiflag: 4 + reimax: 150.0 + reimin: 10.0 + rewmax: 10.0 + rewmin: 5.0 + coupler_nml: + atmos_nthreads: 1 + calendar: julian + current_date: + - 2016 + - 8 + - 1 + - 0 + - 0 + - 0 + days: 0 + dt_atmos: 225 + dt_ocean: 225 + hours: 0 + memuse_verbose: false + minutes: 3 + months: 0 + ncores_per_node: 32 + seconds: 45 + use_hyper_thread: false + diag_manager_nml: + max_axes: 240 + max_files: 106 + max_num_axis_sets: 100 + prepend_date: false + external_ic_nml: + checker_tr: false + filtered_terrain: true + gfs_dwinds: true + levp: 64 + nt_checker: 0 + fms_io_nml: + checksum_required: false + max_files_r: 100 + max_files_w: 100 + fms_nml: + clock_grain: ROUTINE + domains_stack_size: 3000000 + print_memory_usage: false + fv_core_nml: + a_imp: 1.0 + adjust_dry_mass: false + beta: 0.0 + consv_am: false + consv_te: 0.0 + d2_bg: 0.0 + d2_bg_k1: 0.2 + d2_bg_k2: 0.1 + d4_bg: 0.15 + d_con: 1.0 + d_ext: 0.0 + dddmp: 0.5 + delt_max: 0.002 + disable_fv_restart_write: true + dnats: 1 + do_sat_adj: false + do_vort_damp: true + dwind_2d: false + external_ic: false + fill: true + fill_dp: false + fv_debug: false + fv_sg_adj: 0 + gfs_phil: false + hord_dp: 6 + hord_mt: 6 + hord_tm: 6 + hord_tr: 8 + hord_vt: 6 + hydrostatic: false + io_layout: + - 1 + - 1 + k_split: 7 + ke_bg: 0.0 + kord_mt: 9 + kord_tm: -9 + kord_tr: 9 + kord_wz: 9 + layout: + - 3 + - 3 + make_nh: true + mountain: false + n_split: 8 + n_sponge: 20 + na_init: 1 + ncep_ic: false + nggps_ic: false + nord: 3 + npx: 97 + npy: 97 + npz: 79 + ntiles: 6 + nudge: false + nudge_qv: true + nwat: 6 + p_fac: 0.05 + phys_hydrostatic: false + print_freq: 1 + range_warn: true + reset_eta: false + rf_cutoff: 3000.0 + rf_fast: true + tau: 10.0 + tau_h2o: 0.0 + use_hydro_pressure: false + vtdm4: 0.06 + warm_start: false + z_tracer: true + fv_grid_nml: {} + gfdl_cloud_microphysics_nml: + c_cracw: 0.8 + c_paut: 0.5 + c_pgacs: 0.01 + c_psaci: 0.05 + ccn_l: 300.0 + ccn_o: 100.0 + const_vg: false + const_vi: false + const_vr: false + const_vs: false + de_ice: false + do_qa: true + do_sedi_heat: false + do_sedi_w: true + dw_land: 0.15 + dw_ocean: 0.1 + fast_sat_adj: true + fix_negative: true + icloud_f: 0 + irain_f: 0 + mono_prof: false + mp_time: 225.0 + prog_ccn: false + qi0_crt: 8.0e-05 + qi_lim: 1.0 + ql_gen: 0.001 + ql_mlt: 0.002 + qs0_crt: 0.003 + qs_mlt: 1.0e-06 + rad_graupel: true + rad_rain: true + rad_snow: true + rh_inc: 0.2 + rh_inr: 0.3 + rh_ins: 0.3 + rthresh: 1.0e-05 + sedi_transport: true + tau_g2v: 1200.0 + tau_i2s: 1000.0 + tau_l2v: 300.0 + tau_v2l: 90.0 + use_ccn: true + use_ppm: false + vg_max: 16.0 + vi_max: 1.0 + vr_max: 16.0 + vs_max: 2.0 + z_slope_ice: true + z_slope_liq: true + gfs_physics_nml: + c0s_shal: 0.01 + c1_shal: 0.005 + cal_pre: false + cdmbgwd: + - 0.0 + - 0.0 + - 0.0 + - 0.0 + cnvcld: false + cnvgwd: false + debug: false + do_ocean: false + do_deep: false + dspheat: true + fhcyc: 24.0 + fhlwr: 1800.0 + fhswr: 1800.0 + fhzero: 0.25 + hybedmf: false + iaer: 111 + ialb: 1 + ico2: 2 + iems: 1 + imfdeepcnv: 2 + imfshalcnv: 2 + imp_physics: 11 + isol: 2 + isot: 1 + isubc_lw: 2 + isubc_sw: 2 + ivegsrc: 1 + ldiag3d: false + lwhtr: true + ncld: 5 + nst_anl: true + pdfcld: false + pre_rad: false + prslrd0: 0.0 + random_clds: false + redrag: true + sfc_override: true + satmedmf: true + shal_cnv: true + swhtr: true + trans_trac: true + use_ufo: true + xkzm_h: 0.1 + xkzm_m: 0.01 + xkzminv: 0.0 + interpolator_nml: + interp_method: conserve_great_circle + nam_stochy: + lat_s: 96 + lon_s: 192 + ntrunc: 94 + sfc_prop_override_nml: + ideal_sst: true + sst_max: 305 + sst_profile: 1 + test_case_nml: + test_case: 13 + namsfc: + fabsl: 99999 + faisl: 99999 + faiss: 99999 + fnabsc: grb/global_mxsnoalb.uariz.t1534.3072.1536.rg.grb + fnacna: '' + fnaisc: grb/CFSR.SEAICE.1982.2012.monthly.clim.grb + fnalbc: grb/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb + fnalbc2: grb/global_albedo4.1x1.grb + fnglac: grb/global_glacier.2x2.grb + fnmskh: grb/seaice_newland.grb + fnmxic: grb/global_maxice.2x2.grb + fnslpc: grb/global_slope.1x1.grb + fnsmcc: grb/global_soilmgldas.t1534.3072.1536.grb + fnsnoa: '' + fnsnoc: grb/global_snoclim.1.875.grb + fnsotc: grb/global_soiltype.statsgo.t1534.3072.1536.rg.grb + fntg3c: grb/global_tg3clim.2.6x1.5.grb + fntsfa: '' + fntsfc: grb/RTGSST.1982.2012.monthly.clim.grb + fnvegc: grb/global_vegfrac.0.144.decpercent.grb + fnvetc: grb/global_vegtype.igbp.t1534.3072.1536.rg.grb + fnvmnc: grb/global_shdmin.0.144x0.144.grb + fnvmxc: grb/global_shdmax.0.144x0.144.grb + fnzorc: igbp + fsicl: 99999 + fsics: 99999 + fslpl: 99999 + fsmcl: + - 99999 + - 99999 + - 99999 + fsnol: 99999 + fsnos: 99999 + fsotl: 99999 + ftsfl: 99999 + ftsfs: 90 + fvetl: 99999 + fvmnl: 99999 + fvmxl: 99999 + ldebug: false diff --git a/tests/serialized_test_data_generation/configs/c96_6ranks_baroclinic.yml b/tests/serialized_test_data_generation/configs/c96_6ranks_baroclinic.yml new file mode 100644 index 000000000..4669a1ea7 --- /dev/null +++ b/tests/serialized_test_data_generation/configs/c96_6ranks_baroclinic.yml @@ -0,0 +1,329 @@ +data_table: default +diag_table: gs://vcm-fv3config/config/diag_table/no_output/v1.0/diag_table +field_table: gs://vcm-fv3config/config/field_table/baroclinic_test/v1.0/field_table +experiment_name: c96_6ranks_baroclinic +forcing: gs://vcm-fv3config/data/base_forcing/v1.1/ +orographic_forcing: gs://vcm-fv3config/data/orographic_data/v1.0 +# To use internal test cases, point to an empty ICs directory +initial_conditions: "" +namelist: + amip_interp_nml: + data_set: hurrell + date_out_of_range: climo + do_sst_pert: false + interp_oi_sst: true + no_anom_sst: true + sst_pert: 0.0 + sst_pert_type: random + use_daily: false + use_ncep_ice: true + use_ncep_sst: true + atmos_model_nml: + blocksize: -1 + chksum_debug: false + disable_phys_restart_write: true + dycore_only: true + fdiag: 1.0 + cires_ugwp_nml: + knob_ugwp_azdir: + - 2 + - 4 + - 4 + - 4 + knob_ugwp_doaxyz: 1 + knob_ugwp_doheat: 1 + knob_ugwp_dokdis: 0 + knob_ugwp_effac: + - 1 + - 1 + - 1 + - 1 + knob_ugwp_ndx4lh: 4 + knob_ugwp_solver: 2 + knob_ugwp_source: + - 1 + - 1 + - 1 + - 0 + knob_ugwp_stoch: + - 0 + - 0 + - 0 + - 0 + knob_ugwp_version: 0 + knob_ugwp_wvspec: + - 1 + - 32 + - 32 + - 32 + launch_level: 55 + cloud_diagnosis_nml: + reiflag: 4 + reimax: 150.0 + reimin: 10.0 + rewmax: 10.0 + rewmin: 5.0 + coupler_nml: + atmos_nthreads: 1 + calendar: julian + current_date: + - 2016 + - 8 + - 1 + - 0 + - 0 + - 0 + days: 0 + dt_atmos: 225 + dt_ocean: 225 + hours: 0 + memuse_verbose: false + minutes: 3 + months: 0 + ncores_per_node: 32 + seconds: 45 + use_hyper_thread: false + diag_manager_nml: + max_axes: 240 + max_files: 106 + max_num_axis_sets: 100 + prepend_date: false + external_ic_nml: + checker_tr: false + filtered_terrain: true + gfs_dwinds: true + levp: 64 + nt_checker: 0 + fms_io_nml: + checksum_required: false + max_files_r: 100 + max_files_w: 100 + fms_nml: + clock_grain: ROUTINE + domains_stack_size: 3000000 + print_memory_usage: false + fv_core_nml: + a_imp: 1.0 + adjust_dry_mass: false + beta: 0.0 + consv_am: false + consv_te: 0.0 + d2_bg: 0.0 + d2_bg_k1: 0.2 + d2_bg_k2: 0.1 + d4_bg: 0.15 + d_con: 1.0 + d_ext: 0.0 + dddmp: 0.5 + delt_max: 0.002 + disable_fv_restart_write: true + dnats: 1 + do_sat_adj: false + do_vort_damp: true + dwind_2d: false + external_ic: false + fill: true + fill_dp: false + fv_debug: false + fv_sg_adj: 0 + gfs_phil: false + hord_dp: 6 + hord_mt: 6 + hord_tm: 6 + hord_tr: 8 + hord_vt: 6 + hydrostatic: false + io_layout: + - 1 + - 1 + k_split: 7 + ke_bg: 0.0 + kord_mt: 9 + kord_tm: -9 + kord_tr: 9 + kord_wz: 9 + layout: + - 1 + - 1 + make_nh: true + mountain: false + n_split: 8 + n_sponge: 20 + na_init: 1 + ncep_ic: false + nggps_ic: false + nord: 3 + npx: 97 + npy: 97 + npz: 79 + ntiles: 6 + nudge: false + nudge_qv: true + nwat: 6 + p_fac: 0.05 + phys_hydrostatic: false + print_freq: 1 + range_warn: true + reset_eta: false + rf_cutoff: 3000.0 + rf_fast: true + tau: 10.0 + tau_h2o: 0.0 + use_hydro_pressure: false + vtdm4: 0.06 + warm_start: false + z_tracer: true + fv_grid_nml: {} + gfdl_cloud_microphysics_nml: + c_cracw: 0.8 + c_paut: 0.5 + c_pgacs: 0.01 + c_psaci: 0.05 + ccn_l: 300.0 + ccn_o: 100.0 + const_vg: false + const_vi: false + const_vr: false + const_vs: false + de_ice: false + do_qa: true + do_sedi_heat: false + do_sedi_w: true + dw_land: 0.15 + dw_ocean: 0.1 + fast_sat_adj: true + fix_negative: true + icloud_f: 0 + irain_f: 0 + mono_prof: false + mp_time: 225.0 + prog_ccn: false + qi0_crt: 8.0e-05 + qi_lim: 1.0 + ql_gen: 0.001 + ql_mlt: 0.002 + qs0_crt: 0.003 + qs_mlt: 1.0e-06 + rad_graupel: true + rad_rain: true + rad_snow: true + rh_inc: 0.2 + rh_inr: 0.3 + rh_ins: 0.3 + rthresh: 1.0e-05 + sedi_transport: true + tau_g2v: 1200.0 + tau_i2s: 1000.0 + tau_l2v: 300.0 + tau_v2l: 90.0 + use_ccn: true + use_ppm: false + vg_max: 16.0 + vi_max: 1.0 + vr_max: 16.0 + vs_max: 2.0 + z_slope_ice: true + z_slope_liq: true + gfs_physics_nml: + c0s_shal: 0.01 + c1_shal: 0.005 + cal_pre: false + cdmbgwd: + - 0.0 + - 0.0 + - 0.0 + - 0.0 + cnvcld: false + cnvgwd: false + debug: false + do_ocean: false + do_deep: false + dspheat: true + fhcyc: 24.0 + fhlwr: 1800.0 + fhswr: 1800.0 + fhzero: 0.25 + hybedmf: false + iaer: 111 + ialb: 1 + ico2: 2 + iems: 1 + imfdeepcnv: 2 + imfshalcnv: 2 + imp_physics: 11 + isol: 2 + isot: 1 + isubc_lw: 2 + isubc_sw: 2 + ivegsrc: 1 + ldiag3d: false + lwhtr: true + ncld: 5 + nst_anl: true + pdfcld: false + pre_rad: false + prslrd0: 0.0 + random_clds: false + redrag: true + sfc_override: true + satmedmf: true + shal_cnv: true + swhtr: true + trans_trac: true + use_ufo: true + xkzm_h: 0.1 + xkzm_m: 0.01 + xkzminv: 0.0 + interpolator_nml: + interp_method: conserve_great_circle + nam_stochy: + lat_s: 96 + lon_s: 192 + ntrunc: 94 + sfc_prop_override_nml: + ideal_sst: true + sst_max: 305 + sst_profile: 1 + test_case_nml: + test_case: 13 + namsfc: + fabsl: 99999 + faisl: 99999 + faiss: 99999 + fnabsc: grb/global_mxsnoalb.uariz.t1534.3072.1536.rg.grb + fnacna: '' + fnaisc: grb/CFSR.SEAICE.1982.2012.monthly.clim.grb + fnalbc: grb/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb + fnalbc2: grb/global_albedo4.1x1.grb + fnglac: grb/global_glacier.2x2.grb + fnmskh: grb/seaice_newland.grb + fnmxic: grb/global_maxice.2x2.grb + fnslpc: grb/global_slope.1x1.grb + fnsmcc: grb/global_soilmgldas.t1534.3072.1536.grb + fnsnoa: '' + fnsnoc: grb/global_snoclim.1.875.grb + fnsotc: grb/global_soiltype.statsgo.t1534.3072.1536.rg.grb + fntg3c: grb/global_tg3clim.2.6x1.5.grb + fntsfa: '' + fntsfc: grb/RTGSST.1982.2012.monthly.clim.grb + fnvegc: grb/global_vegfrac.0.144.decpercent.grb + fnvetc: grb/global_vegtype.igbp.t1534.3072.1536.rg.grb + fnvmnc: grb/global_shdmin.0.144x0.144.grb + fnvmxc: grb/global_shdmax.0.144x0.144.grb + fnzorc: igbp + fsicl: 99999 + fsics: 99999 + fslpl: 99999 + fsmcl: + - 99999 + - 99999 + - 99999 + fsnol: 99999 + fsnos: 99999 + fsotl: 99999 + ftsfl: 99999 + ftsfs: 90 + fvetl: 99999 + fvmnl: 99999 + fvmxl: 99999 + ldebug: false diff --git a/tests/serialized_test_data_generation/make_all_datasets.sh b/tests/serialized_test_data_generation/make_all_datasets.sh index 84bca0061..a385dd5a8 100755 --- a/tests/serialized_test_data_generation/make_all_datasets.sh +++ b/tests/serialized_test_data_generation/make_all_datasets.sh @@ -48,7 +48,7 @@ for exp_file in ${EXPERIMENTS} ; do echo "=====================================================" echo "Generating data for ${exp_name} ..." npx=`cat ${exp_file} | grep npx | sed s/npx://g | sed 's/^ *//g'` - if [ ${npx} -gt 48 ] ; then + if [ ${npx} -gt 49 ] ; then export SER_ENV="ONLY_DRIVER" else export SER_ENV="ALL"