From 7b39eb39bca2061a971ae76220998c383eea5cd5 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 5 Mar 2021 11:07:53 -0800 Subject: [PATCH 01/54] Change saturation --- FV3/atmos_cubed_sphere/model/fv_cmp.F90 | 4 ++-- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 index e2f281cfa..0d75ee4ab 100644 --- a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 +++ b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 @@ -406,7 +406,7 @@ subroutine fv_sat_adj (mdt, zvir, is, ie, js, je, ng, hydrostatic, consv_te, te0 ! factor = - min (1., fac_l2v * sqrt (max (0., ql (i, j)) / 1.e-5) * 10. * (1. - qv (i, j) / wqsat (i))) ! factor = - fac_l2v ! factor = - 1 - factor = - min (1., fac_l2v * 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% + factor = - fac_l2v * min (1., 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% src (i) = - min (ql (i, j), factor * dq0) endif qv (i, j) = qv (i, j) - src (i) @@ -449,7 +449,7 @@ subroutine fv_sat_adj (mdt, zvir, is, ie, js, je, ng, hydrostatic, consv_te, te0 ! factor = - min (1., fac_l2v * sqrt (max (0., ql (i, j)) / 1.e-5) * 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% ! factor = - fac_l2v ! factor = - 1 - factor = - min (1., fac_l2v * 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% + factor = - fac_l2v * min (1., 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% src (i) = - min (ql (i, j), factor * dq0) endif adj_fac = 1. diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 9a1820465..8f6e8bb36 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2083,7 +2083,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = min (1., fac_l2v * (10. * dq0 / qsw)) ! the rh dependent factor = 1 at 90% + factor = fac_l2v * min (1., 10. * dq0 / qsw) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From 52118a1c759a451734003be3a6baddfe08a5d6d4 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 5 Mar 2021 13:47:18 -0800 Subject: [PATCH 02/54] Another suggestion of Linjiongs e.g., factor = - fac_l2v * min (1., 10. * (1. - qv (i, j) / wqsat (i))) change it to factor = - fac_l2v * min (1., 1. * (1. - qv (i, j) / wqsat (i))) I will expect much larger cloud water change in this case because your relative humidity is very high. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 8f6e8bb36..96632ef19 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2083,7 +2083,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = fac_l2v * min (1., 10. * dq0 / qsw) ! the rh dependent factor = 1 at 90% + factor = fac_l2v * min (1., 1. * dq0 / qsw) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From 7a2b2b4a4b0baf6447eb183cf08f4580eb074577 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 5 Mar 2021 14:19:42 -0800 Subject: [PATCH 03/54] Decrease even further --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 96632ef19..d51900c86 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2083,7 +2083,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = fac_l2v * min (1., 1. * dq0 / qsw) ! the rh dependent factor = 1 at 90% + factor = fac_l2v * min (1., 0.5 * dq0 / qsw) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From 1015da4fe1f8c4ba4adbc11ea31cfde3007bae5e Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 5 Mar 2021 14:44:54 -0800 Subject: [PATCH 04/54] Revert "Decrease even further" This reverts commit 7a2b2b4a4b0baf6447eb183cf08f4580eb074577. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index d51900c86..96632ef19 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2083,7 +2083,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = fac_l2v * min (1., 0.5 * dq0 / qsw) ! the rh dependent factor = 1 at 90% + factor = fac_l2v * min (1., 1. * dq0 / qsw) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From e40c856a62d0d4ae03a71f11f729819a39a40872 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 5 Mar 2021 14:44:57 -0800 Subject: [PATCH 05/54] Revert "Another suggestion of Linjiongs" This reverts commit 52118a1c759a451734003be3a6baddfe08a5d6d4. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 96632ef19..8f6e8bb36 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2083,7 +2083,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = fac_l2v * min (1., 1. * dq0 / qsw) ! the rh dependent factor = 1 at 90% + factor = fac_l2v * min (1., 10. * dq0 / qsw) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From 3a38a3827f7e174328a00e154c170eceacf724e5 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 5 Mar 2021 14:44:59 -0800 Subject: [PATCH 06/54] Revert "Change saturation" This reverts commit 7b39eb39bca2061a971ae76220998c383eea5cd5. --- FV3/atmos_cubed_sphere/model/fv_cmp.F90 | 4 ++-- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 index 0d75ee4ab..e2f281cfa 100644 --- a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 +++ b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 @@ -406,7 +406,7 @@ subroutine fv_sat_adj (mdt, zvir, is, ie, js, je, ng, hydrostatic, consv_te, te0 ! factor = - min (1., fac_l2v * sqrt (max (0., ql (i, j)) / 1.e-5) * 10. * (1. - qv (i, j) / wqsat (i))) ! factor = - fac_l2v ! factor = - 1 - factor = - fac_l2v * min (1., 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% + factor = - min (1., fac_l2v * 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% src (i) = - min (ql (i, j), factor * dq0) endif qv (i, j) = qv (i, j) - src (i) @@ -449,7 +449,7 @@ subroutine fv_sat_adj (mdt, zvir, is, ie, js, je, ng, hydrostatic, consv_te, te0 ! factor = - min (1., fac_l2v * sqrt (max (0., ql (i, j)) / 1.e-5) * 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% ! factor = - fac_l2v ! factor = - 1 - factor = - fac_l2v * min (1., 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% + factor = - min (1., fac_l2v * 10. * (1. - qv (i, j) / wqsat (i))) ! the rh dependent factor = 1 at 90% src (i) = - min (ql (i, j), factor * dq0) endif adj_fac = 1. diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 8f6e8bb36..9a1820465 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2083,7 +2083,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = fac_l2v * min (1., 10. * dq0 / qsw) ! the rh dependent factor = 1 at 90% + factor = min (1., fac_l2v * (10. * dq0 / qsw)) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From 2b5aab208ba61834a25159e5cd4fa61643cec7eb Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 5 Mar 2021 15:10:14 -0800 Subject: [PATCH 07/54] Reduce rh dependent factor --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 9a1820465..f92854ca3 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2083,7 +2083,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = min (1., fac_l2v * (10. * dq0 / qsw)) ! the rh dependent factor = 1 at 90% + factor = min (1., fac_l2v * (0.5 * dq0 / qsw)) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From f038d356cbf439cabaf66174173174e44897463c Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Mon, 8 Mar 2021 17:46:47 -0800 Subject: [PATCH 08/54] Change auto-conversion temperature threshold --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index f92854ca3..9bb567ce9 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -1275,7 +1275,7 @@ subroutine warm_rain (dt, ktop, kbot, dp, dz, tz, qv, ql, qr, qi, qs, qg, & do k = ktop, kbot qc0 = fac_rc * ccn (k) - if (tz (k) > t_wfr + dt_fr) then + if (tz (k) > t_wfr) then dl (k) = min (max (1.e-6, dl (k)), 0.5 * ql (k)) ! -------------------------------------------------------------------- ! as in klein's gfdl am2 stratiform scheme (with subgrid variations) From 36a17824622a7eb1ea51fa967ef5578a24cec5ed Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 17:37:04 -0700 Subject: [PATCH 09/54] nix: build fv3 with nix --- .circleci/config.yml | 41 ++++++++++ FV3/conf/configure.fv3 | 25 ++---- default.nix | 19 +++++ docker/Dockerfile | 2 + nix/esmf/build.sh | 21 +++++ nix/esmf/default.nix | 84 ++++++++++++++++++++ nix/fms/default.nix | 51 ++++++++++++ nix/fv3/configure.fv3 | 135 ++++++++++++++++++++++++++++++++ nix/fv3/default.nix | 87 ++++++++++++++++++++ nix/nceplibs/default.nix | 39 +++++++++ tests/pytest/conftest.py | 2 +- tests/pytest/test_regression.py | 14 ++++ 12 files changed, 502 insertions(+), 18 deletions(-) create mode 100644 default.nix create mode 100644 nix/esmf/build.sh create mode 100644 nix/esmf/default.nix create mode 100644 nix/fms/default.nix create mode 100644 nix/fv3/configure.fv3 create mode 100644 nix/fv3/default.nix create mode 100644 nix/nceplibs/default.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index daeffe862..4fdc3c747 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,6 +3,45 @@ orbs: gcp-cli: circleci/gcp-cli@2.1.0 gcp-gcr: circleci/gcp-gcr@0.6.1 jobs: + nix-mac: + macos: + xcode: 11.3.0 + steps: + - run: sudo sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume + - run: nix-env -i git openssh + - checkout + - run: + name: "Pull Submodules" + command: | + git submodule init + git submodule update + - restore_cache: + keys: + - key + - run: nix-build . -j 4 + nix: + docker: + - image: nixos/nix + environment: + FV3CONFIG_CACHE_DIR: /tmp/.fv3config + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json + steps: + - run: nix-env -i git openssh google-cloud-sdk + - checkout + - run: + name: "gcloud auth" + command: | + echo $ENCODED_GCR_KEY | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS + - run: + name: "Pull Submodules" + command: | + git submodule init + git submodule update + - run: | + nix-build nix -j 4 + - run: + name: Test + command: nix-shell --run "pytest tests/pytest --capture=no --verbose --refdir /Users/noah/workspace/VulcanClimateModeling/fv3gfs-fortran/tests/pytest/reference/circleci --image_runner native" build_default: machine: docker_layer_caching: true @@ -67,3 +106,5 @@ workflows: build: jobs: - build_default + - nix + - nix-mac diff --git a/FV3/conf/configure.fv3 b/FV3/conf/configure.fv3 index b941f6b98..4f5dc3d32 100644 --- a/FV3/conf/configure.fv3 +++ b/FV3/conf/configure.fv3 @@ -2,10 +2,10 @@ ############ # commands # ############ -FC = FC=gfortran mpifort -CC = CC=gcc mpicc +FC = mpif90 +CC = mpicc CXX = g++ -LD = mpifort +LD = mpif90 NETCDF_DIR = /usr @@ -17,7 +17,7 @@ DEBUG = REPRO = Y VERBOSE = OPENMP = Y -AVX2 = N +AVX2 = Y HYDRO = N 32BIT = N @@ -63,8 +63,8 @@ FFLAGS += -fdefault-double-8 -fdefault-real-8 endif ifeq ($(AVX2),Y) -FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align -CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +FFLAGS += -mavx +CFLAGS += -mavx endif FFLAGS_OPT = -O2 @@ -84,19 +84,13 @@ CFLAGS_OPENMP = -fopenmp CFLAGS_DEBUG = -O0 -g CFLAGS_GCOV = --coverage -# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT -# *_TEST will match the production if no new option(s) is(are) to be tested. -FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits -CFLAGS_TEST = -O2 - -LDFLAGS := -L${ESMF_DIR}/lib/libO3/Linux.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ +LDFLAGS := -L${ESMF_DIR}/lib -L${FMS_DIR}/libFMS/.libs/ LDFLAGS_OPENMP := -fopenmp LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M LDFLAGS_GCOV = --coverage # start with blank LIBS LIBS := - LIBS += -lgfortran ifneq ($(REPRO),) @@ -127,9 +121,6 @@ ifneq ($(OPENMP),) CFLAGS += $(CFLAGS_OPENMP) FFLAGS += $(FFLAGS_OPENMP) LDFLAGS += $(LDFLAGS_OPENMP) -# to correct a loader bug on gaea: envars below set by module load intel -#LIBS += -L$(INTEL_PATH)/$(INTEL_MAJOR_VERSION)/$(INTEL_MINOR_VERSION)/lib/intel64 -lifcoremt -#LIBS += -lifcoremt endif ifneq ($(VERBOSE),) @@ -138,6 +129,6 @@ FFLAGS += $(FFLAGS_VERBOSE) LDFLAGS += $(LDFLAGS_VERBOSE) endif -LIBS += -lFMS -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt +LIBS += -lFMS -L/usr/local/esmf/lib -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt LDFLAGS += $(LIBS) diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..90b57caae --- /dev/null +++ b/default.nix @@ -0,0 +1,19 @@ +# Copied from https://nixos.org/nixos/nix-pills/callpackage-design-pattern.html +let + overlay = self: super: with nixpkgs; rec { + # ensure that everything uses mpich for mpi + fms = self.callPackage ./nix/fms { }; + esmf = self.callPackage ./nix/esmf { }; + nceplibs = self.callPackage ./nix/nceplibs { }; + fv3 = self.callPackage ./nix/fv3 { }; + }; + + nixpkgs = import (builtins.fetchTarball { + # Descriptive name to make the store path easier to identify + name = "release-20.09"; + # Commit hash for nixos-unstable as of 2018-09-12 + url = "https://github.com/nixos/nixpkgs/archive/20.09.tar.gz"; + # Hash obtained using `nix-prefetch-url --unpack ` + sha256 = "1wg61h4gndm3vcprdcg7rc4s1v3jkm5xd7lw8r2f67w502y94gcy"; +}) {overlays = [overlay];}; +in nixpkgs.fv3 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 91a49a1fc..3af0e8fb7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -221,6 +221,8 @@ COPY --from=fv3gfs-sources FV3/atmos_cubed_sphere /FV3/atmos_cubed_sphere RUN cd /FV3/atmos_cubed_sphere && make clean && cd /FV3 && make -j16 +RUN PREFIX=/usr/local make -C /FV3/atmos_cubed_sphere install + ## Define final "user" image for running tests ##--------------------------------------------------------------------------------- FROM fv3gfs-build AS fv3gfs-compiled diff --git a/nix/esmf/build.sh b/nix/esmf/build.sh new file mode 100644 index 000000000..b0598b138 --- /dev/null +++ b/nix/esmf/build.sh @@ -0,0 +1,21 @@ +source $stdenv/setup + + +# copy from read-only git repo +mkdir -p $out +cp -r $src src +chmod -R +w src +cd src + +# set build variables +export ESMF_DIR=$(pwd) +export ESMF_INSTALL_PREFIX=$out +export ESMF_NETCDF_INCLUDE=$netcdffortran/include +export ESMF_NETCDF_LIBS="-lnetcdf -lnetcdff" +export ESMF_BOPT=O3 +export ESMF_CXXCOMPILEOPTS="$ESMF_CXXCOMPILEOPTS -Wno-format-security" + +# compile +make lib -j8 +make install +make installcheck diff --git a/nix/esmf/default.nix b/nix/esmf/default.nix new file mode 100644 index 000000000..adcdb341a --- /dev/null +++ b/nix/esmf/default.nix @@ -0,0 +1,84 @@ +with import {}; +# { stdenv +# , netcdffortran +# , perl +# , gfortran +# , lapack +# , blas +# , openmpi +# , fetchgit +# }: +# +{ + stdenvNoCC + ,netcdffortran + ,gfortran + ,mpich + ,coreutils + ,which + ,llvmPackages +}: +stdenvNoCC.mkDerivation rec { + pname = "esmf"; + version = "0.0.0"; + + src = builtins.fetchGit { + url = "https://git.code.sf.net/p/esmf/esmf"; + rev = "f5d862d2ec066e76647f53c239b8c58c7af28e45"; + }; + + + buildPhase = '' + # set build variables + export ESMF_DIR=$(pwd) + + export ESMF_INSTALL_PREFIX=$out + export ESMF_INSTALL_HEADERDIR=$out/include + export ESMF_INSTALL_MODDIR=$out/include + export ESMF_INSTALL_LIBDIR=$out/lib + export ESMF_INSTALL_BINDIR=$out/bin + export ESMF_INSTALL_DOCDIR=$out/share/docs + + export ESMF_NETCDF_INCLUDE=$netcdffortran/include + export ESMF_NETCDF_LIBS="-lnetcdf -lnetcdff" + export ESMF_BOPT=O3 + export ESMF_CXXCOMPILEOPTS="$ESMF_CXXCOMPILEOPTS -Wno-format-security" + + # compile + make lib -j8 + make install + make installcheck + ''; + + # need to fix the linked libraries for some reason. + # The "id" of these dylibs points to the build directory + postFixup = stdenvNoCC.lib.optionalString stdenvNoCC.isDarwin '' + function fixNameLib { + install_name_tool -id "$1" "$1" + } + fixNameLib $out/lib/libesmf.dylib + fixNameLib $out/lib/libesmf_fullylinked.dylib + ''; + + # nativeBuildInputs = [ m4 ]; + # buildInputs = [ hdf5 curl mpi ]; + buildInputs = [ netcdffortran gfortran mpich gfortran.cc coreutils which + (lib.optional stdenv.isDarwin llvmPackages.openmp) + ] ; + inherit netcdffortran gfortran; + CXX="${gfortran}/bin/g++"; + CC="${gfortran}/bin/gcc"; + ESMF_CXXCOMPILER="${CXX}"; + ESMF_CCOMPILER="${CC}"; + + + + meta = { + description = "Libraries for the Unidata network Common Data Format"; + platforms = stdenv.lib.platforms.unix; + homepage = "https://www.unidata.ucar.edu/software/netcdf/"; + license = { + url = "https://www.unidata.ucar.edu/software/netcdf/docs/copyright.html"; + }; + }; +} diff --git a/nix/fms/default.nix b/nix/fms/default.nix new file mode 100644 index 000000000..1f9d94898 --- /dev/null +++ b/nix/fms/default.nix @@ -0,0 +1,51 @@ +{ + stdenv + ,bash + ,rsync + ,gfortran + ,mpich + ,automake + ,autoconf + ,m4 + ,libtool + ,bats + ,netcdffortran + ,netcdf + ,lib + ,llvmPackages +} : +stdenv.mkDerivation rec { + pname = "fms"; + version = "0.0.0"; + + src = ../../FMS/.; + + # nativeBuildInputs = [ m4 ]; + # buildInputs = [ hdf5 curl mpi ]; + buildInputs = [ bash rsync gfortran mpich automake autoconf m4 libtool bats netcdffortran netcdf + (lib.optional stdenv.isDarwin llvmPackages.openmp) + ]; + inherit netcdffortran; + + configurePhase = '' + mkdir m4 + autoreconf --install + + export CC=mpicc + export FC=mpifort + export LOG_DRIVER_FLAGS="--comments" + export CPPFLAGS="-Duse_LARGEFILE -DMAXFIELDMETHODS_=500 -DGFS_PHYS" + export FCFLAGS="-fcray-pointer -Waliasing -ffree-line-length-none -fno-range-check -fdefault-real-8 -fdefault-double-8 -fopenmp -I$netcdffortran/include" + + ./configure --prefix=$out + ''; + + meta = { + description = "Libraries for the Unidata network Common Data Format"; + platforms = stdenv.lib.platforms.unix; + homepage = "https://www.unidata.ucar.edu/software/netcdf/"; + license = { + url = "https://www.unidata.ucar.edu/software/netcdf/docs/copyright.html"; + }; + }; +} diff --git a/nix/fv3/configure.fv3 b/nix/fv3/configure.fv3 new file mode 100644 index 000000000..1eb218ca0 --- /dev/null +++ b/nix/fv3/configure.fv3 @@ -0,0 +1,135 @@ + +############ +# commands # +############ +SHELL = bash +FC = mpif90 +CC = mpicc +CXX = g++ +LD = mpif90 +CPP = cpp + + +NETCDF_DIR = /usr + +######### +# flags # +######### +# default is 64-bit OpenMP non-hydrostatic build +DEBUG = +REPRO = Y +VERBOSE = +OPENMP = Y +AVX2 = N +HYDRO = N +32BIT = N + +NEMSIOINC = -I$(NEMSIO_INC) + +NCEPLIBS_DIR ?= /opt/NCEPlibs/lib +NCEPLIBS = -L$(NCEPLIBS_DIR) -lnemsio_d -lbacio_4 -lsp_v2.0.2_d -lw3emc_d -lw3nco_d + +############################################## +# Need to use at least GNU Make version 3.81 # +############################################## +need := 3.81 +ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need)))) +ifneq ($(need),$(ok)) +$(error Need at least make version $(need). Load module gmake/3.81) +endif + +NETCDF_ROOT = $(NETCDF_DIR) +INCLUDE ?= -I$(NETCDF_ROOT)/include + +FPPFLAGS := -cpp -Wp,-w $(INCLUDE) -fPIC +CFLAGS := $(INCLUDE) -fPIC + +FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fPIC + +CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -Duse_LARGEFILE -DUSE_GFSL63 -DGFS_PHYS -DNO_INLINE_POST +CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML + +ifeq ($(HYDRO),Y) +CPPDEFS += +else +CPPDEFS += -DMOIST_CAPPA -DUSE_COND +endif + +ifeq ($(32BIT),Y) +CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8 +else +FFLAGS += -fdefault-double-8 -fdefault-real-8 +endif + +ifeq ($(AVX2),Y) +FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +endif + +FFLAGS_OPT = -O2 +FFLAGS_REPRO = -O2 -g -fbacktrace +FFLAGS_DEBUG = -O0 -g -fbacktrace -fno-fast-math -ffree-line-length-none -fno-backslash -pedantic -Waliasing -Wampersand -Wline-truncation -Wsurprising -Wtabs -Wunderflow -fdump-core -ffpe-trap=invalid,zero,overflow -fbounds-check -finit-real=nan -finit-integer=9999999 -finit-logical=true -finit-character=35 + +TRANSCENDENTALS := -fast-transcendentals +FFLAGS_OPENMP = -fopenmp +FFLAGS_VERBOSE = -v -V -what + +CFLAGS += -D__IFC + +CFLAGS_OPT = -O2 +CFLAGS_REPRO = -O2 +CFLAGS_OPENMP = -fopenmp +CFLAGS_DEBUG = -O0 -g + +# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT +# *_TEST will match the production if no new option(s) is(are) to be tested. +FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits +CFLAGS_TEST = -O2 + +#LDFLAGS := -L${ESMF_DIR}/lib/libO3/Darwin.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ +LDFLAGS_OPENMP := -fopenmp +LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M + +# start with blank LIBS +LIBS := + +LIBS += -lgfortran + +ifneq ($(REPRO),) +CFLAGS += $(CFLAGS_REPRO) +FFLAGS += $(FFLAGS_REPRO) +FAST := +else ifneq ($(DEBUG),) +CFLAGS += $(CFLAGS_DEBUG) +FFLAGS += $(FFLAGS_DEBUG) +FAST := +else ifneq ($(TEST),) +CFLAGS += $(CFLAGS_TEST) +FFLAGS += $(FFLAGS_TEST) +FAST := +else +CFLAGS += $(CFLAGS_OPT) +FFLAGS += $(FFLAGS_OPT) +FAST := $(TRANSCENDENTALS) +endif + +ifneq ($(OPENMP),) +CFLAGS += $(CFLAGS_OPENMP) +FFLAGS += $(FFLAGS_OPENMP) +LDFLAGS += $(LDFLAGS_OPENMP) +# to correct a loader bug on gaea: envars below set by module load intel +#LIBS += -L$(INTEL_PATH)/$(INTEL_MAJOR_VERSION)/$(INTEL_MINOR_VERSION)/lib/intel64 -lifcoremt +#LIBS += -lifcoremt +endif + +ifneq ($(VERBOSE),) +CFLAGS += $(CFLAGS_VERBOSE) +FFLAGS += $(FFLAGS_VERBOSE) +LDFLAGS += $(LDFLAGS_VERBOSE) +endif + +# static linking of esmf works on darwin +#LIBS += -lFMS -lnetcdff -lnetcdf -llapack -lblas ${ESMF_DIR}/lib/libesmf.a -lstdc++ #-lc -lrt +LIBS += -lFMS -lnetcdff -lnetcdf -llapack -lblas -lesmf #-lc -lrt + +LDFLAGS += $(LIBS) diff --git a/nix/fv3/default.nix b/nix/fv3/default.nix new file mode 100644 index 000000000..d26e82181 --- /dev/null +++ b/nix/fv3/default.nix @@ -0,0 +1,87 @@ +{ + stdenv + , bash + , fms + , esmf + , nceplibs + , netcdf + , netcdffortran + , lapack + , blas + , mpich + , perl + , gfortran + , getopt +} : +stdenv.mkDerivation { + name = "fv3"; + buildInputs = [ + fms + esmf + nceplibs + netcdf + netcdffortran + lapack + blas + mpich + perl + gfortran + getopt + ]; + + srcs = [ + ../../FV3/. + ../../stochastic_physics/. + ]; + +# need some fancy logic to unpack the two source directories +# https://nixos.org/nixpkgs/manual/#sec-language-texlive-custom-packages +unpackPhase = '' + runHook preUnpack + + for _src in $srcs; do + cp -r "$_src" $(stripHash "$_src") + done + + chmod -R +w . + + runHook postUnpack +''; + +patchPhase = '' + +patchShebangs FV3/configure + # need to call this since usr/bin/env isn't + # installed in sandboxed build environment + # there goes 1 hr... + patchShebangs FV3/mkDepends.pl +''; + + +config = ./configure.fv3; + +configurePhase = '' + cd FV3 + cp $config conf/configure.fv3 + # ./configure gnu_docker + cd .. +''; + +buildPhase = '' + make -C FV3 -j 4 +''; + +installPhase = '' + PREFIX=$out make -C FV3 install -j 4 +''; + + + SHELL = "${bash}/bin/bash"; + FMS_DIR="${fms}/include"; + ESMF_DIR="${esmf}"; + LD_LIBRARY_PATH="${esmf}/lib/:${fms}/libFMS/.libs/:$${SERIALBOX_DIR}/lib"; + INCLUDE="-I${fms}/include -I${netcdffortran}/include -I${esmf}/include/"; + NCEPLIBS_DIR="${nceplibs}/lib"; + OMPI_CC="${gfortran.cc}/bin/gcc"; +} + diff --git a/nix/nceplibs/default.nix b/nix/nceplibs/default.nix new file mode 100644 index 000000000..ae62fee74 --- /dev/null +++ b/nix/nceplibs/default.nix @@ -0,0 +1,39 @@ +{ + stdenvNoCC + ,bash + ,fetchgit + ,system + ,rsync + ,gfortran + ,mpich + ,coreutils + ,perl +} : +stdenvNoCC.mkDerivation rec { + pname = "nceplibs"; + version = "0.0.0"; + + src = fetchgit { + url = "https://github.com/NCAR/NCEPlibs.git"; + rev = "3da51e139d5cd731c9fc27f39d88cb4e1328212b"; + sha256 = "03yjjz34452gbq292x24lvgxj4ybasm8z2aqlps0l6hg04blkhji"; + fetchSubmodules = false; + leaveDotGit = true; + }; + + buildPhase = '' + mkdir -p $out + echo "y" | bash make_ncep_libs.sh -s ${arch} -c gnu -d $out -o 1 + ''; + + dontInstall = true; + + # nativeBuildInputs = [ m4 ]; + # buildInputs = [ hdf5 curl mpi ]; + buildInputs = [ bash rsync gfortran mpich coreutils perl ]; + FFLAGS="-I${gfortran.libc}/include"; + arch = if system == "x86_64-darwin" then "macosx" else "linux"; + + + +} diff --git a/tests/pytest/conftest.py b/tests/pytest/conftest.py index 1675098ff..ca68c4c08 100644 --- a/tests/pytest/conftest.py +++ b/tests/pytest/conftest.py @@ -9,7 +9,7 @@ def pytest_addoption(parser): "--image_version", action="store", default="latest", help="The image version to run" ) parser.addoption( - "--image_runner", action="store", default="docker", choices=("docker", "sarus"), help="The image runner to use" + "--image_runner", action="store", default="docker", choices=("docker", "sarus", "native"), help="The image runner to use" ) parser.addoption( "--image", action="store", default=DEFAULT_MODEL_IMAGE, help="The image name to run, without tags" diff --git a/tests/pytest/test_regression.py b/tests/pytest/test_regression.py index 0120684f9..493e6cffa 100644 --- a/tests/pytest/test_regression.py +++ b/tests/pytest/test_regression.py @@ -110,6 +110,18 @@ def ensure_reference_exists(filename): ) +def run_model_native(rundir, n_processes): + if USE_LOCAL_ARCHIVE: + archive = fv3config.get_cache_dir() + archive_mount = ['-v', f'{archive}:{archive}'] + else: + archive_mount = [] + + rundir_abs = os.path.abspath(rundir) + flags = ["--oversubscribe"] + subprocess.check_call([ 'mpirun'] + flags + ['-np', str(n_processes), 'fv3.exe' ], cwd=rundir_abs) + + def run_model_docker(rundir, model_image, n_processes): if USE_LOCAL_ARCHIVE: archive = fv3config.get_cache_dir() @@ -179,6 +191,8 @@ def run_model(config, run_dir, model_image, image_runner): run_model_docker(run_dir, model_image, n_processes) elif image_runner == "sarus": run_model_sarus(run_dir, model_image, n_processes) + elif image_runner == "native": + run_model_native(run_dir, n_processes) else: raise NotImplementedError("image_runner must be one of 'docker' or 'sarus'") From b5f1bf48120df4b65408b4e9ec1cd25a20ffd9ad Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 17:48:16 -0700 Subject: [PATCH 10/54] Repair bug in CI --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fdc3c747..1e76e7d7e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: - restore_cache: keys: - key - - run: nix-build . -j 4 + - run: nix-build -j 4 nix: docker: - image: nixos/nix @@ -38,7 +38,7 @@ jobs: git submodule init git submodule update - run: | - nix-build nix -j 4 + nix-build -j 4 - run: name: Test command: nix-shell --run "pytest tests/pytest --capture=no --verbose --refdir /Users/noah/workspace/VulcanClimateModeling/fv3gfs-fortran/tests/pytest/reference/circleci --image_runner native" From 7367c7a4a6bab65a0bbbf1859b9ab613c590e126 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 18:06:21 -0700 Subject: [PATCH 11/54] Use cachix in circle build --- .circleci/config.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e76e7d7e..b104a630f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,17 +8,17 @@ jobs: xcode: 11.3.0 steps: - run: sudo sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume - - run: nix-env -i git openssh + - run: | + nix-env -i git openssh + nix-env -iA cachix -f https://cachix.org/api/v1/install + cachix use nbren12 - checkout - run: name: "Pull Submodules" command: | git submodule init git submodule update - - restore_cache: - keys: - - key - - run: nix-build -j 4 + - run: nix-build -j 4 | cachix push nix: docker: - image: nixos/nix @@ -37,8 +37,18 @@ jobs: command: | git submodule init git submodule update + - restore_cache: + keys: + - v0.1 - run: | - nix-build -j 4 + nix-env -iA cachix -f https://cachix.org/api/v1/install + cachix use nbren12 + nix-build -j 4 | cachix push + - save_cache: + paths: + - $FV3CONFIG_CACHE_DIR + - /opt/circleci/.pyenv/versions/3.6.2 + key: v1.4-{{ checksum "requirements.txt"}} - run: name: Test command: nix-shell --run "pytest tests/pytest --capture=no --verbose --refdir /Users/noah/workspace/VulcanClimateModeling/fv3gfs-fortran/tests/pytest/reference/circleci --image_runner native" From 5fd19bc0198a5da1aa0d245152ccf249c7d8519f Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 18:09:25 -0700 Subject: [PATCH 12/54] Push to specified cache --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b104a630f..20bd16c51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: command: | git submodule init git submodule update - - run: nix-build -j 4 | cachix push + - run: nix-build -j 4 | cachix push nbren12 nix: docker: - image: nixos/nix @@ -43,7 +43,7 @@ jobs: - run: | nix-env -iA cachix -f https://cachix.org/api/v1/install cachix use nbren12 - nix-build -j 4 | cachix push + nix-build -j 4 | cachix push nbren12 - save_cache: paths: - $FV3CONFIG_CACHE_DIR From 9efbbe136bf525cf1919cf7e072d7d54334335b8 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 18:59:22 -0700 Subject: [PATCH 13/54] Run tests in nix CircleCI builds --- .circleci/config.yml | 2 +- default.nix | 2 +- nix/fv3/default.nix | 4 ++++ tests.nix | 8 ++++++++ tests/pytest/test_regression.py | 9 +-------- tox.ini | 18 ++++++++++++++++++ 6 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 tests.nix create mode 100644 tox.ini diff --git a/.circleci/config.yml b/.circleci/config.yml index 20bd16c51..5207ac5da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,7 +51,7 @@ jobs: key: v1.4-{{ checksum "requirements.txt"}} - run: name: Test - command: nix-shell --run "pytest tests/pytest --capture=no --verbose --refdir /Users/noah/workspace/VulcanClimateModeling/fv3gfs-fortran/tests/pytest/reference/circleci --image_runner native" + command: nix-shell tests.nix --run tox build_default: machine: docker_layer_caching: true diff --git a/default.nix b/default.nix index 90b57caae..cfe9aef35 100644 --- a/default.nix +++ b/default.nix @@ -16,4 +16,4 @@ let # Hash obtained using `nix-prefetch-url --unpack ` sha256 = "1wg61h4gndm3vcprdcg7rc4s1v3jkm5xd7lw8r2f67w502y94gcy"; }) {overlays = [overlay];}; -in nixpkgs.fv3 \ No newline at end of file +in nixpkgs \ No newline at end of file diff --git a/nix/fv3/default.nix b/nix/fv3/default.nix index d26e82181..2671956d2 100644 --- a/nix/fv3/default.nix +++ b/nix/fv3/default.nix @@ -29,6 +29,10 @@ stdenv.mkDerivation { getopt ]; + propagatedBuildInputs = [ + mpich + ]; + srcs = [ ../../FV3/. ../../stochastic_physics/. diff --git a/tests.nix b/tests.nix new file mode 100644 index 000000000..54993d4d6 --- /dev/null +++ b/tests.nix @@ -0,0 +1,8 @@ +let + nixpkgs = import ./.; +in + with nixpkgs; + mkShell { + # nativeBuildInputs is usually what you want -- tools you need to run + nativeBuildInputs = [ fv3 python3Packages.tox ]; +} diff --git a/tests/pytest/test_regression.py b/tests/pytest/test_regression.py index 493e6cffa..59cf861c5 100644 --- a/tests/pytest/test_regression.py +++ b/tests/pytest/test_regression.py @@ -111,15 +111,8 @@ def ensure_reference_exists(filename): def run_model_native(rundir, n_processes): - if USE_LOCAL_ARCHIVE: - archive = fv3config.get_cache_dir() - archive_mount = ['-v', f'{archive}:{archive}'] - else: - archive_mount = [] - rundir_abs = os.path.abspath(rundir) - flags = ["--oversubscribe"] - subprocess.check_call([ 'mpirun'] + flags + ['-np', str(n_processes), 'fv3.exe' ], cwd=rundir_abs) + subprocess.check_call([ 'mpirun', '-np', str(n_processes), 'fv3.exe' ], cwd=rundir_abs) def run_model_docker(rundir, model_image, n_processes): diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..cdf45615a --- /dev/null +++ b/tox.ini @@ -0,0 +1,18 @@ +# tox (https://tox.readthedocs.io/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py3 +skipsdist = true + +[testenv] +allowlist_externals=make mpirun +passenv = * +deps = + # other versions of pytest don't work with subtests + -r requirements.txt +commands = + pytest tests/pytest/test_regression.py --image_runner=native + From c33d481ad38eb11ad95b0af2a10f6afb8af8ba9f Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 19:03:25 -0700 Subject: [PATCH 14/54] Fix nix-build calls --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5207ac5da..af364c411 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,8 @@ jobs: command: | git submodule init git submodule update - - run: nix-build -j 4 | cachix push nbren12 + - run: | + nix-build -j 4 -A fv3 | cachix push nbren12 nix: docker: - image: nixos/nix @@ -43,7 +44,7 @@ jobs: - run: | nix-env -iA cachix -f https://cachix.org/api/v1/install cachix use nbren12 - nix-build -j 4 | cachix push nbren12 + nix-build -j 4 -A fv3 | cachix push nbren12 - save_cache: paths: - $FV3CONFIG_CACHE_DIR From 21e6049f0c6b86baed6ba3d7bbf0294c1a031fbf Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 19:52:00 -0700 Subject: [PATCH 15/54] Make rudimentary test --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cdf45615a..12d874355 100644 --- a/tox.ini +++ b/tox.ini @@ -14,5 +14,6 @@ deps = # other versions of pytest don't work with subtests -r requirements.txt commands = - pytest tests/pytest/test_regression.py --image_runner=native + write_run_directory tests/pytest/config/default.yml rundir + bash -c 'cd rundir && mpirun -n 6 fv3.exe' From 59058661fa97b7e27af5678eb50cb307e34d7d6f Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 19:54:01 -0700 Subject: [PATCH 16/54] Attempt repair of nix-mac --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af364c411..588c233d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,9 @@ jobs: macos: xcode: 11.3.0 steps: - - run: sudo sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume + - run: | + curl -L -O https://nixos.org/nix/install + sudo sh install --darwin-use-unencrypted-nix-store-volume - run: | nix-env -i git openssh nix-env -iA cachix -f https://cachix.org/api/v1/install From f12f6d740dc3b4f7ef92f6595565e5e8ba6a7b68 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 19:55:29 -0700 Subject: [PATCH 17/54] Fix again --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 588c233d9..cefe53fb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: steps: - run: | curl -L -O https://nixos.org/nix/install - sudo sh install --darwin-use-unencrypted-nix-store-volume + sh install --darwin-use-unencrypted-nix-store-volume - run: | nix-env -i git openssh nix-env -iA cachix -f https://cachix.org/api/v1/install From af37ac05f561103adab138c66e41f70f78ee16a8 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 20:00:40 -0700 Subject: [PATCH 18/54] Delete install script --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cefe53fb5..1894b9637 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,7 @@ jobs: - run: | curl -L -O https://nixos.org/nix/install sh install --darwin-use-unencrypted-nix-store-volume + rm -f install - run: | nix-env -i git openssh nix-env -iA cachix -f https://cachix.org/api/v1/install From ed62097c8de3ea15a76f7529e4a90eb9a0483727 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 20:40:35 -0700 Subject: [PATCH 19/54] Revert dockerfile and test cahnge --- docker/Dockerfile | 2 -- tests/pytest/conftest.py | 2 +- tests/pytest/test_regression.py | 7 ------- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3af0e8fb7..91a49a1fc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -221,8 +221,6 @@ COPY --from=fv3gfs-sources FV3/atmos_cubed_sphere /FV3/atmos_cubed_sphere RUN cd /FV3/atmos_cubed_sphere && make clean && cd /FV3 && make -j16 -RUN PREFIX=/usr/local make -C /FV3/atmos_cubed_sphere install - ## Define final "user" image for running tests ##--------------------------------------------------------------------------------- FROM fv3gfs-build AS fv3gfs-compiled diff --git a/tests/pytest/conftest.py b/tests/pytest/conftest.py index ca68c4c08..1675098ff 100644 --- a/tests/pytest/conftest.py +++ b/tests/pytest/conftest.py @@ -9,7 +9,7 @@ def pytest_addoption(parser): "--image_version", action="store", default="latest", help="The image version to run" ) parser.addoption( - "--image_runner", action="store", default="docker", choices=("docker", "sarus", "native"), help="The image runner to use" + "--image_runner", action="store", default="docker", choices=("docker", "sarus"), help="The image runner to use" ) parser.addoption( "--image", action="store", default=DEFAULT_MODEL_IMAGE, help="The image name to run, without tags" diff --git a/tests/pytest/test_regression.py b/tests/pytest/test_regression.py index 59cf861c5..0120684f9 100644 --- a/tests/pytest/test_regression.py +++ b/tests/pytest/test_regression.py @@ -110,11 +110,6 @@ def ensure_reference_exists(filename): ) -def run_model_native(rundir, n_processes): - rundir_abs = os.path.abspath(rundir) - subprocess.check_call([ 'mpirun', '-np', str(n_processes), 'fv3.exe' ], cwd=rundir_abs) - - def run_model_docker(rundir, model_image, n_processes): if USE_LOCAL_ARCHIVE: archive = fv3config.get_cache_dir() @@ -184,8 +179,6 @@ def run_model(config, run_dir, model_image, image_runner): run_model_docker(run_dir, model_image, n_processes) elif image_runner == "sarus": run_model_sarus(run_dir, model_image, n_processes) - elif image_runner == "native": - run_model_native(run_dir, n_processes) else: raise NotImplementedError("image_runner must be one of 'docker' or 'sarus'") From 9425de043e5b265049452d7cddd72bbe5a51942c Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 22:09:31 -0700 Subject: [PATCH 20/54] Remove unused file --- FV3/conf/configure.fv3 | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/FV3/conf/configure.fv3 b/FV3/conf/configure.fv3 index 4f5dc3d32..b941f6b98 100644 --- a/FV3/conf/configure.fv3 +++ b/FV3/conf/configure.fv3 @@ -2,10 +2,10 @@ ############ # commands # ############ -FC = mpif90 -CC = mpicc +FC = FC=gfortran mpifort +CC = CC=gcc mpicc CXX = g++ -LD = mpif90 +LD = mpifort NETCDF_DIR = /usr @@ -17,7 +17,7 @@ DEBUG = REPRO = Y VERBOSE = OPENMP = Y -AVX2 = Y +AVX2 = N HYDRO = N 32BIT = N @@ -63,8 +63,8 @@ FFLAGS += -fdefault-double-8 -fdefault-real-8 endif ifeq ($(AVX2),Y) -FFLAGS += -mavx -CFLAGS += -mavx +FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align endif FFLAGS_OPT = -O2 @@ -84,13 +84,19 @@ CFLAGS_OPENMP = -fopenmp CFLAGS_DEBUG = -O0 -g CFLAGS_GCOV = --coverage -LDFLAGS := -L${ESMF_DIR}/lib -L${FMS_DIR}/libFMS/.libs/ +# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT +# *_TEST will match the production if no new option(s) is(are) to be tested. +FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits +CFLAGS_TEST = -O2 + +LDFLAGS := -L${ESMF_DIR}/lib/libO3/Linux.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ LDFLAGS_OPENMP := -fopenmp LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M LDFLAGS_GCOV = --coverage # start with blank LIBS LIBS := + LIBS += -lgfortran ifneq ($(REPRO),) @@ -121,6 +127,9 @@ ifneq ($(OPENMP),) CFLAGS += $(CFLAGS_OPENMP) FFLAGS += $(FFLAGS_OPENMP) LDFLAGS += $(LDFLAGS_OPENMP) +# to correct a loader bug on gaea: envars below set by module load intel +#LIBS += -L$(INTEL_PATH)/$(INTEL_MAJOR_VERSION)/$(INTEL_MINOR_VERSION)/lib/intel64 -lifcoremt +#LIBS += -lifcoremt endif ifneq ($(VERBOSE),) @@ -129,6 +138,6 @@ FFLAGS += $(FFLAGS_VERBOSE) LDFLAGS += $(LDFLAGS_VERBOSE) endif -LIBS += -lFMS -L/usr/local/esmf/lib -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt +LIBS += -lFMS -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt LDFLAGS += $(LIBS) From fd1fbce53d194caf0001efc3004a250e6396acc4 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 22:12:52 -0700 Subject: [PATCH 21/54] Cleanup unused code --- nix/esmf/default.nix | 25 ++----------------------- nix/fms/default.nix | 8 -------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/nix/esmf/default.nix b/nix/esmf/default.nix index adcdb341a..f33665c31 100644 --- a/nix/esmf/default.nix +++ b/nix/esmf/default.nix @@ -1,14 +1,3 @@ -with import {}; -# { stdenv -# , netcdffortran -# , perl -# , gfortran -# , lapack -# , blas -# , openmpi -# , fetchgit -# }: -# { stdenvNoCC ,netcdffortran @@ -17,6 +6,7 @@ with import {}; ,coreutils ,which ,llvmPackages + ,lib }: stdenvNoCC.mkDerivation rec { pname = "esmf"; @@ -63,22 +53,11 @@ stdenvNoCC.mkDerivation rec { # nativeBuildInputs = [ m4 ]; # buildInputs = [ hdf5 curl mpi ]; buildInputs = [ netcdffortran gfortran mpich gfortran.cc coreutils which - (lib.optional stdenv.isDarwin llvmPackages.openmp) + (lib.optional stdenvNoCC.isDarwin llvmPackages.openmp) ] ; inherit netcdffortran gfortran; CXX="${gfortran}/bin/g++"; CC="${gfortran}/bin/gcc"; ESMF_CXXCOMPILER="${CXX}"; ESMF_CCOMPILER="${CC}"; - - - - meta = { - description = "Libraries for the Unidata network Common Data Format"; - platforms = stdenv.lib.platforms.unix; - homepage = "https://www.unidata.ucar.edu/software/netcdf/"; - license = { - url = "https://www.unidata.ucar.edu/software/netcdf/docs/copyright.html"; - }; - }; } diff --git a/nix/fms/default.nix b/nix/fms/default.nix index 1f9d94898..e17f85b19 100644 --- a/nix/fms/default.nix +++ b/nix/fms/default.nix @@ -40,12 +40,4 @@ stdenv.mkDerivation rec { ./configure --prefix=$out ''; - meta = { - description = "Libraries for the Unidata network Common Data Format"; - platforms = stdenv.lib.platforms.unix; - homepage = "https://www.unidata.ucar.edu/software/netcdf/"; - license = { - url = "https://www.unidata.ucar.edu/software/netcdf/docs/copyright.html"; - }; - }; } From a15d2b89fef10592761628a0bf00c16b9b9f1062 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 22:39:55 -0700 Subject: [PATCH 22/54] Use vulcanclimatemodeling cachix the previous account had too personal of a name (nbren12). --- .circleci/config.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1894b9637..48af3d5f5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ jobs: - run: | nix-env -i git openssh nix-env -iA cachix -f https://cachix.org/api/v1/install - cachix use nbren12 + cachix use vulcanclimatemodeling - checkout - run: name: "Pull Submodules" @@ -22,7 +22,10 @@ jobs: git submodule init git submodule update - run: | - nix-build -j 4 -A fv3 | cachix push nbren12 + nix-build -j 4 -A fv3 | cachix push vulcanclimatemodeling + - run: + name: Test + command: nix-shell tests.nix --run tox nix: docker: - image: nixos/nix @@ -41,18 +44,10 @@ jobs: command: | git submodule init git submodule update - - restore_cache: - keys: - - v0.1 - run: | nix-env -iA cachix -f https://cachix.org/api/v1/install - cachix use nbren12 - nix-build -j 4 -A fv3 | cachix push nbren12 - - save_cache: - paths: - - $FV3CONFIG_CACHE_DIR - - /opt/circleci/.pyenv/versions/3.6.2 - key: v1.4-{{ checksum "requirements.txt"}} + cachix use vulcanclimatemodeling + nix-build -j 4 -A fv3 | cachix push vulcanclimatemodeling - run: name: Test command: nix-shell tests.nix --run tox From 49d737817d11e953e856932bca5f55211beb28e8 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 23:04:12 -0700 Subject: [PATCH 23/54] Google auth for mac builds --- .circleci/config.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 48af3d5f5..212665d40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,15 +6,19 @@ jobs: nix-mac: macos: xcode: 11.3.0 + environment: + FV3CONFIG_CACHE_DIR: /tmp/.fv3config + GOOGLE_APPLICATION_CREDENTIALS: /tmp/key.json steps: - run: | curl -L -O https://nixos.org/nix/install sh install --darwin-use-unencrypted-nix-store-volume rm -f install - run: | - nix-env -i git openssh + nix-env -i git openssh google-cloud-sdk nix-env -iA cachix -f https://cachix.org/api/v1/install cachix use vulcanclimatemodeling + echo $ENCODED_GCR_KEY | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - checkout - run: name: "Pull Submodules" From c5222a5d513bf9070fbc56615fdfa4de7cb0334c Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 16 Mar 2021 23:25:46 -0700 Subject: [PATCH 24/54] Delete unused file --- nix/esmf/build.sh | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 nix/esmf/build.sh diff --git a/nix/esmf/build.sh b/nix/esmf/build.sh deleted file mode 100644 index b0598b138..000000000 --- a/nix/esmf/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -source $stdenv/setup - - -# copy from read-only git repo -mkdir -p $out -cp -r $src src -chmod -R +w src -cd src - -# set build variables -export ESMF_DIR=$(pwd) -export ESMF_INSTALL_PREFIX=$out -export ESMF_NETCDF_INCLUDE=$netcdffortran/include -export ESMF_NETCDF_LIBS="-lnetcdf -lnetcdff" -export ESMF_BOPT=O3 -export ESMF_CXXCOMPILEOPTS="$ESMF_CXXCOMPILEOPTS -Wno-format-security" - -# compile -make lib -j8 -make install -make installcheck From 1b128838561b3c1a324456c234d9a9dfb74b9e42 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Wed, 17 Mar 2021 00:12:25 -0700 Subject: [PATCH 25/54] Document the nix workflow --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ shell.nix | 1 + 2 files changed, 49 insertions(+) create mode 100644 shell.nix diff --git a/README.md b/README.md index 20747a5c1..e3196c253 100644 --- a/README.md +++ b/README.md @@ -191,3 +191,51 @@ directory (it is `/FV3` by default) that you have bind-mounted in to the contain If you are using a version of docker that supports it, you can enable buildkit by setting `DOCKER_BUILDKIT=1` as an environment variable. This can be useful when building docker targets in this repo, because it will avoid building mulit-stage targets that are not required for the final image. + +# Nix + + + +FV3 can also be installed using the [nix](https://nixos.org/) package +manager. This package manager is available on Mac and Linux, and provides a +light weight means to distribute isolated software environments. + +## Installation + +To begin, install nix following [these instructions](https://nixos.org/download.html). + +We host binaries using a tool called cachix, and this will greatly speed up any builds. To use our binaries, [install cachix](https://github.com/cachix/cachix#installation) and then run + + cachix use vulcanclimatemodeling + +Without using the cachix cache, FV3 and all its dependencies will need to build from source (~20 minutes). + +## Developing + +To develop the model, you can use the environment specified in `shell.nix` by running + + nix-shell + +Then copy the nix build configuration file to the magic location harcoded in +the FV3 makefiles: + + cp -f conf/fv3/configure.fv3 FV3/conf/ + +And build the model + + cd FV3 + make + +## Running simple tests + +Now you can enter a shell with fv3 and all its dependencies installed by +running + + nix-shell tests.nix + +This will download all the dependencies from the internet, building any +uncached packages from scratch. + +Then, you can run a simple test by running + + tox \ No newline at end of file diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..4ea3465e8 --- /dev/null +++ b/shell.nix @@ -0,0 +1 @@ +(import ./default.nix).fv3 From df00733ca1fcb20d35180c72f6bcb7c4cac2272f Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Wed, 17 Mar 2021 00:14:35 -0700 Subject: [PATCH 26/54] More doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e3196c253..5462d412a 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ We host binaries using a tool called cachix, and this will greatly speed up any cachix use vulcanclimatemodeling -Without using the cachix cache, FV3 and all its dependencies will need to build from source (~20 minutes). +Without using the cachix cache, FV3 and all its dependencies will need to build from source (~20 minutes). This only happens once per machine, but it is slow. ## Developing From be810c4d3c01c4747ece16b19ec316bce7eff1d5 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Wed, 17 Mar 2021 00:15:47 -0700 Subject: [PATCH 27/54] Make it clear that cachix is optional --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5462d412a..df395507f 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ light weight means to distribute isolated software environments. To begin, install nix following [these instructions](https://nixos.org/download.html). -We host binaries using a tool called cachix, and this will greatly speed up any builds. To use our binaries, [install cachix](https://github.com/cachix/cachix#installation) and then run +(optional) We host binaries using a tool called cachix, and this will greatly speed up any builds. To use our binaries, [install cachix](https://github.com/cachix/cachix#installation) and then run cachix use vulcanclimatemodeling From c709d7f43ea4bd02499fa917d11154af9c195c2e Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Wed, 17 Mar 2021 00:18:16 -0700 Subject: [PATCH 28/54] Better docs --- README.md | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index df395507f..a51f37ff9 100644 --- a/README.md +++ b/README.md @@ -208,8 +208,26 @@ To begin, install nix following [these instructions](https://nixos.org/download. cachix use vulcanclimatemodeling +Finally, you can build the model like this + + nix-build -A fv3 + Without using the cachix cache, FV3 and all its dependencies will need to build from source (~20 minutes). This only happens once per machine, but it is slow. +## Running simple tests + +Now you can enter a shell with fv3 and all its dependencies installed by +running + + nix-shell tests.nix + +This will download all the dependencies from the internet, building any +uncached packages from scratch. + +Then, you can run a simple test by running + + tox + ## Developing To develop the model, you can use the environment specified in `shell.nix` by running @@ -224,18 +242,4 @@ the FV3 makefiles: And build the model cd FV3 - make - -## Running simple tests - -Now you can enter a shell with fv3 and all its dependencies installed by -running - - nix-shell tests.nix - -This will download all the dependencies from the internet, building any -uncached packages from scratch. - -Then, you can run a simple test by running - - tox \ No newline at end of file + make \ No newline at end of file From f4a6a9de9dd161fc40df0c052abc83d4c5259e64 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Wed, 17 Mar 2021 09:06:39 -0700 Subject: [PATCH 29/54] Comment on tox test --- FV3/conf/configure.fv3 | 143 ----------------------------------------- tox.ini | 8 +-- 2 files changed, 3 insertions(+), 148 deletions(-) delete mode 100644 FV3/conf/configure.fv3 diff --git a/FV3/conf/configure.fv3 b/FV3/conf/configure.fv3 deleted file mode 100644 index b941f6b98..000000000 --- a/FV3/conf/configure.fv3 +++ /dev/null @@ -1,143 +0,0 @@ - -############ -# commands # -############ -FC = FC=gfortran mpifort -CC = CC=gcc mpicc -CXX = g++ -LD = mpifort - -NETCDF_DIR = /usr - -######### -# flags # -######### -# default is 64-bit OpenMP non-hydrostatic build -DEBUG = -REPRO = Y -VERBOSE = -OPENMP = Y -AVX2 = N -HYDRO = N -32BIT = N - -NEMSIOINC = -I$(NEMSIO_INC) - -NCEPLIBS_DIR ?= /opt/NCEPlibs/lib -NCEPLIBS = -L$(NCEPLIBS_DIR) -lnemsio_d -lbacio_4 -lsp_v2.0.2_d -lw3emc_d -lw3nco_d - -############################################## -# Need to use at least GNU Make version 3.81 # -############################################## -need := 3.81 -ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need)))) -ifneq ($(need),$(ok)) -$(error Need at least make version $(need). Load module gmake/3.81) -endif - -NETCDF_ROOT = $(NETCDF_DIR) -INCLUDE = -I$(NETCDF_ROOT)/include - -FPPFLAGS := -cpp -Wp,-w $(INCLUDE) -fPIC -CFLAGS := $(INCLUDE) -fPIC - -FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fPIC - -CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -Duse_LARGEFILE -DUSE_GFSL63 -DGFS_PHYS -DNO_INLINE_POST -CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML - -ifeq ($(GT4PY_DEV),Y) -CPPDEFS += -DGT4PY_DEV -endif - -ifeq ($(HYDRO),Y) -CPPDEFS += -else -CPPDEFS += -DMOIST_CAPPA -DUSE_COND -endif - -ifeq ($(32BIT),Y) -CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8 -else -FFLAGS += -fdefault-double-8 -fdefault-real-8 -endif - -ifeq ($(AVX2),Y) -FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align -CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align -endif - -FFLAGS_OPT = -O2 -FFLAGS_REPRO = -O2 -g -fbacktrace -FFLAGS_DEBUG = -O0 -g -fbacktrace -fno-fast-math -ffree-line-length-none -fno-backslash -pedantic -Waliasing -Wampersand -Wline-truncation -Wsurprising -Wtabs -Wunderflow -fdump-core -ffpe-trap=invalid,zero,overflow -fbounds-check -finit-real=nan -finit-integer=9999999 -finit-logical=true -finit-character=35 -FFLAGS_GCOV = --coverage - -TRANSCENDENTALS := -fast-transcendentals -FFLAGS_OPENMP = -fopenmp -FFLAGS_VERBOSE = -v -V -what - -CFLAGS += -D__IFC - -CFLAGS_OPT = -O2 -CFLAGS_REPRO = -O2 -CFLAGS_OPENMP = -fopenmp -CFLAGS_DEBUG = -O0 -g -CFLAGS_GCOV = --coverage - -# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT -# *_TEST will match the production if no new option(s) is(are) to be tested. -FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits -CFLAGS_TEST = -O2 - -LDFLAGS := -L${ESMF_DIR}/lib/libO3/Linux.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ -LDFLAGS_OPENMP := -fopenmp -LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M -LDFLAGS_GCOV = --coverage - -# start with blank LIBS -LIBS := - -LIBS += -lgfortran - -ifneq ($(REPRO),) -CFLAGS += $(CFLAGS_REPRO) -FFLAGS += $(FFLAGS_REPRO) -FAST := -else ifneq ($(DEBUG),) -CFLAGS += $(CFLAGS_DEBUG) -FFLAGS += $(FFLAGS_DEBUG) -FAST := -else ifneq ($(TEST),) -CFLAGS += $(CFLAGS_TEST) -FFLAGS += $(FFLAGS_TEST) -FAST := -else -CFLAGS += $(CFLAGS_OPT) -FFLAGS += $(FFLAGS_OPT) -FAST := $(TRANSCENDENTALS) -endif - -ifneq ($(GCOV),) -CFLAGS += $(CFLAGS_GCOV) -FFLAGS += $(FFLAGS_GCOV) -LDFLAGS += $(LDFLAGS_GCOV) -endif - -ifneq ($(OPENMP),) -CFLAGS += $(CFLAGS_OPENMP) -FFLAGS += $(FFLAGS_OPENMP) -LDFLAGS += $(LDFLAGS_OPENMP) -# to correct a loader bug on gaea: envars below set by module load intel -#LIBS += -L$(INTEL_PATH)/$(INTEL_MAJOR_VERSION)/$(INTEL_MINOR_VERSION)/lib/intel64 -lifcoremt -#LIBS += -lifcoremt -endif - -ifneq ($(VERBOSE),) -CFLAGS += $(CFLAGS_VERBOSE) -FFLAGS += $(FFLAGS_VERBOSE) -LDFLAGS += $(LDFLAGS_VERBOSE) -endif - -LIBS += -lFMS -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt - -LDFLAGS += $(LIBS) diff --git a/tox.ini b/tox.ini index 12d874355..289aafd4d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,5 @@ # tox (https://tox.readthedocs.io/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - +# in multiple virtualenvs. [tox] envlist = py3 skipsdist = true @@ -11,8 +8,9 @@ skipsdist = true allowlist_externals=make mpirun passenv = * deps = - # other versions of pytest don't work with subtests -r requirements.txt +# Just a simple check with no validation. Migrating the full test suite +# would be nice in the future. commands = write_run_directory tests/pytest/config/default.yml rundir bash -c 'cd rundir && mpirun -n 6 fv3.exe' From 61a260eba88815ff43416e0ec3dd9c1070da981d Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Wed, 17 Mar 2021 09:12:40 -0700 Subject: [PATCH 30/54] Revert change to configure.fv3 --- FV3/conf/configure.fv3 | 143 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 FV3/conf/configure.fv3 diff --git a/FV3/conf/configure.fv3 b/FV3/conf/configure.fv3 new file mode 100644 index 000000000..b941f6b98 --- /dev/null +++ b/FV3/conf/configure.fv3 @@ -0,0 +1,143 @@ + +############ +# commands # +############ +FC = FC=gfortran mpifort +CC = CC=gcc mpicc +CXX = g++ +LD = mpifort + +NETCDF_DIR = /usr + +######### +# flags # +######### +# default is 64-bit OpenMP non-hydrostatic build +DEBUG = +REPRO = Y +VERBOSE = +OPENMP = Y +AVX2 = N +HYDRO = N +32BIT = N + +NEMSIOINC = -I$(NEMSIO_INC) + +NCEPLIBS_DIR ?= /opt/NCEPlibs/lib +NCEPLIBS = -L$(NCEPLIBS_DIR) -lnemsio_d -lbacio_4 -lsp_v2.0.2_d -lw3emc_d -lw3nco_d + +############################################## +# Need to use at least GNU Make version 3.81 # +############################################## +need := 3.81 +ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need)))) +ifneq ($(need),$(ok)) +$(error Need at least make version $(need). Load module gmake/3.81) +endif + +NETCDF_ROOT = $(NETCDF_DIR) +INCLUDE = -I$(NETCDF_ROOT)/include + +FPPFLAGS := -cpp -Wp,-w $(INCLUDE) -fPIC +CFLAGS := $(INCLUDE) -fPIC + +FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fPIC + +CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -Duse_LARGEFILE -DUSE_GFSL63 -DGFS_PHYS -DNO_INLINE_POST +CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML + +ifeq ($(GT4PY_DEV),Y) +CPPDEFS += -DGT4PY_DEV +endif + +ifeq ($(HYDRO),Y) +CPPDEFS += +else +CPPDEFS += -DMOIST_CAPPA -DUSE_COND +endif + +ifeq ($(32BIT),Y) +CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8 +else +FFLAGS += -fdefault-double-8 -fdefault-real-8 +endif + +ifeq ($(AVX2),Y) +FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align +endif + +FFLAGS_OPT = -O2 +FFLAGS_REPRO = -O2 -g -fbacktrace +FFLAGS_DEBUG = -O0 -g -fbacktrace -fno-fast-math -ffree-line-length-none -fno-backslash -pedantic -Waliasing -Wampersand -Wline-truncation -Wsurprising -Wtabs -Wunderflow -fdump-core -ffpe-trap=invalid,zero,overflow -fbounds-check -finit-real=nan -finit-integer=9999999 -finit-logical=true -finit-character=35 +FFLAGS_GCOV = --coverage + +TRANSCENDENTALS := -fast-transcendentals +FFLAGS_OPENMP = -fopenmp +FFLAGS_VERBOSE = -v -V -what + +CFLAGS += -D__IFC + +CFLAGS_OPT = -O2 +CFLAGS_REPRO = -O2 +CFLAGS_OPENMP = -fopenmp +CFLAGS_DEBUG = -O0 -g +CFLAGS_GCOV = --coverage + +# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT +# *_TEST will match the production if no new option(s) is(are) to be tested. +FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits +CFLAGS_TEST = -O2 + +LDFLAGS := -L${ESMF_DIR}/lib/libO3/Linux.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ +LDFLAGS_OPENMP := -fopenmp +LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M +LDFLAGS_GCOV = --coverage + +# start with blank LIBS +LIBS := + +LIBS += -lgfortran + +ifneq ($(REPRO),) +CFLAGS += $(CFLAGS_REPRO) +FFLAGS += $(FFLAGS_REPRO) +FAST := +else ifneq ($(DEBUG),) +CFLAGS += $(CFLAGS_DEBUG) +FFLAGS += $(FFLAGS_DEBUG) +FAST := +else ifneq ($(TEST),) +CFLAGS += $(CFLAGS_TEST) +FFLAGS += $(FFLAGS_TEST) +FAST := +else +CFLAGS += $(CFLAGS_OPT) +FFLAGS += $(FFLAGS_OPT) +FAST := $(TRANSCENDENTALS) +endif + +ifneq ($(GCOV),) +CFLAGS += $(CFLAGS_GCOV) +FFLAGS += $(FFLAGS_GCOV) +LDFLAGS += $(LDFLAGS_GCOV) +endif + +ifneq ($(OPENMP),) +CFLAGS += $(CFLAGS_OPENMP) +FFLAGS += $(FFLAGS_OPENMP) +LDFLAGS += $(LDFLAGS_OPENMP) +# to correct a loader bug on gaea: envars below set by module load intel +#LIBS += -L$(INTEL_PATH)/$(INTEL_MAJOR_VERSION)/$(INTEL_MINOR_VERSION)/lib/intel64 -lifcoremt +#LIBS += -lifcoremt +endif + +ifneq ($(VERBOSE),) +CFLAGS += $(CFLAGS_VERBOSE) +FFLAGS += $(FFLAGS_VERBOSE) +LDFLAGS += $(LDFLAGS_VERBOSE) +endif + +LIBS += -lFMS -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt + +LDFLAGS += $(LIBS) From 75183181a44603be5e1abdb9d3453476ca17dbe6 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Thu, 18 Mar 2021 20:28:32 +0000 Subject: [PATCH 31/54] Revert "Change auto-conversion temperature threshold" This reverts commit f038d356cbf439cabaf66174173174e44897463c. This had a minimal effect --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 9bb567ce9..f92854ca3 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -1275,7 +1275,7 @@ subroutine warm_rain (dt, ktop, kbot, dp, dz, tz, qv, ql, qr, qi, qs, qg, & do k = ktop, kbot qc0 = fac_rc * ccn (k) - if (tz (k) > t_wfr) then + if (tz (k) > t_wfr + dt_fr) then dl (k) = min (max (1.e-6, dl (k)), 0.5 * ql (k)) ! -------------------------------------------------------------------- ! as in klein's gfdl am2 stratiform scheme (with subgrid variations) From 8d097eeeb3bfe6275b5b533eff3e3616f7bc5500 Mon Sep 17 00:00:00 2001 From: linjiongzhou Date: Thu, 18 Mar 2021 11:53:41 -0400 Subject: [PATCH 32/54] Revise the cloud ice deposition and sublimation process. --- .../physics/gfdl_cloud_microphys.F90 | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index f92854ca3..0e7cca097 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -215,14 +215,6 @@ module gfdl_cloud_microphys_mod real :: rthresh = 10.0e-6 !< critical cloud drop radius (micro m) - ! ----------------------------------------------------------------------- - ! wrf / wsm6 scheme: qi_gen = 4.92e-11 * (1.e3 * exp (0.1 * tmp)) ** 1.33 - ! optimized: qi_gen = 4.92e-11 * exp (1.33 * log (1.e3 * exp (0.1 * tmp))) - ! qi_gen ~ 4.808e-7 at 0 c; 1.818e-6 at - 10 c, 9.82679e-5 at - 40c - ! the following value is constructed such that qc_crt = 0 at zero c and @ - 10c matches - ! wrf / wsm6 ice initiation scheme; qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den - ! ----------------------------------------------------------------------- - real :: sat_adj0 = 0.90 !< adjustment factor (0: no, 1: full) during fast_sat_adj real :: qc_crt = 5.0e-8 !< mini condensate mixing ratio to allow partial cloudiness @@ -233,7 +225,6 @@ module gfdl_cloud_microphys_mod real :: qs_mlt = 1.0e-6 !< max cloud water due to snow melt real :: ql_gen = 1.0e-3 !< max cloud water generation during remapping step if fast_sat_adj = .t. - real :: qi_gen = 1.82e-6 !< max cloud ice generation during remapping step ! cloud condensate upper bounds: "safety valves" for ql & qi @@ -291,6 +282,9 @@ module gfdl_cloud_microphys_mod ! real :: global_area = - 1. real :: log_10, tice0, t_wfr + + integer :: inflag = 1 ! ice nucleation scheme + integer :: igflag = 3 ! ice generation scheme integer :: reiflag = 1 ! 1: Heymsfield and Mcfarquhar, 1996 @@ -311,7 +305,7 @@ module gfdl_cloud_microphys_mod namelist / gfdl_cloud_microphysics_nml / & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & vi_fac, vr_fac, vs_fac, vg_fac, ql_mlt, do_qa, fix_negative, vi_max, & - vs_max, vg_max, vr_max, qs_mlt, qs0_crt, qi_gen, ql0_max, qi0_max, & + vs_max, vg_max, vr_max, qs_mlt, qs0_crt, ql0_max, qi0_max, & qi0_crt, qr0_crt, fast_sat_adj, rh_inc, rh_ins, rh_inr, const_vi, & const_vs, const_vg, const_vr, use_ccn, rthresh, ccn_l, ccn_o, qc_crt, & tau_g2v, tau_v2g, sat_adj0, c_piacr, tau_imlt, tau_v2l, tau_l2v, & @@ -320,12 +314,12 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs + resmin, resmax, regmin, regmax, tintqs, inflag, igflag public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & vi_fac, vr_fac, vs_fac, vg_fac, ql_mlt, do_qa, fix_negative, vi_max, & - vs_max, vg_max, vr_max, qs_mlt, qs0_crt, qi_gen, ql0_max, qi0_max, & + vs_max, vg_max, vr_max, qs_mlt, qs0_crt, ql0_max, qi0_max, & qi0_crt, qr0_crt, fast_sat_adj, rh_inc, rh_ins, rh_inr, const_vi, & const_vs, const_vg, const_vr, use_ccn, rthresh, ccn_l, ccn_o, qc_crt, & tau_g2v, tau_v2g, sat_adj0, c_piacr, tau_imlt, tau_v2l, tau_l2v, & @@ -334,7 +328,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs + resmin, resmax, regmin, regmax, tintqs, inflag, igflag contains @@ -1562,7 +1556,7 @@ subroutine icloud (ktop, kbot, tzk, p1, qvk, qlk, qrk, qik, qsk, qgk, dp1, & dtmp = t_wfr - tzk (k) factor = min (1., dtmp / dt_fr) sink = min (qlk (k) * factor, dtmp / icpk (k)) - qi_crt = qi_gen * min (qi_lim, 0.1 * (tice - tzk (k))) / den (k) + qi_crt = 1.82e-6 * min (qi_lim, 0.1 * (tice - tzk (k))) / den (k) tmp = min (sink, dim (qi_crt, qik (k))) qlk (k) = qlk (k) - sink qsk (k) = qsk (k) + sink - tmp @@ -1977,7 +1971,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & real :: fac_v2l, fac_l2v - real :: pidep, qi_crt + real :: pidep, qi_crt, cin, qi_gen ! ----------------------------------------------------------------------- ! qstar over water may be accurate only down to - 80 deg c with ~10% uncertainty @@ -2167,18 +2161,41 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & dq = qv (k) - qsi sink = dq / (1. + tcpk (k) * dqsdt) if (qi (k) > qrmin) then - ! eq 9, hong et al. 2004, mwr - ! for a and b, see dudhia 1989: page 3103 eq (b7) and (b8) - pidep = dt_pisub * dq * 349138.78 * exp (0.875 * log (qi (k) * den (k))) & + if (inflag .eq. 1) & + ! Hong et al., 2004 + cin = 5.38e7 * exp (0.75 * log (qi (k) * den (k))) + if (inflag .eq. 2) & + ! Meyers et al., 1992 + cin = exp (-2.80 + 0.262 * (tice - tz (k))) * 1000.0 + if (inflag .eq. 3) & + ! Meyers et al., 1992 + cin = exp (-0.639 + 12.96 * (qv (k) / qsi - 1.0)) * 1000.0 + if (inflag .eq. 4) & + ! Cooper, 1986 + cin = 5.e-3 * exp (0.304 * (tice - tz (k))) * 1000.0 + if (inflag .eq. 5) & + ! Flecther, 1962 + cin = 1.e-5 * exp (0.5 * (tice - tz (k))) * 1000.0 + pidep = dt_pisub * dq * 4.0 * 11.9 * exp (0.5 * log (qi (k) * den (k) * cin)) & / (qsi * den (k) * lat2 / (0.0243 * rvgas * tz (k) ** 2) + 4.42478e4) else pidep = 0. endif if (dq > 0.) then ! vapor - > ice tmp = tice - tz (k) - ! 20160912: the following should produce more ice at higher altitude - ! qi_crt = 4.92e-11 * exp (1.33 * log (1.e3 * exp (0.1 * tmp))) / den (k) - qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den (k) + qi_gen = 4.92e-11 * exp (1.33 * log (1.e3 * exp (0.1 * tmp))) + if (igflag .eq. 1) & + ! WSM6 + qi_crt = qi_gen / den (k) + if (igflag .eq. 2) & + ! WSM6 with 0 at 0 C + qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den (k) + if (igflag .eq. 3) & + ! WSM6 with 0 at 0 C and fixed value at -10 C + qi_crt = 1.82e-6 * min (qi_lim, 0.1 * tmp) / den (k) + if (igflag .eq. 4) & + ! combination of 1 and 3 + qi_crt = max (qi_gen, 1.82e-6) * min (qi_lim, 0.1 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor pidep = pidep * min (1., dim (tz (k), t_sub) * 0.2) From 7e613f2e6b3879bd689a3c76632e2056ea234fa1 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Thu, 18 Mar 2021 20:39:57 +0000 Subject: [PATCH 33/54] fv_cmp.f90: workaround reference to qi_gen Linjiongs changes made this a local variable in the gfdl microphysics suite. replace this reference with a local variable in fv_cmp.f90 WARNING only valid if fv_cmp.f90 is never used (e.g. do_sat_adj = false) --- FV3/atmos_cubed_sphere/model/fv_cmp.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 index e2f281cfa..cf835922b 100644 --- a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 +++ b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 @@ -57,7 +57,7 @@ module fv_cmp_mod use constants_mod, only: rvgas, rdgas, grav, hlv, hlf, cp_air use fv_mp_mod, only: is_master use fv_arrays_mod, only: r_grid - use gfdl_cloud_microphys_mod, only: ql_gen, qi_gen, qi0_max, ql_mlt, ql0_max, qi_lim, qs_mlt + use gfdl_cloud_microphys_mod, only: ql_gen, qi0_max, ql_mlt, ql0_max, qi_lim, qs_mlt use gfdl_cloud_microphys_mod, only: icloud_f, sat_adj0, t_sub, cld_min use gfdl_cloud_microphys_mod, only: tau_r2g, tau_smlt, tau_i2s, tau_v2l, tau_l2v, tau_imlt, tau_l2r use gfdl_cloud_microphys_mod, only: rad_rain, rad_snow, rad_graupel, dw_ocean, dw_land, tintqs @@ -109,6 +109,7 @@ module fv_cmp_mod real :: d0_vap !< the same as dc_vap, except that cp_vap can be cp_vap or cv_vap real :: lv00 !< the same as lv0, except that cp_vap can be cp_vap or cv_vap + real :: qi_gen ! TODO this is no longer provided in Linjiongs fork. real, allocatable :: table (:), table2 (:), tablew (:), des2 (:), desw (:) From d8a36b3c7d133312515d0b453595f5245b1f8207 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Thu, 18 Mar 2021 21:55:29 +0000 Subject: [PATCH 34/54] microphysics: enhance effect of qi_lim --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 0e7cca097..b93e43063 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2189,13 +2189,13 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & qi_crt = qi_gen / den (k) if (igflag .eq. 2) & ! WSM6 with 0 at 0 C - qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den (k) + qi_crt = qi_gen * min (qi_lim, 1.0 * tmp) / den (k) if (igflag .eq. 3) & ! WSM6 with 0 at 0 C and fixed value at -10 C - qi_crt = 1.82e-6 * min (qi_lim, 0.1 * tmp) / den (k) + qi_crt = 1.82e-6 * min (qi_lim, 1.0 * tmp) / den (k) if (igflag .eq. 4) & ! combination of 1 and 3 - qi_crt = max (qi_gen, 1.82e-6) * min (qi_lim, 0.1 * tmp) / den (k) + qi_crt = max (qi_gen, 1.82e-6) * min (qi_lim, 1.0 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor pidep = pidep * min (1., dim (tz (k), t_sub) * 0.2) From 9abeb3928985040d3a28e2965ed5162ace892a99 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 19 Mar 2021 10:43:23 -0700 Subject: [PATCH 35/54] nix: big sur mac builds nixpkgs 20.09 doesn't support big sur. Use an older commit from nixpkgs-unstable instead. --- default.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/default.nix b/default.nix index cfe9aef35..58d43c7fe 100644 --- a/default.nix +++ b/default.nix @@ -10,10 +10,11 @@ let nixpkgs = import (builtins.fetchTarball { # Descriptive name to make the store path easier to identify - name = "release-20.09"; - # Commit hash for nixos-unstable as of 2018-09-12 - url = "https://github.com/nixos/nixpkgs/archive/20.09.tar.gz"; + name = "nixos-unstable-as-of-2021-19"; + # this commit has caches on cache.nixos.org, but doesn't update the mpich + # version from 20.09 + url = "https://github.com/nixos/nixpkgs/archive/7750e6a2c95dd157d4f75a6af00923910870dd5e.tar.gz"; # Hash obtained using `nix-prefetch-url --unpack ` - sha256 = "1wg61h4gndm3vcprdcg7rc4s1v3jkm5xd7lw8r2f67w502y94gcy"; + sha256 = "0mw0w4mk65a6k2v6mdwa5id5rq01sjbx1klcmri9m7i77q7mzd5x"; }) {overlays = [overlay];}; -in nixpkgs \ No newline at end of file +in nixpkgs From c626c0b9ffbfe30dfff3caed8e2dadc461fd7d4d Mon Sep 17 00:00:00 2001 From: Noah D Brenowitz Date: Fri, 19 Mar 2021 13:55:34 -0700 Subject: [PATCH 36/54] Update README.md Co-authored-by: Spencer Clark --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a51f37ff9..716511250 100644 --- a/README.md +++ b/README.md @@ -237,9 +237,9 @@ To develop the model, you can use the environment specified in `shell.nix` by ru Then copy the nix build configuration file to the magic location harcoded in the FV3 makefiles: - cp -f conf/fv3/configure.fv3 FV3/conf/ + cp -f nix/fv3/configure.fv3 FV3/conf/ And build the model cd FV3 - make \ No newline at end of file + make From 7093a566bb944dd493d872cf32b817cff21c1b70 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 19 Mar 2021 14:24:41 -0700 Subject: [PATCH 37/54] microphysics: reduce cloud ice sublimation --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index b93e43063..2cc0ce7f5 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -2198,7 +2198,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & qi_crt = max (qi_gen, 1.82e-6) * min (qi_lim, 1.0 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor - pidep = pidep * min (1., dim (tz (k), t_sub) * 0.2) + pidep = pidep * min (1., dim (tz (k), t_sub) / (tice - t_sub)) sink = max (pidep, sink, - qi (k)) endif qv (k) = qv (k) - sink From 50eaf72b0665d02278854075cbd00943b18e18b4 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 19 Mar 2021 15:45:02 -0700 Subject: [PATCH 38/54] cloud-ice: new tuning parameter --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 2cc0ce7f5..95f3b9df0 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -298,6 +298,10 @@ module gfdl_cloud_microphys_mod real :: resmin = 150.0, resmax = 10000.0 real :: regmin = 300.0, regmax = 10000.0 + ! Linjiong-Noah custom tunining parameters + ! TODO give these better names + real :: p20210319 = 1.0 + ! ----------------------------------------------------------------------- ! namelist ! ----------------------------------------------------------------------- @@ -314,7 +318,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, inflag, igflag + resmin, resmax, regmin, regmax, tintqs, inflag, igflag, p20210319 public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & @@ -328,7 +332,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, inflag, igflag + resmin, resmax, regmin, regmax, tintqs, inflag, igflag, p20210319 contains @@ -2198,7 +2202,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & qi_crt = max (qi_gen, 1.82e-6) * min (qi_lim, 1.0 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor - pidep = pidep * min (1., dim (tz (k), t_sub) / (tice - t_sub)) + pidep = pidep * min (1., dim (tz (k), t_sub) / (tice - t_sub) * p20210319) sink = max (pidep, sink, - qi (k)) endif qv (k) = qv (k) - sink From 297922e6b57d6b995f4e347d57f16178b75bc21e Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 30 Mar 2021 10:07:03 -0700 Subject: [PATCH 39/54] Revert "Revise the cloud ice deposition and sublimation process." This reverts commit 8d097eeeb3bfe6275b5b533eff3e3616f7bc5500. --- .../physics/gfdl_cloud_microphys.F90 | 59 +++++++------------ 1 file changed, 21 insertions(+), 38 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 95f3b9df0..d336eb1bd 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -215,6 +215,14 @@ module gfdl_cloud_microphys_mod real :: rthresh = 10.0e-6 !< critical cloud drop radius (micro m) + ! ----------------------------------------------------------------------- + ! wrf / wsm6 scheme: qi_gen = 4.92e-11 * (1.e3 * exp (0.1 * tmp)) ** 1.33 + ! optimized: qi_gen = 4.92e-11 * exp (1.33 * log (1.e3 * exp (0.1 * tmp))) + ! qi_gen ~ 4.808e-7 at 0 c; 1.818e-6 at - 10 c, 9.82679e-5 at - 40c + ! the following value is constructed such that qc_crt = 0 at zero c and @ - 10c matches + ! wrf / wsm6 ice initiation scheme; qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den + ! ----------------------------------------------------------------------- + real :: sat_adj0 = 0.90 !< adjustment factor (0: no, 1: full) during fast_sat_adj real :: qc_crt = 5.0e-8 !< mini condensate mixing ratio to allow partial cloudiness @@ -225,6 +233,7 @@ module gfdl_cloud_microphys_mod real :: qs_mlt = 1.0e-6 !< max cloud water due to snow melt real :: ql_gen = 1.0e-3 !< max cloud water generation during remapping step if fast_sat_adj = .t. + real :: qi_gen = 1.82e-6 !< max cloud ice generation during remapping step ! cloud condensate upper bounds: "safety valves" for ql & qi @@ -282,9 +291,6 @@ module gfdl_cloud_microphys_mod ! real :: global_area = - 1. real :: log_10, tice0, t_wfr - - integer :: inflag = 1 ! ice nucleation scheme - integer :: igflag = 3 ! ice generation scheme integer :: reiflag = 1 ! 1: Heymsfield and Mcfarquhar, 1996 @@ -309,7 +315,7 @@ module gfdl_cloud_microphys_mod namelist / gfdl_cloud_microphysics_nml / & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & vi_fac, vr_fac, vs_fac, vg_fac, ql_mlt, do_qa, fix_negative, vi_max, & - vs_max, vg_max, vr_max, qs_mlt, qs0_crt, ql0_max, qi0_max, & + vs_max, vg_max, vr_max, qs_mlt, qs0_crt, qi_gen, ql0_max, qi0_max, & qi0_crt, qr0_crt, fast_sat_adj, rh_inc, rh_ins, rh_inr, const_vi, & const_vs, const_vg, const_vr, use_ccn, rthresh, ccn_l, ccn_o, qc_crt, & tau_g2v, tau_v2g, sat_adj0, c_piacr, tau_imlt, tau_v2l, tau_l2v, & @@ -318,12 +324,12 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, inflag, igflag, p20210319 + resmin, resmax, regmin, regmax, tintqs, p20210319 public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & vi_fac, vr_fac, vs_fac, vg_fac, ql_mlt, do_qa, fix_negative, vi_max, & - vs_max, vg_max, vr_max, qs_mlt, qs0_crt, ql0_max, qi0_max, & + vs_max, vg_max, vr_max, qs_mlt, qs0_crt, qi_gen, ql0_max, qi0_max, & qi0_crt, qr0_crt, fast_sat_adj, rh_inc, rh_ins, rh_inr, const_vi, & const_vs, const_vg, const_vr, use_ccn, rthresh, ccn_l, ccn_o, qc_crt, & tau_g2v, tau_v2g, sat_adj0, c_piacr, tau_imlt, tau_v2l, tau_l2v, & @@ -332,7 +338,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, inflag, igflag, p20210319 + resmin, resmax, regmin, regmax, tintqs, p20210319 contains @@ -1560,7 +1566,7 @@ subroutine icloud (ktop, kbot, tzk, p1, qvk, qlk, qrk, qik, qsk, qgk, dp1, & dtmp = t_wfr - tzk (k) factor = min (1., dtmp / dt_fr) sink = min (qlk (k) * factor, dtmp / icpk (k)) - qi_crt = 1.82e-6 * min (qi_lim, 0.1 * (tice - tzk (k))) / den (k) + qi_crt = qi_gen * min (qi_lim, 0.1 * (tice - tzk (k))) / den (k) tmp = min (sink, dim (qi_crt, qik (k))) qlk (k) = qlk (k) - sink qsk (k) = qsk (k) + sink - tmp @@ -1975,7 +1981,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & real :: fac_v2l, fac_l2v - real :: pidep, qi_crt, cin, qi_gen + real :: pidep, qi_crt ! ----------------------------------------------------------------------- ! qstar over water may be accurate only down to - 80 deg c with ~10% uncertainty @@ -2165,41 +2171,18 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & dq = qv (k) - qsi sink = dq / (1. + tcpk (k) * dqsdt) if (qi (k) > qrmin) then - if (inflag .eq. 1) & - ! Hong et al., 2004 - cin = 5.38e7 * exp (0.75 * log (qi (k) * den (k))) - if (inflag .eq. 2) & - ! Meyers et al., 1992 - cin = exp (-2.80 + 0.262 * (tice - tz (k))) * 1000.0 - if (inflag .eq. 3) & - ! Meyers et al., 1992 - cin = exp (-0.639 + 12.96 * (qv (k) / qsi - 1.0)) * 1000.0 - if (inflag .eq. 4) & - ! Cooper, 1986 - cin = 5.e-3 * exp (0.304 * (tice - tz (k))) * 1000.0 - if (inflag .eq. 5) & - ! Flecther, 1962 - cin = 1.e-5 * exp (0.5 * (tice - tz (k))) * 1000.0 - pidep = dt_pisub * dq * 4.0 * 11.9 * exp (0.5 * log (qi (k) * den (k) * cin)) & + ! eq 9, hong et al. 2004, mwr + ! for a and b, see dudhia 1989: page 3103 eq (b7) and (b8) + pidep = dt_pisub * dq * 349138.78 * exp (0.875 * log (qi (k) * den (k))) & / (qsi * den (k) * lat2 / (0.0243 * rvgas * tz (k) ** 2) + 4.42478e4) else pidep = 0. endif if (dq > 0.) then ! vapor - > ice tmp = tice - tz (k) - qi_gen = 4.92e-11 * exp (1.33 * log (1.e3 * exp (0.1 * tmp))) - if (igflag .eq. 1) & - ! WSM6 - qi_crt = qi_gen / den (k) - if (igflag .eq. 2) & - ! WSM6 with 0 at 0 C - qi_crt = qi_gen * min (qi_lim, 1.0 * tmp) / den (k) - if (igflag .eq. 3) & - ! WSM6 with 0 at 0 C and fixed value at -10 C - qi_crt = 1.82e-6 * min (qi_lim, 1.0 * tmp) / den (k) - if (igflag .eq. 4) & - ! combination of 1 and 3 - qi_crt = max (qi_gen, 1.82e-6) * min (qi_lim, 1.0 * tmp) / den (k) + ! 20160912: the following should produce more ice at higher altitude + ! qi_crt = 4.92e-11 * exp (1.33 * log (1.e3 * exp (0.1 * tmp))) / den (k) + qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor pidep = pidep * min (1., dim (tz (k), t_sub) / (tice - t_sub) * p20210319) From 4dc5a7454fa7a751654ce74b4174d0acaf0135df Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 30 Mar 2021 10:34:05 -0700 Subject: [PATCH 40/54] microphysics: simplify ice-->vapor transition rate remove the temperature dependent denominator change the namelist option name --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index d336eb1bd..cc91a1118 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -306,7 +306,7 @@ module gfdl_cloud_microphys_mod ! Linjiong-Noah custom tunining parameters ! TODO give these better names - real :: p20210319 = 1.0 + real :: rs_factor = 0.2 ! ----------------------------------------------------------------------- ! namelist @@ -324,7 +324,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, p20210319 + resmin, resmax, regmin, regmax, tintqs, rs_factor public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & @@ -338,7 +338,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, p20210319 + resmin, resmax, regmin, regmax, tintqs, rs_factor contains @@ -2185,7 +2185,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor - pidep = pidep * min (1., dim (tz (k), t_sub) / (tice - t_sub) * p20210319) + pidep = pidep * min (1., dim (tz (k), t_sub) * rs_factor) sink = max (pidep, sink, - qi (k)) endif qv (k) = qv (k) - sink From cd7dddd03f3dcdcf3efceeaa6e47b2307ada4f97 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 30 Mar 2021 10:43:19 -0700 Subject: [PATCH 41/54] microphysics: parameterize water-vapor transition add a namelist option rh_factor --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index cc91a1118..571e858c3 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -307,6 +307,7 @@ module gfdl_cloud_microphys_mod ! Linjiong-Noah custom tunining parameters ! TODO give these better names real :: rs_factor = 0.2 + real :: rh_factor = 10.0 ! ----------------------------------------------------------------------- ! namelist @@ -324,7 +325,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, rs_factor + resmin, resmax, regmin, regmax, tintqs, rs_factor, rh_factor public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & @@ -338,7 +339,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, rs_factor + resmin, resmax, regmin, regmax, tintqs, rs_factor, rh_factor contains @@ -2087,7 +2088,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & ! factor = min (1., fac_l2v * sqrt (max (0., ql (k)) / 1.e-5) * 10. * dq0 / qsw) ! factor = fac_l2v ! factor = 1 - factor = min (1., fac_l2v * (0.5 * dq0 / qsw)) ! the rh dependent factor = 1 at 90% + factor = min (1., fac_l2v * (rh_factor * dq0 / qsw)) ! the rh dependent factor = 1 at 90% evap = min (ql (k), factor * dq0 / (1. + tcp3 (k) * dwsdt)) else ! condensate all excess vapor into cloud water ! ----------------------------------------------------------------------- From 9ec1005bc2507423178eb2ec4b98d384884107cb Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 30 Mar 2021 11:59:19 -0700 Subject: [PATCH 42/54] microphysics: remove limits on h_var magnitude h_var is used to compute the saturation deficit for rain evaporation. This change was not suggested by Linjiong, but the 0.2 limit on h_var limits the utility of this parameter. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 571e858c3..93f6d283f 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -845,7 +845,6 @@ subroutine mpdrv (hydrostatic, uin, vin, w, delp, pt, qv, ql, qr, qi, qs, & t_land = dw_land * s_leng t_ocean = dw_ocean * s_leng h_var = t_land * land (i) + t_ocean * (1. - land (i)) - h_var = min (0.20, max (0.01, h_var)) ! if (id_var > 0) w_var (i, j) = h_var ! ----------------------------------------------------------------------- From f1a238d9fc6151e271759a74de1f0e6f79f0ef70 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 30 Mar 2021 12:12:54 -0700 Subject: [PATCH 43/54] microphysics: delete another limiter h_var still had little effect in the parent commit. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 93f6d283f..00a55ea95 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -1353,7 +1353,6 @@ subroutine revap_racc (ktop, kbot, dt, tz, qv, ql, qr, qi, qs, qg, den, denfac, qpz = qv (k) + ql (k) qsat = wqs2 (tin, den (k), dqsdt) dqh = max (ql (k), h_var * max (qpz, qcmin)) - dqh = min (dqh, 0.2 * qpz) ! new limiter dqv = qsat - qv (k) ! use this to prevent super - sat the gird box q_minus = qpz - dqh q_plus = qpz + dqh From 09ef08d55b5c19cf9b1e7ab9a76651be267d11e2 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 30 Mar 2021 12:18:32 -0700 Subject: [PATCH 44/54] Revert "microphysics: delete another limiter" This reverts commit f1a238d9fc6151e271759a74de1f0e6f79f0ef70. This had no effect --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 00a55ea95..93f6d283f 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -1353,6 +1353,7 @@ subroutine revap_racc (ktop, kbot, dt, tz, qv, ql, qr, qi, qs, qg, den, denfac, qpz = qv (k) + ql (k) qsat = wqs2 (tin, den (k), dqsdt) dqh = max (ql (k), h_var * max (qpz, qcmin)) + dqh = min (dqh, 0.2 * qpz) ! new limiter dqv = qsat - qv (k) ! use this to prevent super - sat the gird box q_minus = qpz - dqh q_plus = qpz + dqh From 3fcf3c1c76660c43d9212ea1c86d3332285c0d63 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 30 Mar 2021 12:24:40 -0700 Subject: [PATCH 45/54] Revert "microphysics: remove limits on h_var magnitude" This reverts commit 9ec1005bc2507423178eb2ec4b98d384884107cb. this did not seem to have much effect. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 93f6d283f..571e858c3 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -845,6 +845,7 @@ subroutine mpdrv (hydrostatic, uin, vin, w, delp, pt, qv, ql, qr, qi, qs, & t_land = dw_land * s_leng t_ocean = dw_ocean * s_leng h_var = t_land * land (i) + t_ocean * (1. - land (i)) + h_var = min (0.20, max (0.01, h_var)) ! if (id_var > 0) w_var (i, j) = h_var ! ----------------------------------------------------------------------- From 691cffe7ae845d28d189fcabad13c44946f0f2f5 Mon Sep 17 00:00:00 2001 From: linjiongzhou Date: Wed, 31 Mar 2021 09:16:14 -0400 Subject: [PATCH 46/54] Remove the changes in fv_cmp.F90. Change the namelist variable name "rs_factor" to "ts_factor". --- FV3/atmos_cubed_sphere/model/fv_cmp.F90 | 3 +-- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 index cf835922b..e2f281cfa 100644 --- a/FV3/atmos_cubed_sphere/model/fv_cmp.F90 +++ b/FV3/atmos_cubed_sphere/model/fv_cmp.F90 @@ -57,7 +57,7 @@ module fv_cmp_mod use constants_mod, only: rvgas, rdgas, grav, hlv, hlf, cp_air use fv_mp_mod, only: is_master use fv_arrays_mod, only: r_grid - use gfdl_cloud_microphys_mod, only: ql_gen, qi0_max, ql_mlt, ql0_max, qi_lim, qs_mlt + use gfdl_cloud_microphys_mod, only: ql_gen, qi_gen, qi0_max, ql_mlt, ql0_max, qi_lim, qs_mlt use gfdl_cloud_microphys_mod, only: icloud_f, sat_adj0, t_sub, cld_min use gfdl_cloud_microphys_mod, only: tau_r2g, tau_smlt, tau_i2s, tau_v2l, tau_l2v, tau_imlt, tau_l2r use gfdl_cloud_microphys_mod, only: rad_rain, rad_snow, rad_graupel, dw_ocean, dw_land, tintqs @@ -109,7 +109,6 @@ module fv_cmp_mod real :: d0_vap !< the same as dc_vap, except that cp_vap can be cp_vap or cv_vap real :: lv00 !< the same as lv0, except that cp_vap can be cp_vap or cv_vap - real :: qi_gen ! TODO this is no longer provided in Linjiongs fork. real, allocatable :: table (:), table2 (:), tablew (:), des2 (:), desw (:) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 571e858c3..7543deed6 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -306,7 +306,7 @@ module gfdl_cloud_microphys_mod ! Linjiong-Noah custom tunining parameters ! TODO give these better names - real :: rs_factor = 0.2 + real :: ts_factor = 0.2 real :: rh_factor = 10.0 ! ----------------------------------------------------------------------- @@ -325,7 +325,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, rs_factor, rh_factor + resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & @@ -339,7 +339,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, rs_factor, rh_factor + resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor contains @@ -2186,7 +2186,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor - pidep = pidep * min (1., dim (tz (k), t_sub) * rs_factor) + pidep = pidep * min (1., dim (tz (k), t_sub) * ts_factor) sink = max (pidep, sink, - qi (k)) endif qv (k) = qv (k) - sink From 98160470ce6371a65bea71467135d0eff1d37e27 Mon Sep 17 00:00:00 2001 From: linjiongzhou Date: Wed, 31 Mar 2021 09:29:01 -0400 Subject: [PATCH 47/54] Add the RH factor to the rain evaporation. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 7543deed6..25bc3b624 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -308,6 +308,7 @@ module gfdl_cloud_microphys_mod ! TODO give these better names real :: ts_factor = 0.2 real :: rh_factor = 10.0 + real :: re_factor = 100.0 ! ----------------------------------------------------------------------- ! namelist @@ -325,7 +326,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor + resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor, re_factor public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & @@ -339,7 +340,7 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor + resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor, re_factor contains @@ -1332,7 +1333,7 @@ subroutine revap_racc (ktop, kbot, dt, tz, qv, ql, qr, qi, qs, qg, den, denfac, real, dimension (ktop:kbot) :: lhl, cvm, q_liq, q_sol, lcpk real :: dqv, qsat, dqsdt, evap, t2, qden, q_plus, q_minus, sink - real :: qpz, dq, dqh, tin + real :: qpz, dq, dqh, tin, factor integer :: k @@ -1382,7 +1383,8 @@ subroutine revap_racc (ktop, kbot, dt, tz, qv, ql, qr, qi, qs, qg, den, denfac, t2 = tin * tin evap = crevp (1) * t2 * dq * (crevp (2) * sqrt (qden) + crevp (3) * & exp (0.725 * log (qden))) / (crevp (4) * t2 + crevp (5) * qsat * den (k)) - evap = min (qr (k), dt * evap, dqv / (1. + lcpk (k) * dqsdt)) + factor = min (1., re_factor * dqv / qsat) + evap = min (qr (k), dt * evap, factor * dqv / (1. + lcpk (k) * dqsdt)) ! ----------------------------------------------------------------------- ! alternative minimum evap in dry environmental air ! sink = min (qr (k), dim (rh_rain * qsat, qv (k)) / (1. + lcpk (k) * dqsdt)) From 396a274f06b8fd220c7986cc4001adb0a9a087f4 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Mon, 5 Apr 2021 12:14:33 -0700 Subject: [PATCH 48/54] microphysics: parameterize sublimation processes added parameters for ice snow and graupel sublimation --- .../physics/gfdl_cloud_microphys.F90 | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 25bc3b624..9517ac5f1 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -306,7 +306,9 @@ module gfdl_cloud_microphys_mod ! Linjiong-Noah custom tunining parameters ! TODO give these better names - real :: ts_factor = 0.2 + real :: ice_sublimation_factor = 0.2 + real :: snow_sublimation_factor = 0.2 + real :: graupel_sublimation_factor = 0.1 real :: rh_factor = 10.0 real :: re_factor = 100.0 @@ -326,7 +328,9 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor, re_factor + resmin, resmax, regmin, regmax, tintqs, rh_factor, re_factor, & + snow_sublimation_factor, ice_sublimation_factor, & + graupel_sublimation_factor public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & @@ -340,7 +344,9 @@ module gfdl_cloud_microphys_mod rad_snow, rad_graupel, rad_rain, cld_min, use_ppm, mono_prof, & do_sedi_heat, sedi_transport, do_sedi_w, de_ice, icloud_f, irain_f, & mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & - resmin, resmax, regmin, regmax, tintqs, ts_factor, rh_factor, re_factor + resmin, resmax, regmin, regmax, tintqs, rh_factor, re_factor, & + snow_sublimation_factor, ice_sublimation_factor, & + graupel_sublimation_factor contains @@ -2188,7 +2194,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & qi_crt = qi_gen * min (qi_lim, 0.1 * tmp) / den (k) sink = min (sink, max (qi_crt - qi (k), pidep), tmp / tcpk (k)) else ! ice -- > vapor - pidep = pidep * min (1., dim (tz (k), t_sub) * ts_factor) + pidep = pidep * min (1., dim (tz (k), t_sub) * ice_sublimation_factor) sink = max (pidep, sink, - qi (k)) endif qv (k) = qv (k) - sink @@ -2223,7 +2229,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & sqrt (denfac (k))) / (cssub (4) * tsq + cssub (5) * qsi * den (k)) pssub = (qsi - qv (k)) * dts * pssub if (pssub > 0.) then ! qs -- > qv, sublimation - pssub = min (pssub * min (1., dim (tz (k), t_sub) * 0.2), qs (k)) + pssub = min (pssub * min (1., dim (tz (k), t_sub) * snow_sublimation_factor), qs (k)) else if (tz (k) > tice) then pssub = 0. ! no deposition @@ -2264,7 +2270,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & (tice - tz (k)) / tcpk (k)) endif else ! submilation - pgsub = max (fac_g2v * pgsub, dq) * min (1., dim (tz (k), t_sub) * 0.1) + pgsub = max (fac_g2v * pgsub, dq) * min (1., dim (tz (k), t_sub) * graupel_sublimation_factor) endif qg (k) = qg (k) + pgsub qv (k) = qv (k) - pgsub From dcee2780387a25df7e8e7a60193baf486c27e89d Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Mon, 12 Apr 2021 14:18:57 -0700 Subject: [PATCH 49/54] add graupel deposition parameter --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 9517ac5f1..68831e8ec 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -309,6 +309,7 @@ module gfdl_cloud_microphys_mod real :: ice_sublimation_factor = 0.2 real :: snow_sublimation_factor = 0.2 real :: graupel_sublimation_factor = 0.1 + real :: graupel_deposition_factor = 0.2 real :: rh_factor = 10.0 real :: re_factor = 100.0 @@ -330,7 +331,7 @@ module gfdl_cloud_microphys_mod mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & resmin, resmax, regmin, regmax, tintqs, rh_factor, re_factor, & snow_sublimation_factor, ice_sublimation_factor, & - graupel_sublimation_factor + graupel_sublimation_factor, graupel_deposition_factor public & mp_time, t_min, t_sub, tau_r2g, tau_smlt, tau_g2r, dw_land, dw_ocean, & @@ -346,7 +347,7 @@ module gfdl_cloud_microphys_mod mp_print, reiflag, rewmin, rewmax, reimin, reimax, rermin, rermax, & resmin, resmax, regmin, regmax, tintqs, rh_factor, re_factor, & snow_sublimation_factor, ice_sublimation_factor, & - graupel_sublimation_factor + graupel_sublimation_factor, graupel_deposition_factor contains @@ -2266,7 +2267,7 @@ subroutine subgrid_z_proc (ktop, kbot, p1, den, denfac, dts, rh_adj, tz, qv, & if (tz (k) > tice) then pgsub = 0. ! no deposition else - pgsub = min (fac_v2g * pgsub, 0.2 * dq, ql (k) + qr (k), & + pgsub = min (fac_v2g * pgsub, graupel_deposition_factor * dq, ql (k) + qr (k), & (tice - tz (k)) / tcpk (k)) endif else ! submilation From 73f87fca250b247d0c4e8e01e40d0a2b8771958d Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Thu, 22 Apr 2021 15:54:49 -0700 Subject: [PATCH 50/54] Parameterize convective trigger thresholds Add options: - gfs_physics_nml.{cinpcrmx,cinpcrmn} --- FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 | 3 ++- FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 | 12 +++++++++++- FV3/gfsphysics/physics/samfdeepcnv.f | 10 ++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 b/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 index b5700764c..4784b0b4a 100644 --- a/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 +++ b/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 @@ -3594,7 +3594,8 @@ subroutine GFS_physics_driver & Model%clam_deep, Model%c0s_deep, & Model%c1_deep, Model%betal_deep, Model%betas_deep, & Model%evfact_deep, Model%evfactl_deep, & - Model%pgcon_deep, Model%asolfac_deep) + Model%pgcon_deep, Model%asolfac_deep,& + Model%cinpcrmx, Model%cinpcrmn) ! if (lprnt) print *,' rain1=',rain1(ipr) !elseif (Model%imfdeepcnv == 3) then ! if (Model%me==0) then diff --git a/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 b/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 index 54e6c10ce..cdc414ef1 100644 --- a/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 +++ b/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 @@ -892,6 +892,8 @@ module GFS_typedefs real(kind=kind_phys) :: ral_ts !< time scale for Rayleigh damping in days !--- mass flux deep convection + real(kind=kind_phys) :: cinpcrmx + real(kind=kind_phys) :: cinpcrmn real(kind=kind_phys) :: clam_deep !< c_e for deep convection (Han and Pan, 2011, eq(6)) real(kind=kind_phys) :: c0s_deep !< convective rain conversion parameter real(kind=kind_phys) :: c1_deep !< conversion parameter of detrainment from liquid water into grid-scale cloud water @@ -3124,6 +3126,10 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys) :: pertalb = -999. real(kind=kind_phys) :: pertvegf = -999. +! convection parameterization + real(kind=kind_phys) :: cinpcrmx = 180.0 + real(kind=kind_phys) :: cinpcrmn = 120.0 + !--- aerosol scavenging factors character(len=20) :: fscav_aero(20) = 'default' @@ -3221,7 +3227,9 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & !--- aerosol scavenging factors ('name:value' string array) fscav_aero, & sst_perturbation, & - override_surface_radiative_fluxes, use_climatological_sst + override_surface_radiative_fluxes, use_climatological_sst, & + cinpcrmx, cinpcrmn + !--- other parameters integer :: nctp = 0 !< number of cloud types in CS scheme @@ -3601,6 +3609,8 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%ral_ts = ral_ts !--- mass flux deep convection + Model%cinpcrmn = cinpcrmn + Model%cinpcrmx = cinpcrmx Model%clam_deep = clam_deep Model%c0s_deep = c0s_deep Model%c1_deep = c1_deep diff --git a/FV3/gfsphysics/physics/samfdeepcnv.f b/FV3/gfsphysics/physics/samfdeepcnv.f index 76204ebb4..a15921719 100644 --- a/FV3/gfsphysics/physics/samfdeepcnv.f +++ b/FV3/gfsphysics/physics/samfdeepcnv.f @@ -86,7 +86,8 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, & QLCN, QICN, w_upi, cf_upi, CNV_MFD, ! & QLCN, QICN, w_upi, cf_upi, CNV_MFD, CNV_PRC3, & CNV_DQLDT,CLCN,CNV_FICE,CNV_NDROP,CNV_NICE,mp_phys, - & clam,c0s,c1,betal,betas,evfact,evfactl,pgcon,asolfac) + & clam,c0s,c1,betal,betas,evfact,evfactl,pgcon,asolfac, + & cinpcrmx, cinpcrmn) ! use machine , only : kind_phys use funcphys , only : fpvs @@ -115,7 +116,7 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, real(kind=kind_phys) clam, c0s, c1, & betal, betas, asolfac, - & evfact, evfactl, pgcon + & evfact, evfactl, pgcon, cinpcrmn, cinpcrmx ! !------local variables integer i, indx, jmn, k, kk, km1, n @@ -182,8 +183,7 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, ! real(kind=kind_phys) c0(im) cj - real(kind=kind_phys) cinpcr, cinpcrmx, cinpcrmn, - & cinacr, cinacrmx, cinacrmn + real(kind=kind_phys) cinpcr, cinacr, cinacrmx, cinacrmn cj ! ! parameters for updraft velocity calculation @@ -213,8 +213,6 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, parameter(dtke=tkemx-tkemn) parameter(dbeta=0.1) parameter(cthk=200.,dthk=25.) - parameter(cinpcrmx=180.,cinpcrmn=120.) -! parameter(cinacrmx=-120.,cinacrmn=-120.) parameter(cinacrmx=-120.,cinacrmn=-80.) parameter(bet1=1.875,cd1=.506,f1=2.0,gam1=.5) parameter(betaw=.03,dxcrtas=8.e3,dxcrtuf=15.e3) From 8eae74103880ecc723357ecf3d5038ab82c1cc5c Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Tue, 4 May 2021 13:57:08 -0700 Subject: [PATCH 51/54] revert rain evaporation factor change this was the one real code difference compared to master, the remaining changes simply parameterize constants. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index 68831e8ec..d8ef9df3d 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -1390,8 +1390,7 @@ subroutine revap_racc (ktop, kbot, dt, tz, qv, ql, qr, qi, qs, qg, den, denfac, t2 = tin * tin evap = crevp (1) * t2 * dq * (crevp (2) * sqrt (qden) + crevp (3) * & exp (0.725 * log (qden))) / (crevp (4) * t2 + crevp (5) * qsat * den (k)) - factor = min (1., re_factor * dqv / qsat) - evap = min (qr (k), dt * evap, factor * dqv / (1. + lcpk (k) * dqsdt)) + evap = min (qr (k), dt * evap, dqv / (1. + lcpk (k) * dqsdt)) ! ----------------------------------------------------------------------- ! alternative minimum evap in dry environmental air ! sink = min (qr (k), dim (rh_rain * qsat, qv (k)) / (1. + lcpk (k) * dqsdt)) From 87fe79af20e62e3503b21b47a12c3bc452b06fca Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Wed, 12 May 2021 16:37:24 -0700 Subject: [PATCH 52/54] Revert "revert rain evaporation factor change" This reverts commit 8eae74103880ecc723357ecf3d5038ab82c1cc5c. Without this change the rain water becomes far too low. --- FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 index d8ef9df3d..68831e8ec 100644 --- a/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 +++ b/FV3/gfsphysics/physics/gfdl_cloud_microphys.F90 @@ -1390,7 +1390,8 @@ subroutine revap_racc (ktop, kbot, dt, tz, qv, ql, qr, qi, qs, qg, den, denfac, t2 = tin * tin evap = crevp (1) * t2 * dq * (crevp (2) * sqrt (qden) + crevp (3) * & exp (0.725 * log (qden))) / (crevp (4) * t2 + crevp (5) * qsat * den (k)) - evap = min (qr (k), dt * evap, dqv / (1. + lcpk (k) * dqsdt)) + factor = min (1., re_factor * dqv / qsat) + evap = min (qr (k), dt * evap, factor * dqv / (1. + lcpk (k) * dqsdt)) ! ----------------------------------------------------------------------- ! alternative minimum evap in dry environmental air ! sink = min (qr (k), dim (rh_rain * qsat, qv (k)) / (1. + lcpk (k) * dqsdt)) From 674dd1182dddc6e7a16654b508d1cac276a4da63 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 28 May 2021 13:05:37 -0700 Subject: [PATCH 53/54] deep conv: parameterize cinarcmn --- FV3/conf/configure.fv3 | 30 +++++++------------ .../GFS_layer/GFS_physics_driver.F90 | 2 +- FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 | 5 +++- FV3/gfsphysics/physics/samfdeepcnv.f | 4 +-- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/FV3/conf/configure.fv3 b/FV3/conf/configure.fv3 index b941f6b98..1eb218ca0 100644 --- a/FV3/conf/configure.fv3 +++ b/FV3/conf/configure.fv3 @@ -2,10 +2,13 @@ ############ # commands # ############ -FC = FC=gfortran mpifort -CC = CC=gcc mpicc +SHELL = bash +FC = mpif90 +CC = mpicc CXX = g++ -LD = mpifort +LD = mpif90 +CPP = cpp + NETCDF_DIR = /usr @@ -36,7 +39,7 @@ $(error Need at least make version $(need). Load module gmake/3.81) endif NETCDF_ROOT = $(NETCDF_DIR) -INCLUDE = -I$(NETCDF_ROOT)/include +INCLUDE ?= -I$(NETCDF_ROOT)/include FPPFLAGS := -cpp -Wp,-w $(INCLUDE) -fPIC CFLAGS := $(INCLUDE) -fPIC @@ -46,10 +49,6 @@ FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fP CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -Duse_LARGEFILE -DUSE_GFSL63 -DGFS_PHYS -DNO_INLINE_POST CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -ifeq ($(GT4PY_DEV),Y) -CPPDEFS += -DGT4PY_DEV -endif - ifeq ($(HYDRO),Y) CPPDEFS += else @@ -70,7 +69,6 @@ endif FFLAGS_OPT = -O2 FFLAGS_REPRO = -O2 -g -fbacktrace FFLAGS_DEBUG = -O0 -g -fbacktrace -fno-fast-math -ffree-line-length-none -fno-backslash -pedantic -Waliasing -Wampersand -Wline-truncation -Wsurprising -Wtabs -Wunderflow -fdump-core -ffpe-trap=invalid,zero,overflow -fbounds-check -finit-real=nan -finit-integer=9999999 -finit-logical=true -finit-character=35 -FFLAGS_GCOV = --coverage TRANSCENDENTALS := -fast-transcendentals FFLAGS_OPENMP = -fopenmp @@ -82,17 +80,15 @@ CFLAGS_OPT = -O2 CFLAGS_REPRO = -O2 CFLAGS_OPENMP = -fopenmp CFLAGS_DEBUG = -O0 -g -CFLAGS_GCOV = --coverage # Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT # *_TEST will match the production if no new option(s) is(are) to be tested. FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits CFLAGS_TEST = -O2 -LDFLAGS := -L${ESMF_DIR}/lib/libO3/Linux.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ +#LDFLAGS := -L${ESMF_DIR}/lib/libO3/Darwin.gfortran.64.mpiuni.default/ -L${FMS_DIR}/libFMS/.libs/ LDFLAGS_OPENMP := -fopenmp LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M -LDFLAGS_GCOV = --coverage # start with blank LIBS LIBS := @@ -117,12 +113,6 @@ FFLAGS += $(FFLAGS_OPT) FAST := $(TRANSCENDENTALS) endif -ifneq ($(GCOV),) -CFLAGS += $(CFLAGS_GCOV) -FFLAGS += $(FFLAGS_GCOV) -LDFLAGS += $(LDFLAGS_GCOV) -endif - ifneq ($(OPENMP),) CFLAGS += $(CFLAGS_OPENMP) FFLAGS += $(FFLAGS_OPENMP) @@ -138,6 +128,8 @@ FFLAGS += $(FFLAGS_VERBOSE) LDFLAGS += $(LDFLAGS_VERBOSE) endif -LIBS += -lFMS -lesmf -lnetcdff -lnetcdf -llapack -lblas -lc -lrt +# static linking of esmf works on darwin +#LIBS += -lFMS -lnetcdff -lnetcdf -llapack -lblas ${ESMF_DIR}/lib/libesmf.a -lstdc++ #-lc -lrt +LIBS += -lFMS -lnetcdff -lnetcdf -llapack -lblas -lesmf #-lc -lrt LDFLAGS += $(LIBS) diff --git a/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 b/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 index 4784b0b4a..d53696059 100644 --- a/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 +++ b/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 @@ -3595,7 +3595,7 @@ subroutine GFS_physics_driver & Model%c1_deep, Model%betal_deep, Model%betas_deep, & Model%evfact_deep, Model%evfactl_deep, & Model%pgcon_deep, Model%asolfac_deep,& - Model%cinpcrmx, Model%cinpcrmn) + Model%cinpcrmx, Model%cinpcrmn, Model%cinacrmn) ! if (lprnt) print *,' rain1=',rain1(ipr) !elseif (Model%imfdeepcnv == 3) then ! if (Model%me==0) then diff --git a/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 b/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 index cdc414ef1..ef3bb3ac9 100644 --- a/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 +++ b/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 @@ -894,6 +894,7 @@ module GFS_typedefs !--- mass flux deep convection real(kind=kind_phys) :: cinpcrmx real(kind=kind_phys) :: cinpcrmn + real(kind=kind_phys) :: cinacrmn real(kind=kind_phys) :: clam_deep !< c_e for deep convection (Han and Pan, 2011, eq(6)) real(kind=kind_phys) :: c0s_deep !< convective rain conversion parameter real(kind=kind_phys) :: c1_deep !< conversion parameter of detrainment from liquid water into grid-scale cloud water @@ -3129,6 +3130,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & ! convection parameterization real(kind=kind_phys) :: cinpcrmx = 180.0 real(kind=kind_phys) :: cinpcrmn = 120.0 + real(kind=kind_phys) :: cinacrmn = -120.0 !--- aerosol scavenging factors character(len=20) :: fscav_aero(20) = 'default' @@ -3228,7 +3230,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & fscav_aero, & sst_perturbation, & override_surface_radiative_fluxes, use_climatological_sst, & - cinpcrmx, cinpcrmn + cinpcrmx, cinpcrmn, cinacrmn !--- other parameters @@ -3611,6 +3613,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & !--- mass flux deep convection Model%cinpcrmn = cinpcrmn Model%cinpcrmx = cinpcrmx + Model%cinacrmn = cinacrmn Model%clam_deep = clam_deep Model%c0s_deep = c0s_deep Model%c1_deep = c1_deep diff --git a/FV3/gfsphysics/physics/samfdeepcnv.f b/FV3/gfsphysics/physics/samfdeepcnv.f index a15921719..66ae72935 100644 --- a/FV3/gfsphysics/physics/samfdeepcnv.f +++ b/FV3/gfsphysics/physics/samfdeepcnv.f @@ -87,7 +87,7 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, ! & QLCN, QICN, w_upi, cf_upi, CNV_MFD, CNV_PRC3, & CNV_DQLDT,CLCN,CNV_FICE,CNV_NDROP,CNV_NICE,mp_phys, & clam,c0s,c1,betal,betas,evfact,evfactl,pgcon,asolfac, - & cinpcrmx, cinpcrmn) + & cinpcrmx, cinpcrmn, cinacrmn) ! use machine , only : kind_phys use funcphys , only : fpvs @@ -213,7 +213,7 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, parameter(dtke=tkemx-tkemn) parameter(dbeta=0.1) parameter(cthk=200.,dthk=25.) - parameter(cinacrmx=-120.,cinacrmn=-80.) + parameter(cinacrmx=-120.) parameter(bet1=1.875,cd1=.506,f1=2.0,gam1=.5) parameter(betaw=.03,dxcrtas=8.e3,dxcrtuf=15.e3) ! From 2caa87913818e8ad81258719bdf5f6393b9a6621 Mon Sep 17 00:00:00 2001 From: "Noah D. Brenowitz" Date: Fri, 28 May 2021 13:15:33 -0700 Subject: [PATCH 54/54] deep conv: add timescale_grid_factor parameter Jongil suggested we remove the grid dependence of the convective timescale. The time scale is currently proportional to (1 + 1 / 75 km dx) Adjust the scaling to be (1 + timescale_grid_factor * dx) so we can use different grid scalings for coarse resolution. --- FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 | 3 ++- FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 | 5 ++++- FV3/gfsphysics/physics/samfdeepcnv.f | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 b/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 index d53696059..68012e23f 100644 --- a/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 +++ b/FV3/gfsphysics/GFS_layer/GFS_physics_driver.F90 @@ -3595,7 +3595,8 @@ subroutine GFS_physics_driver & Model%c1_deep, Model%betal_deep, Model%betas_deep, & Model%evfact_deep, Model%evfactl_deep, & Model%pgcon_deep, Model%asolfac_deep,& - Model%cinpcrmx, Model%cinpcrmn, Model%cinacrmn) + Model%cinpcrmx, Model%cinpcrmn, Model%cinacrmn,& + Model%timescale_grid_factor) ! if (lprnt) print *,' rain1=',rain1(ipr) !elseif (Model%imfdeepcnv == 3) then ! if (Model%me==0) then diff --git a/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 b/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 index ef3bb3ac9..865534f35 100644 --- a/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 +++ b/FV3/gfsphysics/GFS_layer/GFS_typedefs.F90 @@ -895,6 +895,7 @@ module GFS_typedefs real(kind=kind_phys) :: cinpcrmx real(kind=kind_phys) :: cinpcrmn real(kind=kind_phys) :: cinacrmn + real(kind=kind_phys) :: timescale_grid_factor real(kind=kind_phys) :: clam_deep !< c_e for deep convection (Han and Pan, 2011, eq(6)) real(kind=kind_phys) :: c0s_deep !< convective rain conversion parameter real(kind=kind_phys) :: c1_deep !< conversion parameter of detrainment from liquid water into grid-scale cloud water @@ -3131,6 +3132,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & real(kind=kind_phys) :: cinpcrmx = 180.0 real(kind=kind_phys) :: cinpcrmn = 120.0 real(kind=kind_phys) :: cinacrmn = -120.0 + real(kind=kind_phys) :: timescale_grid_factor = 1.33333e-05 !--- aerosol scavenging factors character(len=20) :: fscav_aero(20) = 'default' @@ -3230,7 +3232,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & fscav_aero, & sst_perturbation, & override_surface_radiative_fluxes, use_climatological_sst, & - cinpcrmx, cinpcrmn, cinacrmn + cinpcrmx, cinpcrmn, cinacrmn, timescale_grid_factor !--- other parameters @@ -3614,6 +3616,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%cinpcrmn = cinpcrmn Model%cinpcrmx = cinpcrmx Model%cinacrmn = cinacrmn + Model%timescale_grid_factor = timescale_grid_factor Model%clam_deep = clam_deep Model%c0s_deep = c0s_deep Model%c1_deep = c1_deep diff --git a/FV3/gfsphysics/physics/samfdeepcnv.f b/FV3/gfsphysics/physics/samfdeepcnv.f index 66ae72935..cacbf5b87 100644 --- a/FV3/gfsphysics/physics/samfdeepcnv.f +++ b/FV3/gfsphysics/physics/samfdeepcnv.f @@ -87,7 +87,7 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, ! & QLCN, QICN, w_upi, cf_upi, CNV_MFD, CNV_PRC3, & CNV_DQLDT,CLCN,CNV_FICE,CNV_NDROP,CNV_NICE,mp_phys, & clam,c0s,c1,betal,betas,evfact,evfactl,pgcon,asolfac, - & cinpcrmx, cinpcrmn, cinacrmn) + & cinpcrmx, cinpcrmn, cinacrmn, timescale_grid_factor) ! use machine , only : kind_phys use funcphys , only : fpvs @@ -114,6 +114,8 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, & rn(im), cnvw(ix,km), cnvc(ix,km), & ud_mf(im,km),dd_mf(im,km), dt_mf(im,km) + + real(kind=kind_phys) timescale_grid_factor real(kind=kind_phys) clam, c0s, c1, & betal, betas, asolfac, & evfact, evfactl, pgcon, cinpcrmn, cinpcrmx @@ -2273,7 +2275,7 @@ subroutine samfdeepcnv(im,ix,km,delt,itc,ntc,ntk,ntr,delp, if(cnvflg(i)) then tem = zi(i,ktcon1(i)) - zi(i,kbcon1(i)) dtconv(i) = tem / wc(i) - tfac = 1. + gdx(i) / 75000. + tfac = 1. + timescale_grid_factor * gdx(i) dtconv(i) = tfac * dtconv(i) dtconv(i) = max(dtconv(i),dtmin) dtconv(i) = min(dtconv(i),dtmax)