Integrate MPAS build infrastructure with CIME

cime_config/buildlib

@@ -141,6 +141,11 @@ def _build_cam():
         #-------------------------------------------------------
         # build the library
         #-------------------------------------------------------
+
+        # If dynamical core is MPAS, setup its build infrastructure so it can be built along with CAM below.
+        if dycore == "mpas":
+            _setup_mpas(case)
+
         complib = os.path.join(libroot, "libatm.a")
         makefile = os.path.join(casetools, "Makefile")
 
@@ -156,6 +161,50 @@ def _build_cam():
         _LOGGER.info("%s: \n\n output:\n %s \n\n err:\n\n%s\n", cmd, out, err)
         expect(retcode == 0, f"Command {cmd} failed with rc={retcode}")
 
+def _setup_mpas(case: Case) -> None:
+    """
+    Setup MPAS build infrastructure.
+    """
+
+    atm_src_root = os.path.normpath(case.get_value("COMP_ROOT_DIR_ATM"))
+    atm_bld_root = os.path.normpath(os.path.join(case.get_value("EXEROOT"), "atm", "obj"))
+
+    mpas_dycore_src_root = os.path.join(atm_src_root, "src", "dynamics", "mpas", "dycore", "src")
+    mpas_dycore_bld_root = os.path.join(atm_bld_root, "mpas")
+
+    # Make sure `mpas_dycore_src_root` exists. If not, it is likely that `./manage_externals/checkout_externals` did not succeed.
+    if os.path.isfile(mpas_dycore_src_root):
+        raise FileExistsError(1, "Unexpected file", mpas_dycore_src_root)
+
+    if not os.path.isdir(mpas_dycore_src_root):
+        raise FileNotFoundError(1, "No such directory", mpas_dycore_src_root)
+
+    # MPAS supports "in-source" build only. Copy source code to `mpas_dycore_bld_root` to avoid polluting `mpas_dycore_src_root`.
+    if os.path.isfile(mpas_dycore_bld_root):
+        raise FileExistsError(1, "Unexpected file", mpas_dycore_bld_root)
+
+    shutil.copytree(mpas_dycore_src_root, mpas_dycore_bld_root, copy_function=_copy2_as_needed, dirs_exist_ok=True)
+    shutil.move(os.path.join(mpas_dycore_bld_root, "Makefile"), os.path.join(mpas_dycore_bld_root, "Makefile.CESM"))
+    _copy2_as_needed(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "Makefile")), os.path.join(mpas_dycore_bld_root, "Makefile"))
+    _copy2_as_needed(os.path.normpath(os.path.join(mpas_dycore_src_root, os.pardir, os.pardir, "Makefile.in.CESM")), os.path.join(mpas_dycore_bld_root, "Makefile.in.CESM"))
+
+def _copy2_as_needed(src: str, dst: str) -> None:
+    """
+    Wrapper around `shutil.copy2`. Copy the file `src` to the file or directory `dst` as needed.
+    """
+
+    if os.path.isfile(src):
+        if os.path.isfile(dst):
+            if int(os.path.getmtime(src)) != int(os.path.getmtime(dst)) or os.path.getsize(src) != os.path.getsize(dst):
+                # `src` and `dst` are both files but their modification time or size differ.
+                # Example scenario: User modified some existing source code files.
+                shutil.copy2(src, dst)
+        else:
+            if os.path.isdir(dst) or os.path.isdir(os.path.dirname(dst)):
+                # `src` is a new file that does not exist at `dst`.
+                # Example scenario: User added some new source code files.
+                shutil.copy2(src, dst)
+
 ###############################################################################
 
 if __name__ == "__main__":

cime_config/config_component.xml

@@ -103,10 +103,11 @@
 
   <entry id="CAM_DYCORE">
     <type>char</type>
-    <valid_values>eul,fv,se,none</valid_values>
+    <valid_values>eul,fv,mpas,se,none</valid_values>
     <default_value>fv</default_value>
     <values match="last">
-      <value grid="a%T[1-9]" >eul</value>
+      <value grid="a%T[1-9]">eul</value>
+      <value grid="a%mpasa[0-9]+">mpas</value>
       <value grid="a%ne[0-9]">se</value>
     </values>
     <group>build_component_cam</group>
@@ -305,9 +306,10 @@
     <type>char</type>
     <valid_values></valid_values>
     <default_value>-lmusica -ljsonfortran</default_value>
-    <values match="last">
-      <!--Turn off for physics testbed -->
-      <value compset="_CAM%PHYSTEST"> </value>
+    <values match="last" modifier="additive">
+      <!-- Turn off for physics testbed -->
+      <value compset="_CAM%PHYSTEST"></value>
+      <value grid="a%mpasa[0-9]+">-lmpas</value>
     </values>
     <group>build_component_cam</group>
     <file>env_build.xml</file>

src/dynamics/mpas/Makefile

@@ -0,0 +1,9 @@
+# This Makefile is invoked by CIME Makefile (i.e., `cime/CIME/Tools/Makefile`).
+
+# Some targets in MPAS build infrastructure are sensitive to this environment variable. Override it to avoid build issues.
+override PWD = $(CURDIR)
+export PWD
+
+all:
+	$(MAKE) -f Makefile.CESM libmpas-prepare ESM="CESM"
+	$(MAKE) -f Makefile.CESM libmpas-build ESM="CESM"

src/dynamics/mpas/Makefile.in.CESM

@@ -0,0 +1,118 @@
+ifeq ($(strip $(LIBROOT)),)
+    $(warning `LIBROOT` should not be empty. Defaulting to `..`)
+
+    LIBROOT = ..
+endif
+
+#
+# Define and export variables used by MPAS build infrastructure.
+#
+
+export CP = cp -afv
+export MKDIR = mkdir -pv
+export RM = rm -frv
+
+# Constants.
+export AUTOCLEAN       = false
+export BUILD_TARGET    = N/A
+export CORE            = atmosphere
+export EXE_NAME        = atmosphere_model
+export GEN_F90         = false
+export GIT_VERSION     = N/A
+export NAMELIST_SUFFIX = atmosphere
+
+# Customize variables (e.g., build options) for use with CESM.
+export AR             := ar
+export ARFLAGS        := -M
+export CPP            := cpp -P -traditional
+export CPPFLAGS       := -D_MPI \
+                         -DMPAS_BUILD_TARGET="$(BUILD_TARGET)" \
+                         -DMPAS_CAM_DYCORE \
+                         -DMPAS_EXE_NAME="$(EXE_NAME)" \
+                         -DMPAS_EXTERNAL_ESMF_LIB \
+                         -DMPAS_GIT_VERSION="$(GIT_VERSION)" \
+                         -DMPAS_NAMELIST_SUFFIX="$(NAMELIST_SUFFIX)" \
+                         -DMPAS_NATIVE_TIMERS \
+                         -DMPAS_NO_ESMF_INIT \
+                         -DMPAS_PIO_SUPPORT
+# `PIODEF` is defined by CIME Makefile (i.e., `cime/CIME/Tools/Makefile`). Its value can be empty or `-DUSE_PIO2`.
+ifneq ($(strip $(PIODEF)),)
+    export CPPFLAGS   += $(strip $(PIODEF))
+endif
+export LINKER         := $(strip $(FC))
+export SCC            := $(strip $(CC))
+export SCXX           := $(strip $(CXX))
+export SFC            := $(strip $(FC))
+
+#
+# Targets.
+#
+
+.PHONY: all
+all:
+	@echo 'Supplemental Makefile for MPAS Dynamical Core in CESM'
+	@echo ''
+	@echo 'MPAS will be built as a static library located at `$${LIBROOT}/libmpas.a`.'
+	@echo 'Users are responsible to provide all necessary build options via environment variables or command line arguments.'
+	@echo ''
+	@echo 'Usage hints:'
+	@echo '    `make libmpas-prepare ESM="CESM" LIBROOT="..."`'
+	@echo '    `make libmpas-build ESM="CESM" LIBROOT="..."`'
+	@echo '    `make libmpas-clean ESM="CESM" LIBROOT="..."`'
+
+.PHONY: libmpas-prepare
+libmpas-prepare: libmpas-archiver-script.txt libmpas-no-physics libmpas-preview
+
+# Combine multiple static libraries into `libmpas.a` via archiver/MRI script. This requires GNU or GNU-like archiver (`ar`) program.
+libmpas-archiver-script.txt:
+	@echo "create libmpas.a"       > $(@)
+	@echo "addlib libdycore.a"    >> $(@)
+	@echo "addlib libframework.a" >> $(@)
+	@echo "addlib libops.a"       >> $(@)
+	@echo "save"                  >> $(@)
+	@echo "end"                   >> $(@)
+
+# Do not use built-in MPAS/WRF physics.
+.PHONY: libmpas-no-physics
+libmpas-no-physics:
+	@sed -E -i -e "s/^ *PHYSICS=.+$$/PHYSICS=/g" core_atmosphere/Makefile
+
+.PHONY: libmpas-preview
+libmpas-preview:
+	@echo "Previewing build options for $(LIBROOT)/libmpas.a:"
+	@echo "AR = $(AR)"
+	@echo "ARFLAGS = $(ARFLAGS)"
+	@echo "CC = $(CC)"
+	@echo "CFLAGS = $(CFLAGS)"
+	@echo "CPP = $(CPP)"
+	@echo "CPPFLAGS = $(CPPFLAGS)"
+	@echo "CPPINCLUDES = $(CPPINCLUDES)"
+	@echo "CXX = $(CXX)"
+	@echo "CXXFLAGS = $(CXXFLAGS)"
+	@echo "FC = $(FC)"
+	@echo "FCINCLUDES = $(FCINCLUDES)"
+	@echo "FFLAGS = $(FFLAGS)"
+	@echo "LDFLAGS = $(LDFLAGS)"
+	@echo "LIBS = $(LIBS)"
+	@echo "LINKER = $(LINKER)"
+	@echo "SCC = $(SCC)"
+	@echo "SCXX = $(SCXX)"
+	@echo "SFC = $(SFC)"
+
+.PHONY: libmpas-build
+libmpas-build: $(LIBROOT)/libmpas.a
+
+$(LIBROOT)/libmpas.a: libmpas.a
+	$(MKDIR) $(LIBROOT)
+	$(CP) $(<) $(@)
+
+libmpas.a: $(AUTOCLEAN_DEPS) dycore externals frame ops
+	$(AR) $(ARFLAGS) < libmpas-archiver-script.txt
+
+.PHONY: libmpas-clean
+libmpas-clean: clean
+	$(RM) $(LIBROOT)/libmpas.a libmpas.a
+
+.PHONY: externals
+externals: $(AUTOCLEAN_DEPS)
+	( cd external; $(MAKE) FC="$(FC)" SFC="$(SFC)" CC="$(CC)" SCC="$(SCC)" FFLAGS="$(FFLAGS)" CFLAGS="$(CFLAGS)" CPP="$(CPP)" NETCDF="$(NETCDF)" CORE="$(CORE)" ezxml-lib )