Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature/mpas dycore #2451

Draft
wants to merge 16 commits into
base: develop
Choose a base branch
from
Draft

Feature/mpas dycore #2451

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
b411c6c
Create feature/mpas-dycore branch
dustinswales Jul 18, 2024
82dbe8c
Reorganize code for MPAS dycore. Isolate FV3 and MPAS pieces.
dustinswales Jul 19, 2024
4b3b5e3
Update submodules
dustinswales Jul 19, 2024
b01ced7
Added more build step pieces, up until the MPAS dycore build
dustinswales Jul 19, 2024
9e82655
Some more changes. Almost building.
dustinswales Jul 30, 2024
f0c3a78
MPAS building in UFSATM! Some more reorganization. Initial infrastruc…
dustinswales Aug 1, 2024
fe47b33
Forgot to link to personal for of MPAS submodule
dustinswales Aug 7, 2024
f26d538
Address reviewers comments
dustinswales Aug 13, 2024
4c2f900
Address comments
dustinswales Aug 13, 2024
4952814
Add MPAS to default_vars.sh
dustinswales Aug 23, 2024
d6418b4
Move MPAS to ufs-community reporitory
dustinswales Sep 3, 2024
7d0e826
Revert unneeded change
dustinswales Sep 3, 2024
ee33231
Synced FV3
dustinswales Oct 2, 2024
55f4f66
Updated FV3 submodule hash
dustinswales Oct 2, 2024
e12268e
Merge branch 'develop' of https://github.com/ufs-community/ufs-weathe…
dustinswales Oct 2, 2024
d08975f
Housekeeping
dustinswales Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[submodule "FV3"]
path = FV3
url = https://github.com/NOAA-EMC/fv3atm
branch = develop
url = https://github.com/NCAR/fv3atm
branch = feature/mpas-dycore
[submodule "WW3"]
path = WW3
url = https://github.com/NOAA-EMC/WW3
Expand Down
34 changes: 28 additions & 6 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
###############################################################################

# Valid applications and choices
list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML ATMF LND S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML ATMF ATMMPAS LND S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
set(APP NONE CACHE BOOL "Application Name")
if(NOT (APP IN_LIST VALID_APPS))
message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
endif()

set(FMS OFF CACHE BOOL "Enable FMS")
set(FV3 OFF CACHE BOOL "Enable FV3")
set(FV3 OFF CACHE BOOL "Enable FV3 dycore")
set(MPAS OFF CACHE BOOL "Enable MPAS dycore")
set(AQM OFF CACHE BOOL "Enable AQM")
set(UFS_GOCART OFF CACHE BOOL "Enable GOCART")
set(MOM6 OFF CACHE BOOL "Enable MOM6")
Expand All @@ -42,7 +43,8 @@ include(cmake/configure_apps.cmake)

message("")
message("FMS .............. ${FMS}")
message("FV3 .............. ${FV3}")
message("FV3_dy ........... ${FV3}")
message("MPAS_dy .......... ${MPAS}")
message("AQM .............. ${AQM}")
message("GOCART ........... ${UFS_GOCART}")
message("MOM6 ............. ${MOM6}")
Expand Down Expand Up @@ -157,7 +159,7 @@ if(FMS)
elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|ATM_DS2S|ATM_DS2S-PCICE|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
add_library(fms ALIAS FMS::fms_r8)
endif()
if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|ATMF|HAFS|HAFS-ALL)$")
if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|ATMF|ATMMPAS|HAFS|HAFS-ALL)$")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove if MPAS does not need FMS

if(32BIT)
add_library(fms ALIAS FMS::fms_r4)
else()
Expand All @@ -174,6 +176,12 @@ find_package(bacio 2.4.0 REQUIRED)
find_package(sp 2.3.3 REQUIRED)
find_package(w3emc 2.9.2 REQUIRED)

if(MPAS)
find_package(PnetCDF REQUIRED COMPONENTS Fortran)
find_package(PIO REQUIRED COMPONENTS Fortran C)
set(FV3 OFF)
endif()

# Configure Python
find_package(Python 3.6 REQUIRED COMPONENTS Interpreter)
message("Found Python: ${Python_EXECUTABLE}")
Expand All @@ -187,10 +195,18 @@ endif()

###############################################################################
### Atmosphere Components [FV3, MPAS?]
### DJS2024: There will be a generalized atmospheric component with multiple
### dycores, not multiple atmospheric components (one for each dycore).
### So, - rename the atmospheric component FV3 to UFSATM or ATM, then
### - if FV3 or MPAS, add_sudirectory(UFSATM or ATM)
###
###############################################################################
if(FV3)
add_subdirectory(FV3)
endif()
if (MPAS)
add_subdirectory(FV3)
endif()

###############################################################################
### AQM
Expand Down Expand Up @@ -306,9 +322,15 @@ if(STOCH_PHYS)
endif()

if(FV3)
add_dependencies(ufs fv3atm)
add_dependencies(ufs ufsatm_fv3)
list(APPEND _ufs_defs_private FRONT_FV3=fv3atm_cap_mod)
list(APPEND _ufs_libs_public fv3atm)
list(APPEND _ufs_libs_public ufsatm_fv3)
endif()

if(MPAS)
add_dependencies(ufs ufsatm_mpas)
list(APPEND _ufs_defs_private FRONT_MPAS=mpasatm_cap_mod)
list(APPEND _ufs_libs_public ufsatm_mpas)
endif()

if(MOM6)
Expand Down
2 changes: 1 addition & 1 deletion FV3
Open in desktop
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This submodule pointer update seems to be wrong. If I click on the changes, I get taken to a update of 22 files all within the NCAR fv3atm fork - should this PR not update the code in the noaa-emc fv3atm? Maybe I am missing something (it's late).

Submodule FV3 updated 22 files
+6 −2 .gitmodules
+190 −128 CMakeLists.txt
+58 −24 ccpp/CMakeLists.txt
+0 −0 fv3/atmos_cubed_sphere
+0 −0 fv3/atmos_model.F90
+4 −4 fv3/ccpp/config/ccpp_prebuild_config.py
+0 −0 fv3/fv3_cap.F90
+0 −0 fv3/module_fcst_grid_comp.F90
+0 −0 fv3/module_fv3_config.F90
+0 −0 fv3/moving_nest/bounding_box.F90
+0 −0 fv3/moving_nest/fv_moving_nest.F90
+0 −0 fv3/moving_nest/fv_moving_nest_main.F90
+0 −0 fv3/moving_nest/fv_moving_nest_physics.F90
+0 −0 fv3/moving_nest/fv_moving_nest_types.F90
+0 −0 fv3/moving_nest/fv_moving_nest_utils.F90
+0 −0 fv3/moving_nest/fv_tracker.F90
+131 −0 mpas/CMakeLists.txt
+1 −0 mpas/MPAS-Model
+251 −0 mpas/ccpp/config/ccpp_prebuild_config.py
+19 −0 mpas/mpas_cap.F90
+26 −0 mpas/ufs_mpas.F90
+560 −0 mpas/ufs_mpas_subdriver.F90
5 changes: 4 additions & 1 deletion cmake/configure_apps.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
###############################################################################
### Configure Application Components
###############################################################################
if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML|ATMF)$")
if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML|ATMF|ATMMPAS)$")
set(FMS ON CACHE BOOL "Enable FMS" FORCE)
set(FV3 ON CACHE BOOL "Enable FV3" FORCE)
set(STOCH_PHYS ON CACHE BOOL "Enable Stochastic Physics" FORCE)
Expand All @@ -31,6 +31,9 @@ if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML|ATMF)$")
set(CMEPS ON CACHE BOOL "Enable CMEPS" FORCE)
set(NOAHMP ON CACHE BOOL "Enable NOAHMP" FORCE)
message("Configuring UFS app in Atmosphere with Air Quality mode")
elseif(APP MATCHES "ATMMPAS")
set(MPAS ON CACHE BOOL "Enable MPAS dycore" FORCE)
message("Configuring UFS app in Atmosphere with MPAS dycore")
elseif(APP MATCHES "ATMF")
set(FIRE_BEHAVIOR ON CACHE BOOL "Enable Fire Behavior" FORCE)
else()
Expand Down
11 changes: 11 additions & 0 deletions driver/UFSDriver.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ MODULE UFSDriver
#ifdef FRONT_FV3
use FRONT_FV3, only: FV3_SS => SetServices
#endif
#ifdef FRONT_MPAS
use FRONT_MPAS, only: MPAS_SS => SetServices
#endif
#ifdef FRONT_CDEPS_DATM
use FRONT_CDEPS_DATM, only: DATM_SS => SetServices
#endif
Expand Down Expand Up @@ -367,6 +370,14 @@ subroutine SetModelServices(driver, rc)
found_comp = .true.
end if
#endif
#ifdef FRONT_MPAS
! if (trim(model) == "mpas") then
! call NUOPC_DriverAddComp(driver, trim(prefix), MPAS_SS, &
! info=info, petList=petList, comp=comp, rc=rc)
! if (ChkErr(rc,__LINE__,u_FILE_u)) return
! found_comp = .true.
! end if
#endif
Comment on lines +373 to +380
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason this is commented out currently?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's just a placeholder for the timebeing.
The NUOPC pieces for MPAS will be added later, when the underlying atmospheric component work on MPAS is further along.

#if defined FRONT_CDEPS_DATM
if (trim(model) == "datm" ) then
!TODO: Remove bail code and pass info and SetVM to DriverAddComp
Expand Down
7 changes: 7 additions & 0 deletions tests/default_vars.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,7 @@ export POSTXCONFIG=postxconfig-NT-gfs.txt
export POSTXCONFIG_FH00=postxconfig-NT-gfs_FH00.txt

export FV3=true
export MPAS=true
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest export MPAS=false here in export_fv3 so both aren't true. Making a separate export_mpas () is one option. If you want many of the same settings while turning on MPAS-A, it may be more concise to set:

export_fv3
export FV3=false
export MPAS=true

in a regression test until there are defaults to set.

export S2S=false
export HAFS=false
export AQM=false
Expand Down Expand Up @@ -1221,6 +1222,7 @@ export_cmeps() {
export_cpl ()
{
export FV3=true
export MPAS=false
export S2S=true
export HAFS=false
export AQM=false
Expand Down Expand Up @@ -1414,6 +1416,7 @@ export_35d_run ()
export_datm_cdeps ()
{
export FV3=false
export MPAS=false
export S2S=false
export HAFS=false
export AQM=false
Expand Down Expand Up @@ -1492,6 +1495,7 @@ export_datm_cdeps ()
export_hafs_datm_cdeps ()
{
export FV3=false
export MPAS=false
export S2S=false
export HAFS=true
export AQM=false
Expand All @@ -1512,6 +1516,7 @@ export_hafs_datm_cdeps ()
export_hafs_docn_cdeps ()
{
export FV3=true
export MPAS=false
export S2S=false
export HAFS=true
export AQM=false
Expand All @@ -1532,6 +1537,7 @@ export_hafs_docn_cdeps ()
export_hafs_regional ()
{
export FV3=true
export MPAS=false
export S2S=false
export HAFS=true
export AQM=false
Expand Down Expand Up @@ -1606,6 +1612,7 @@ export_hafs ()
{
export_fv3_v16
export FV3=true
export MPAS=false
export S2S=false
export HAFS=true
export AQM=false
Expand Down
3 changes: 3 additions & 0 deletions tests/rt.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,3 +433,6 @@ RUN | datm_cdeps_control_cfsr | + hera hercules
### ATM-FBH test ###
COMPILE | atm_fbh | intel | -DAPP=ATMF -DCCPP_SUITES=FV3_HRRR -D32BIT=ON | - wcoss2 noaacloud acorn | fv3 |
RUN | cpld_regional_atm_fbh | - wcoss2 noaacloud acorn | baseline |

### UFS with MPAS dynamical core ###
COMPILE | atm_mpas_dyn32 | intel | -DAPP=ATMMPAS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_p8_ugwpv1 -D32BIT=ON | | mpas |
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do not see RUN step that executes the ufs-model executable built with this COMPILE step. How do we even know the mpas model is compiled correctly. Do you plan to add at least one regression test for ufs with mpas core?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, I see this question has been already asked in one of the previous comments, and the answer was no. I'm now confused, but I do not see what we get by making these changes in the ufs-weather-model at this time. We can discuss the corresponding PR in the fv3atm and if it's necessary merge it, but why updating ufs-wm without any tests is necessary

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

-D32BIT=ON is build flag used to build fv3 dycore using single-precision reals as a default. Is it used in MPAS core? I do not see it used in any mpas CMakeLists.txt. If it's not used, it should be removed from this COMPILE step.