Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flexible restart write times (restart_fh) for med, ocn, ice, wav #2419

Open
wants to merge 29 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 26 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
efb9646
Feature branches of CICE & CMEPS in .gitmodules
NickSzapiro-NOAA Aug 15, 2024
2bd58ee
Update hash: CMEPS & CICE
NickSzapiro-NOAA Aug 16, 2024
051ad6c
Merge branch 'ufs-community:develop' into restart_fh_med_ice
NickSzapiro-NOAA Aug 27, 2024
944454a
MOM6 restart_fh in .gitmodules
NickSzapiro-NOAA Aug 27, 2024
2a65120
fh_s=NINT(restart_fh*3600)
NickSzapiro-NOAA Aug 28, 2024
824706e
Update CMEPS hash
NickSzapiro-NOAA Sep 3, 2024
a24a0eb
Merge remote-tracking branch 'ufs-community/develop' into restart_fh_…
NickSzapiro-NOAA Sep 3, 2024
a570f32
Exercise restart_fh in datm_cdeps_gfs for med,ocn,ice
NickSzapiro-NOAA Sep 5, 2024
85a1dbe
update mom6 hash
NickSzapiro-NOAA Sep 12, 2024
e70b4a5
Merge remote-tracking branch 'ufs-community/develop' into restart_fh_…
NickSzapiro-NOAA Sep 12, 2024
3f22e99
ESMF_ConfigGetAttribute from model_configure
NickSzapiro-NOAA Sep 13, 2024
ba7eec6
Update MOM6 hash
NickSzapiro-NOAA Sep 18, 2024
12cf89c
Shared is_restart_fh module...moving into ufs/cdeps_share
NickSzapiro-NOAA Oct 15, 2024
a5525d4
Update is_restart_fh mode in cice_files.cmake
NickSzapiro-NOAA Oct 15, 2024
d190908
Update is_restart_fh mod in CMEPS-interface/CMakeLists.txt
NickSzapiro-NOAA Oct 15, 2024
2daced1
Update is_restart_fh mod in mom6_files.cmake
NickSzapiro-NOAA Oct 15, 2024
352623d
Update CICE, CMEPS, MOM6 hashes
NickSzapiro-NOAA Oct 16, 2024
0b9f6e0
Merge 'ufs-community/develop' into restart_fh_marine
NickSzapiro-NOAA Oct 16, 2024
ec854eb
Make write_restartfh public in shr_is_restart_fh_mod
NickSzapiro-NOAA Oct 16, 2024
ae98062
Update CICE hash
NickSzapiro-NOAA Oct 17, 2024
4a335f8
No need to add_dependencies(cdeps)
NickSzapiro-NOAA Oct 17, 2024
28e4930
Merge branch 'develop' into restart_fh_marine
NickSzapiro-NOAA Nov 13, 2024
901ed87
Default RESTART_FH is empty
NickSzapiro-NOAA Nov 14, 2024
e909d8e
Merge branch 'ufs-community:develop' into restart_fh_marine
NickSzapiro-NOAA Nov 15, 2024
9a8710e
Test restart_fh in cpld_control_p8 RT
NickSzapiro-NOAA Nov 15, 2024
c976b02
Add med,ocn,ice,wave restart_fh restarts to cpld_control_p8 baseline
NickSzapiro-NOAA Nov 15, 2024
9ecb1f0
Bound RESTART_FH in default_vars for all
NickSzapiro-NOAA Nov 18, 2024
6f01def
Revise is_restart_fh to avoid thread-local storage inter-component li…
NickSzapiro-NOAA Nov 19, 2024
aafd075
Tests now as expected on hercules
NickSzapiro-NOAA Nov 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
branch = develop
[submodule "WW3"]
path = WW3
url = https://github.com/NOAA-EMC/WW3
branch = dev/ufs-weather-model
url = https://github.com/NickSzapiro-NOAA/WW3
branch = restart_fh
[submodule "stochastic_physics"]
path = stochastic_physics
url = https://github.com/NOAA-PSL/stochastic_physics
Expand All @@ -16,20 +16,20 @@
branch = develop
[submodule "CMEPS"]
path = CMEPS-interface/CMEPS
url = https://github.com/NOAA-EMC/CMEPS
branch = emc/develop
url = https://github.com/NickSzapiro-NOAA/CMEPS
branch = restart_fh
[submodule "HYCOM"]
path = HYCOM-interface/HYCOM
url = https://github.com/NOAA-EMC/HYCOM-src
branch = emc/develop
[submodule "MOM6"]
path = MOM6-interface/MOM6
url = https://github.com/NOAA-EMC/MOM6
branch = dev/emc
url = https://github.com/NickSzapiro-NOAA/MOM6
branch = restart_fh
[submodule "CICE"]
path = CICE-interface/CICE
url = https://github.com/NOAA-EMC/CICE
branch = develop
url = https://github.com/NickSzapiro-NOAA/CICE
branch = restart_fh
[submodule "CDEPS"]
path = CDEPS-interface/CDEPS
url = https://github.com/NOAA-EMC/CDEPS
Expand Down
1 change: 1 addition & 0 deletions CDEPS-interface/cdeps_files.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ list(APPEND ufs_cdeps_share_files
ufs/cdeps_share/shr_assert_mod.F90
ufs/cdeps_share/shr_frz_mod.F90
ufs/cdeps_share/shr_infnan_mod.F90
ufs/cdeps_share/shr_is_restart_fh_mod.F90
)

list(APPEND cdeps_share_files
Expand Down
140 changes: 140 additions & 0 deletions CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
module shr_is_restart_fh_mod

! Common methods for components to check if it's time to write forecast hour-based restarts

!use dshr_methods_mod , only : chkerr
use ESMF, only : ESMF_ConfigCreate, ESMF_ConfigDestroy, ESMF_ConfigLoadFile, &
ESMF_ConfigGetLen, ESMF_ConfigGetAttribute, ESMF_TimePrint, &
ESMF_LOGMSG_INFO, ESMF_LogWrite, ESMF_TimeInterval, &
ESMF_Time, ESMF_KIND_R8, ESMF_Config, ESMF_Clock, &
ESMF_TimeIntervalSet, ESMF_TimePrint, operator(+), operator(==), &
ESMF_LogFoundError, ESMF_LOGERR_PASSTHRU

implicit none
private
save

logical :: write_restartfh = .false.
type(ESMF_Time), allocatable :: restartFhTimes(:)
!$OMP THREADPRIVATE (write_restartfh, restartFhTimes)

public :: init_is_restart_fh, is_restart_fh, finalize_restart_fh, write_restartfh

contains

!-----------------------------------------------------------------------
subroutine init_is_restart_fh(currentTime, dtime, lLog)
!
! !DESCRIPTION:
! Process restart_fh attribute from model_configure in UFS
!
! !USES:
!
! !ARGUMENTS:
type(ESMF_Time), intent(in) :: currentTime
integer, intent(in) :: dtime ! time step (s)
logical, intent(in) :: lLog ! If true, this task logs restart_fh info
!
! !LOCAL VARIABLES:
character(len=256) :: timestr
integer :: n, nfh, fh_s, rc
logical :: isPresent
real(kind=ESMF_KIND_R8), allocatable :: restart_fh(:)
type(ESMF_TimeInterval) :: fhInterval
type(ESMF_Config) :: CF_mc

character(len=*), parameter :: subname = 'init_restart_fh'
!-----------------------------------------------------------------------

! set up Times to write non-interval restarts
inquire(FILE='model_configure', EXIST=isPresent)
if (isPresent) then !model_configure exists. this is ufs run
CF_mc = ESMF_ConfigCreate(rc=rc)
call ESMF_ConfigLoadFile(config=CF_mc,filename='model_configure' ,rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
Copy link
Collaborator

@DeniseWorthen DeniseWorthen Oct 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why can't you use the cleaner form ? I see it commented out.

if (ChkErr(rc,__LINE__,u_FILE_u)) return

Add

use dshr_methods_mod , only : chkerr

character(*), parameter :: u_FILE_u = &
       __FILE__

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can, but that brings additional dependencies into this module besides ESMF. There are also quite some duplicate ChkErr functions floating around all just wrapping an ESMF function


nfh = ESMF_ConfigGetLen(config=CF_mc, label ='restart_fh:',rc=rc)
if (nfh .gt. 0) then
allocate(restart_fh(1:nfh))
allocate(restartFhTimes(1:nfh)) !not deallocated here

call ESMF_ConfigGetAttribute(CF_mc,valueList=restart_fh,label='restart_fh:', rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
! create a list of times at each restart_fh
do n = 1,nfh
fh_s = NINT(3600*restart_fh(n))
call ESMF_TimeIntervalSet(fhInterval, s=fh_s, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
restartFhTimes(n) = currentTime + fhInterval
call ESMF_TimePrint(restartFhTimes(n), options="string", &
preString="restart_fh at ", unit=timestr, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
if (lLog) then
if (mod(fh_s,dtime) /= 0) then
call ESMF_LogWrite('restart time NOT to be written for '//trim(timestr), ESMF_LOGMSG_INFO)
else
call ESMF_LogWrite('restart time to be written for '//trim(timestr), ESMF_LOGMSG_INFO)
end if
end if
end do
deallocate(restart_fh)
end if !nfh>0
call ESMF_ConfigDestroy(CF_mc, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
end if !model_configure

end subroutine init_is_restart_fh

function is_restart_fh(clock) result(lWrite)
!
! !DESCRIPTION:
! True/false if time to write restart
!
! !USES:
use ESMF, only : ESMF_ClockGetNextTime

!
! !ARGUMENTS:
type(ESMF_Clock), intent(in) :: clock
logical :: lWrite ! function result
!
! !LOCAL VARIABLES:
integer :: nfh, rc
type(ESMF_Time) :: nextTime

character(len=*), parameter :: subname = 'is_restart_fh'
!-----------------------------------------------------------------------

write_restartfh = .false.
if (allocated(restartFhTimes)) then
! check if next time is == to any restartfhtime
do nfh = 1,size(restartFhTimes)
call ESMF_ClockGetNextTime(clock, nextTime=nexttime, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
if (nextTime == restartFhTimes(nfh)) write_restartfh = .true.
end do
end if

lWrite = write_restartfh

end function is_restart_fh

subroutine finalize_restart_fh()
!
! !DESCRIPTION:
! Clean-up...release allocated memory
!
! !USES:
!
! !ARGUMENTS:
!
! !LOCAL VARIABLES:

character(len=*), parameter :: subname = 'finalize_restart_fh'
!-----------------------------------------------------------------------

if (allocated(restartFhTimes)) deallocate(restartFhTimes)

end subroutine finalize_restart_fh

end module shr_is_restart_fh_mod
2 changes: 1 addition & 1 deletion CICE-interface/CICE
2 changes: 1 addition & 1 deletion CICE-interface/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ list(APPEND lib_src_files
${icepack_files}
${cice_mpi_comm_files}
${cice_nuopc_cmeps_driver_files}
${cice_cdeps_inline_files})
${cice_cdeps_share_files})
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know this is a temporary solution. Does this mean that now cice/mom/ww3 caps now have dependency on the CDEPS share code?

Copy link
Collaborator Author

@NickSzapiro-NOAA NickSzapiro-NOAA Nov 15, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes.

CICE and CMEPS already have dependency for CDEPS inline functionality.

MOM6 and WW3 now have restart_fh module in cmakes. While this is currently under CDEPS-interface, it is a ufs-weather-model file (not a CDEPS subcomponent file)


list(APPEND _cice_defs FORTRANUNDERSCORE
coupled)
Expand Down
7 changes: 5 additions & 2 deletions CICE-interface/cice_files.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,8 @@ list(APPEND cice_nuopc_cmeps_driver_files
CICE/cicecore/drivers/nuopc/cmeps/ice_mesh_mod.F90
)

#-- Using ice prescribed ifndef cesmcoupled
list(APPEND cice_cdeps_inline_files
list(APPEND cice_cdeps_share_files
#-- Using ice prescribed ifndef cesmcoupled
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_orb_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_const_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_abort_mod.F90
Expand All @@ -168,4 +168,7 @@ list(APPEND cice_cdeps_inline_files
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_stream_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_methods_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/dshr/dshr_mod.F90

#restart_fh
${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90
)
2 changes: 1 addition & 1 deletion CMEPS-interface/CMEPS
3 changes: 2 additions & 1 deletion CMEPS-interface/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ list(APPEND _ufs_util_files
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_sys_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_kind_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_assert_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_infnan_mod.F90)
${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_infnan_mod.F90
${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90)

list(APPEND _mediator_files
CMEPS/mediator/med_phases_restart_mod.F90
Expand Down
2 changes: 1 addition & 1 deletion MOM6-interface/MOM6
4 changes: 4 additions & 0 deletions MOM6-interface/mom6_files.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,10 @@ list(APPEND mom6_nuopc_src_files
MOM6/config_src/drivers/timing_tests/time_MOM_EOS.F90
)

list(APPEND mom6_nuopc_src_files
${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90
)

list(APPEND mom6_solo_src_files
MOM6/config_src/drivers/solo_driver/MESO_surface_forcing.F90
MOM6/config_src/drivers/solo_driver/MOM_driver.F90
Expand Down
2 changes: 1 addition & 1 deletion WW3
1 change: 1 addition & 0 deletions tests/default_vars.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1358,6 +1358,7 @@ export_cmeps() {
export pio_rearranger=box
export RUNTYPE=startup
export RESTART_N=${FHMAX}
export RESTART_FH=" "
export CMEPS_RESTART_DIR=./RESTART/
export cap_dbug_flag=0
export WRITE_ENDOFRUN_RESTART=.false.
Expand Down
1 change: 1 addition & 0 deletions tests/parm/datm_cdeps_configure.IN
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ start_second: 0
nhours_fcst: @[FHMAX]
dt_atmos: @[DT_ATMOS]
fhrot: @[FHROT]
restart_fh: @[RESTART_FH]
1 change: 1 addition & 0 deletions tests/parm/model_configure.IN
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ fhrot: @[FHROT]

dt_atmos: @[DT_ATMOS]
restart_interval: @[RESTART_INTERVAL]
restart_fh: @[RESTART_FH]

quilting: @[QUILTING]
quilting_restart: @[QUILTING_RESTART]
Expand Down
7 changes: 6 additions & 1 deletion tests/tests/cpld_control_p8
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,14 @@ export LIST_FILES="sfcf021.tile1.nc \
RESTART/20210323.060000.sfc_data.tile4.nc \
RESTART/20210323.060000.sfc_data.tile5.nc \
RESTART/20210323.060000.sfc_data.tile6.nc \
RESTART/20210322.090000.MOM.res.nc \
RESTART/20210323.060000.MOM.res.nc \
RESTART/iced.2021-03-22-32400.nc \
RESTART/iced.2021-03-23-21600.nc \
RESTART/ufs.cpld.cpl.r.2021-03-22-32400.nc \
RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc \
ufs.cpld.ww3.r.2021-03-23-21600.nc \
ufs.cpld.ww3.r.2021-03-22-32400.nc \
ufs.cpld.ww3.r.2021-03-23-21600.nc \
20210323.060000.out_pnt.ww3 \
20210323.060000.out_grd.ww3 "
export DT_ATMOS=720
Expand All @@ -78,6 +82,7 @@ export CCPP_SUITE=FV3_GFS_v17_coupled_p8_ugwpv1
export RESTART_N=12
export RESTART_INTERVAL="${RESTART_N} -1"
export OUTPUT_FH='0 21 24'
export RESTART_FH="0.5 3"

export MOM6_TOPOEDITS=ufs.topo_edits_011818.nc
export MOM6_ALLOW_LANDMASK_CHANGES=True
Expand Down