diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 index 3302f61f4..68eb7d10e 100755 --- a/src/fesom_module.F90 +++ b/src/fesom_module.F90 @@ -289,7 +289,8 @@ subroutine fesom_init(fesom_total_nsteps) call clock_newyear ! check if it is a new year if (f%mype==0) f%t6=MPI_Wtime() !___CREATE NEW RESTART FILE IF APPLICABLE___________________________________ - call restart(0, 0, 0, r_restart, f%which_readr, f%ice, f%dynamics, f%tracers, f%partit, f%mesh) + + if (r_restart) call read_initial_conditions(0, 0, 0, f%which_readr, f%ice, f%dynamics, f%tracers, f%partit, f%mesh) if (f%mype==0) f%t7=MPI_Wtime() ! store grid information into netcdf file if (.not. r_restart) call write_mesh_info(f%partit, f%mesh) @@ -618,7 +619,7 @@ subroutine fesom_runloop(current_nsteps) !-------------------------- f%t5 = MPI_Wtime() - call restart(n, nstart, f%total_nsteps, .false., f%which_readr, f%ice, f%dynamics, f%tracers, f%partit, f%mesh) + call write_initial_conditions(n, nstart, f%total_nsteps, f%ice, f%dynamics, f%tracers, f%partit, f%mesh) f%t6 = MPI_Wtime() f%rtime_fullice = f%rtime_fullice + f%t2 - f%t1 @@ -790,4 +791,4 @@ subroutine fesom_finalize() ! call clock_finish end subroutine -end module +end module \ No newline at end of file diff --git a/src/gen_model_setup.F90 b/src/gen_model_setup.F90 index 13a669a3a..770323ec5 100755 --- a/src/gen_model_setup.F90 +++ b/src/gen_model_setup.F90 @@ -34,11 +34,13 @@ subroutine setup_model(partit) read (fileunit, NML=geometry) read (fileunit, NML=calendar) read (fileunit, NML=run_config) - read (fileunit,NML=icebergs) !!$ read (fileunit, NML=machine) close (fileunit) - + + ! by default use ResultPath for RestartPath + if (RestartPath == 'same-as-result') RestartPath = ResultPath + ! ========== ! compute dt diff --git a/src/gen_modules_clock.F90 b/src/gen_modules_clock.F90 index 0a287ab7e..3f529f482 100755 --- a/src/gen_modules_clock.F90 +++ b/src/gen_modules_clock.F90 @@ -83,7 +83,7 @@ subroutine clock_init(partit) yearstart=yearnew ! init clock for this run - open(99,file=trim(ResultPath)//trim(runid)//'.clock',status='old') + open(99,file=trim(RestartPath)//trim(runid)//'.clock',status='old') read(99,*) timeold, dayold, yearold read(99,*) timenew, daynew, yearnew close(99) @@ -211,4 +211,4 @@ end subroutine is_fleapyr ! !---------------------------------------------------------------------------- ! -end module g_clock +end module g_clock \ No newline at end of file diff --git a/src/gen_modules_config.F90 b/src/gen_modules_config.F90 index a12ff8c9a..dee26ef89 100755 --- a/src/gen_modules_config.F90 +++ b/src/gen_modules_config.F90 @@ -28,9 +28,10 @@ module g_config character(MAX_PATH) :: ClimateDataPath='./hydrography/' character(MAX_PATH) :: TideForcingPath='./tide_forcing/' character(MAX_PATH) :: ResultPath='./result/' + character(MAX_PATH) :: RestartPath='same-as-result' character(20) :: MeshId='NONE' namelist /paths/ MeshPath, ClimateDataPath, & - TideForcingPath, ResultPath, MeshId + TideForcingPath, ResultPath, RestartPath, MeshId !_____________________________________________________________________________ ! *** restart_log *** @@ -163,4 +164,4 @@ module g_config real(kind=WP) :: dummy=1.e10 -end module g_config +end module g_config \ No newline at end of file diff --git a/src/io_restart.F90 b/src/io_restart.F90 index fd1121278..de78d1f89 100644 --- a/src/io_restart.F90 +++ b/src/io_restart.F90 @@ -20,7 +20,7 @@ MODULE io_RESTART #endif implicit none - public :: restart, finalize_restart + public :: write_initial_conditions, read_initial_conditions, finalize_restart private integer, save :: globalstep=0 ! todo: remove this from module scope as it will mess things up if we use async read/write from the same process @@ -47,24 +47,19 @@ MODULE io_RESTART !-------------------------------------------------------------------------------------------- ! ini_ocean_io initializes ocean_file datatype which contains information of all variables need to be written into ! the ocean restart file. This is the only place need to be modified if a new variable is added! -subroutine ini_ocean_io(year, dynamics, tracers, partit, mesh) +subroutine ini_ocean_io(dynamics, tracers, partit, mesh) #ifdef ENABLE_NVHPC_WORKAROUNDS use nvfortran_subarray_workaround_module #endif - integer, intent(in) :: year integer :: j character(500) :: longname character(500) :: trname, units - character(4) :: cyear type(t_mesh), target :: mesh type(t_partit), intent(inout), target :: partit type(t_tracer), target :: tracers type(t_dyn), target :: dynamics logical, save :: has_been_called = .false. - write(cyear,'(i4)') year - oce_path = trim(ResultPath)//trim(runid)//'.'//cyear//'.oce.restart.nc' - if(has_been_called) return has_been_called = .true. @@ -153,17 +148,12 @@ end subroutine ini_ocean_io !-------------------------------------------------------------------------------------------- ! ini_ice_io initializes ice_file datatype which contains information of all variables need to be written into ! the ice restart file. This is the only place need to be modified if a new variable is added! -subroutine ini_ice_io(year, ice, partit, mesh) - integer, intent(in) :: year - character(4) :: cyear +subroutine ini_ice_io(ice, partit, mesh) type(t_mesh), intent(in) , target :: mesh type(t_partit), intent(inout), target :: partit type(t_ice), target :: ice logical, save :: has_been_called = .false. - write(cyear,'(i4)') year - ice_path = trim(ResultPath)//trim(runid)//'.'//cyear//'.ice.restart.nc' - if(has_been_called) return has_been_called = .true. @@ -198,20 +188,16 @@ end subroutine ini_ice_io ! ini_bio_io initializes bid datatype which contains information of all variables need to be written into ! the bio restart file. This is the only place need to be modified if a new variable is added! #if defined(__recom) -subroutine ini_bio_io(year, tracers, partit, mesh) - integer, intent(in) :: year +subroutine ini_bio_io(tracers, partit, mesh) integer :: j character(500) :: longname character(500) :: trname, units - character(4) :: cyear type(t_mesh), intent(in) , target :: mesh type(t_partit), intent(inout), target :: partit type(t_tracer), target :: tracers logical, save :: has_been_called = .false. - write(cyear,'(i4)') year - bio_path = trim(ResultPath)//trim(runid)//'.'//cyear//'.bio.restart.nc' if(has_been_called) return has_been_called = .true. @@ -231,37 +217,152 @@ end subroutine ini_bio_io ! !-------------------------------------------------------------------------------------------- ! -subroutine restart(istep, nstart, ntotal, l_read, which_readr, ice, dynamics, tracers, partit, mesh) +!! This is the main restart subroutine +!! We check the different restart options (raw, binary, netcdf) in that order and try loading the first one +subroutine read_initial_conditions(istep, nstart, ntotal, which_readr, ice, dynamics, tracers, partit, mesh) #if defined(__icepack) icepack restart not merged here ! produce a compiler error if USE_ICEPACK=ON; todo: merge icepack restart from 68d8b8b #endif use fortran_utils implicit none - ! this is the main restart subroutine - ! if l_read is TRUE the restart file will be read integer :: istep, nstart, ntotal - logical :: l_read logical :: is_portable_restart_write, is_raw_restart_write, is_bin_restart_write type(t_mesh) , intent(inout), target :: mesh type(t_partit), intent(inout), target :: partit type(t_tracer), intent(inout), target :: tracers type(t_dyn) , intent(inout), target :: dynamics type(t_ice) , intent(inout), target :: ice - logical rawfiles_exist, binfiles_exist - logical, save :: initialized_raw = .false. - logical, save :: initialized_bin = .false. - integer mpierr - !which_readr = ... ! 0 ... read netcdf restart ! 1 ... read dump file restart (binary) ! 2 ... read derived type restart (binary) integer, intent(out):: which_readr + + logical rawfiles_exist, binfiles_exist + integer mpierr + integer :: cstep + character(4) :: cyear !! used in string for oce_path + + + ! compute current time based on what is written in fesom.clock file + ctime=timeold+(dayold-1.)*86400 + + raw_restart_dirpath = trim(RestartPath)//"fesom_raw_restart/np"//int_to_txt(partit%npes) + raw_restart_infopath = trim(RestartPath)//"fesom_raw_restart/np"//int_to_txt(partit%npes)//".info" + bin_restart_dirpath = trim(RestartPath)//"fesom_bin_restart/np"//int_to_txt(partit%npes) + bin_restart_infopath = trim(RestartPath)//"fesom_bin_restart/np"//int_to_txt(partit%npes)//".info" + + write(cyear,'(i4)') yearold + oce_path = trim(RestartPath)//trim(runid)//'.'//cyear//'.oce.restart.nc' + ice_path = trim(RestartPath)//trim(runid)//'.'//cyear//'.ice.restart.nc' +#if defined(__recom) + bio_path = trim(RestartPath)//trim(runid)//'.'//cyear//'.bio.restart.nc' +#endif + + call ini_ocean_io(dynamics, tracers, partit, mesh) + if (use_ice) call ini_ice_io (ice, partit, mesh) +#if defined(__recom) + if (use_REcoM) call ini_bio_io (tracers, partit, mesh) +#endif + + ! determine if we can load raw restart dump files --> check if *.info file for + ! raw restarts exist --> if info file exist also the rest must exist --> so + ! core dump restart is readable + if(partit%mype == RAW_RESTART_METADATA_RANK) then + inquire(file=raw_restart_infopath, exist=rawfiles_exist) + end if + call MPI_Bcast(rawfiles_exist, 1, MPI_LOGICAL, RAW_RESTART_METADATA_RANK, partit%MPI_COMM_FESOM, mpierr) + + ! check if folder for derived type binary restarts exist + if(partit%mype == RAW_RESTART_METADATA_RANK) then + inquire(file=bin_restart_infopath, exist=binfiles_exist) + end if + call MPI_Bcast(binfiles_exist, 1, MPI_LOGICAL, RAW_RESTART_METADATA_RANK, partit%MPI_COMM_FESOM, mpierr) + !___________________________________________________________________________ + ! read core dump file restart + if(rawfiles_exist) then + which_readr = 1 + call read_all_raw_restarts(partit%MPI_COMM_FESOM, partit%mype) + + !___________________________________________________________________________ + ! read derived type binary file restart + elseif(binfiles_exist .and. bin_restart_length_unit /= "off") then + which_readr = 2 + if (use_ice) then + call read_all_bin_restarts(bin_restart_dirpath, & + partit = partit, & + mesh = mesh, & + ice = ice, & + dynamics = dynamics, & + tracers = tracers ) + else + call read_all_bin_restarts(bin_restart_dirpath, & + partit = partit, & + mesh = mesh, & + dynamics = dynamics, & + tracers = tracers ) + end if + !___________________________________________________________________________ + ! read netcdf file restart + else + which_readr = 0 + if (partit%mype==RAW_RESTART_METADATA_RANK) print *, achar(27)//'[1;33m'//' --> read restarts from netcdf file: ocean'//achar(27)//'[0m' + call read_restart(oce_path, oce_files, partit%MPI_COMM_FESOM, partit%mype) + if (use_ice) then + if (partit%mype==RAW_RESTART_METADATA_RANK) print *, achar(27)//'[1;33m'//' --> read restarts from netcdf file: ice'//achar(27)//'[0m' + call read_restart(ice_path, ice_files, partit%MPI_COMM_FESOM, partit%mype) + end if + +#if defined(__recom) +!ROM restart +!rd here + if (REcoM_restart) then + if (partit%mype==RAW_RESTART_METADATA_RANK) print *, achar(27)//'[1;33m'//' --> read restarts from netcdf file: bio'//achar(27)//'[0m' + call read_restart(bio_path, bio_files, partit%MPI_COMM_FESOM, partit%mype) + end if +#endif + + end if + +end subroutine read_initial_conditions + +!! this is the main restart writing routine, now named write_initial_conditions +subroutine write_initial_conditions(istep, nstart, ntotal, ice, dynamics, tracers, partit, mesh) + use fortran_utils + implicit none + + integer :: istep, nstart, ntotal + logical :: is_portable_restart_write, is_raw_restart_write, is_bin_restart_write + type(t_mesh) , intent(inout), target :: mesh + type(t_partit), intent(inout), target :: partit + type(t_tracer), intent(inout), target :: tracers + type(t_dyn) , intent(inout), target :: dynamics + type(t_ice) , intent(inout), target :: ice + logical rawfiles_exist, binfiles_exist + logical, save :: initialized_raw = .false. + logical, save :: initialized_bin = .false. + integer mpierr + character(4) :: cyear !! used in string for oce_path integer :: cstep + + ! now for writing we need to change the paths for the restart files + write(cyear,'(i4)') yearnew + oce_path = trim(ResultPath)//trim(runid)//'.'//cyear//'.oce.restart.nc' + ice_path = trim(ResultPath)//trim(runid)//'.'//cyear//'.ice.restart.nc' +#if defined(__recom) + bio_path = trim(ResultPath)//trim(runid)//'.'//cyear//'.bio.restart.nc' +#endif + + call ini_ocean_io(dynamics, tracers, partit, mesh) + if (use_ice) call ini_ice_io (ice, partit, mesh) +#if defined(__recom) + if (use_REcoM) call ini_bio_io (tracers, partit, mesh) +#endif + !_____________________________________________________________________________ ! initialize directory for core dump restart if(.not. initialized_raw) then @@ -298,105 +399,7 @@ subroutine restart(istep, nstart, ntotal, l_read, which_readr, ice, dynamics, tr ! compute current time based on what is written in fesom.clock file ctime=timeold+(dayold-1.)*86400 - !_____________________________________________________________________________ - ! initialise files for netcdf restart if l_read==TRUE --> the restart file - ! will be read - if (.not. l_read) then - call ini_ocean_io(yearnew, dynamics, tracers, partit, mesh) - if (use_ice) call ini_ice_io (yearnew, ice, partit, mesh) -#if defined(__recom) - if (use_REcoM) call ini_bio_io (yearnew, tracers, partit, mesh) -#endif - else - call ini_ocean_io(yearold, dynamics, tracers, partit, mesh) - if (use_ice) call ini_ice_io (yearold, ice, partit, mesh) -#if defined(__recom) - if (REcoM_restart) call ini_bio_io(yearold, tracers, partit, mesh) -#endif - end if - !___READING OF RESTART________________________________________________________ - ! should restart files be readed --> see r_restart in gen_modules_clock.F90 - if (l_read) then - ! determine if we can load raw restart dump files --> check if *.info file for - ! raw restarts exist --> if info file exist also the rest must exist --> so - ! core dump restart is readable - if(partit%mype == RAW_RESTART_METADATA_RANK) then - inquire(file=raw_restart_infopath, exist=rawfiles_exist) - end if - call MPI_Bcast(rawfiles_exist, 1, MPI_LOGICAL, RAW_RESTART_METADATA_RANK, partit%MPI_COMM_FESOM, mpierr) - - ! check if folder for derived type binary restarts exist - if(partit%mype == RAW_RESTART_METADATA_RANK) then - inquire(file=bin_restart_infopath, exist=binfiles_exist) - end if - call MPI_Bcast(binfiles_exist, 1, MPI_LOGICAL, RAW_RESTART_METADATA_RANK, partit%MPI_COMM_FESOM, mpierr) - - !___________________________________________________________________________ - ! read core dump file restart - if(rawfiles_exist) then - which_readr = 1 - call read_all_raw_restarts(partit%MPI_COMM_FESOM, partit%mype) - - !___________________________________________________________________________ - ! read derived type binary file restart - elseif(binfiles_exist .and. bin_restart_length_unit /= "off") then - which_readr = 2 - if (use_ice) then - call read_all_bin_restarts(bin_restart_dirpath, & - partit = partit, & - mesh = mesh, & - ice = ice, & - dynamics = dynamics, & - tracers = tracers ) - else - call read_all_bin_restarts(bin_restart_dirpath, & - partit = partit, & - mesh = mesh, & - dynamics = dynamics, & - tracers = tracers ) - end if - !___________________________________________________________________________ - ! read netcdf file restart - else - which_readr = 0 - if (partit%mype==RAW_RESTART_METADATA_RANK) print *, achar(27)//'[1;33m'//' --> read restarts from netcdf file: ocean'//achar(27)//'[0m' - call read_restart(oce_path, oce_files, partit%MPI_COMM_FESOM, partit%mype) - if (use_ice) then - if (partit%mype==RAW_RESTART_METADATA_RANK) print *, achar(27)//'[1;33m'//' --> read restarts from netcdf file: ice'//achar(27)//'[0m' - call read_restart(ice_path, ice_files, partit%MPI_COMM_FESOM, partit%mype) - end if - -#if defined(__recom) -!RECOM restart -!read here - if (REcoM_restart) then - if (partit%mype==RAW_RESTART_METADATA_RANK) print *, achar(27)//'[1;33m'//' --> read restarts from netcdf file: bio'//achar(27)//'[0m' - call read_restart(bio_path, bio_files, partit%MPI_COMM_FESOM, partit%mype) - end if -#endif - - ! immediately create a raw core dump restart - if(raw_restart_length_unit /= "off") then - call write_all_raw_restarts(istep, partit%MPI_COMM_FESOM, partit%mype) - end if - - ! immediately create a derived type binary restart - if(bin_restart_length_unit /= "off") then - ! current (total) model step --> cstep = globalstep+istep - call write_all_bin_restarts((/globalstep+istep, int(ctime), yearnew/), & - bin_restart_dirpath, & - bin_restart_infopath, & - partit, & - mesh, & - ice, & - dynamics, & - tracers ) - end if - end if - end if - - if (istep==0) return !___WRITING OF RESTART________________________________________________________ ! check whether restart will be written @@ -470,7 +473,7 @@ subroutine restart(istep, nstart, ntotal, l_read, which_readr, ice, dynamics, tr end if end if -end subroutine restart +end subroutine write_initial_conditions ! ! !_______________________________________________________________________________ @@ -877,6 +880,8 @@ subroutine read_restart(path, filegroup, mpicomm, mype) allocate(skip_file(filegroup%nfiles)) skip_file = .false. + + write(*,*) path do i=1, filegroup%nfiles current_iorank_snd = 0