From 538ec00c4cb64c42cad70af9e84999ff3e89379a Mon Sep 17 00:00:00 2001 From: Jan Hegewald Date: Fri, 31 Mar 2023 11:13:17 +0200 Subject: [PATCH 1/4] do not disable multithreading on levante, DKRZ made a configuration error which prevented threads to run properly on compute nodes, this has been solved (see Ticket#124596) --- src/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7f6ce5300..56e95bcea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -35,10 +35,7 @@ if(ALBEDO_INTELMPI_WORKAROUNDS) add_compile_options(-DDISABLE_PARALLEL_RESTART_READ) endif() -if(${FESOM_PLATFORM_STRATEGY} STREQUAL levante.dkrz.de ) - message(STATUS "multithreading disabled for Levante") # multithreading suddenly produces an error, disable it until a fix is found. issue #413 - option(DISABLE_MULTITHREADING "disable asynchronous operations" ON) -elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo ) +if(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo ) message(STATUS "multithreading disabled for Albedo") # multithreading suddenly produces an error, disable it until a fix is found. issue #413 option(DISABLE_MULTITHREADING "disable asynchronous operations" ON) else() From b0c882857d1c01d9553b7cdc86d884ef95214252 Mon Sep 17 00:00:00 2001 From: Jan Hegewald Date: Fri, 31 Mar 2023 11:15:02 +0200 Subject: [PATCH 2/4] for ifort, solve Fesom jobs get stuck on levante if both I/O multithreading and openmp are active --- src/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 56e95bcea..08a4d2f44 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -196,6 +196,18 @@ set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) if(${ENABLE_OPENMP} AND NOT ${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray) target_compile_options(${PROJECT_NAME} PRIVATE ${OpenMP_Fortran_FLAGS}) # currently we only have OpenMP in the Fortran part target_link_libraries(${PROJECT_NAME} OpenMP::OpenMP_Fortran) + + # there is a problem (at least on levante with ifort) with openmp and I/O multithreading being active at the same time + # in this case the simulation stops and never finishes + # this has been observed with IFS-Fesom and Fesom standalone + # until the problem is identified, we exclude the I/O and multithreading files from openmp compilation, i.e. "-qno-openmp" for ifort + if(NOT ${DISABLE_MULTITHREADING}) + if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel ) + set_source_files_properties(${CMAKE_CURRENT_LIST_DIR}/io_restart.F90 PROPERTIES COMPILE_OPTIONS "-qno-openmp") + set_source_files_properties(${CMAKE_CURRENT_LIST_DIR}/async_threads_module.F90 PROPERTIES COMPILE_OPTIONS "-qno-openmp") + set_source_files_properties(${CMAKE_CURRENT_LIST_DIR}/io_meandata.F90 PROPERTIES COMPILE_OPTIONS "-qno-openmp") + endif() + endif() endif() From 2ab4f8a5cd9285053fc7501abafbe03a12deabd7 Mon Sep 17 00:00:00 2001 From: Thomas Rackow Date: Tue, 11 Apr 2023 10:31:07 +0200 Subject: [PATCH 3/4] avoid potential heavy asynchronization at high CPU numbers before MPI call in compute_sigma_xy() --- src/oce_ale_pressure_bv.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/oce_ale_pressure_bv.F90 b/src/oce_ale_pressure_bv.F90 index 5cf1774d7..f6084acb3 100644 --- a/src/oce_ale_pressure_bv.F90 +++ b/src/oce_ale_pressure_bv.F90 @@ -3176,6 +3176,7 @@ subroutine compute_sigma_xy(TF1,SF1, partit, mesh) END DO !$OMP END DO !$OMP END PARALLEL + CALL MPI_BARRIER(MPI_COMM_FESOM,MPIerr) call exchange_nod(sigma_xy, partit) !$OMP BARRIER end subroutine compute_sigma_xy From f2676d3cf2fb05e3a7eb9c6c8555ceb81a4d5f52 Mon Sep 17 00:00:00 2001 From: Thomas Rackow Date: Thu, 13 Apr 2023 12:42:29 +0200 Subject: [PATCH 4/4] change print statement in nemogcmcoup_update_add() dummy routine --- src/ifs_interface/ifs_notused.F90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ifs_interface/ifs_notused.F90 b/src/ifs_interface/ifs_notused.F90 index bc711a8c6..f1501e144 100644 --- a/src/ifs_interface/ifs_notused.F90 +++ b/src/ifs_interface/ifs_notused.F90 @@ -202,8 +202,7 @@ SUBROUTINE nemogcmcoup_update_add( mype, npes, icomm, & ! Local variables if(fesom%mype==0) then - WRITE(0,*)'nemogcmcoup_update_add should not be called when coupling to fesom. Commented ABORT. Proceeding...' - !CALL abort + WRITE(0,*)'In nemogcmcoup_update_add FESOM dummy routine. Proceeding...' endif END SUBROUTINE nemogcmcoup_update_add