From 08c693a9ebf1f96f41b3bfb3b2285581e5177a4d Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Mon, 7 Oct 2024 10:48:06 -0400 Subject: [PATCH 1/5] Setting the compiler for the kokkos testing code to what is used for Kokkos --- testing/adios2/engine/bp/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index b17438bdfb..8dbdb07b4b 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -280,6 +280,10 @@ gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .FileStream ) if(ADIOS2_HAVE_Kokkos) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + gtest_add_tests_helper(WriteReadKokkos MPI_ALLOW BP Engine.BP. .BP5 WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" ) From 7a4ffb088bea5d908f609537224fd0f4e8984725 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Mon, 7 Oct 2024 13:42:49 -0400 Subject: [PATCH 2/5] Removing Kokkos::printf until we move to version 4.2 --- testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp b/testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp index 60a3af71ef..6c7b520750 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp @@ -367,8 +367,9 @@ bool compareSelection2D( if (b(x, y) != a(start_0 + x, start_1 + y)) { lmatch++; - Kokkos::printf(" Non-match at pos = (%d %d) : input = %f : output = %f\n", x, y, - a(start_0 + x, start_1 + y), b(x, y)); + // Add lines back when using Kokkos 4.2 + // Kokkos::printf(" Non-match at pos = (%d %d) : input = %f : output = %f\n", x, + // y, a(start_0 + x, start_1 + y), b(x, y)); } }, match); From 79e2669d6b66caa239713462385c174640b7b226 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Fri, 11 Oct 2024 21:41:30 -0400 Subject: [PATCH 3/5] Move the Kokkos testing to a separate folder to compile the gpu code separately --- testing/adios2/CMakeLists.txt | 4 ++++ testing/adios2/engine/bp/CMakeLists.txt | 14 -------------- testing/adios2/gpu-backend/CMakeLists.txt | 19 +++++++++++++++++++ .../TestBPWriteReadKokkos.cpp | 2 ++ 4 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 testing/adios2/gpu-backend/CMakeLists.txt rename testing/adios2/{engine/bp => gpu-backend}/TestBPWriteReadKokkos.cpp (99%) diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt index f6ff7c0afd..75333117f3 100644 --- a/testing/adios2/CMakeLists.txt +++ b/testing/adios2/CMakeLists.txt @@ -21,3 +21,7 @@ endif() if (ADIOS2_HAVE_Python) add_subdirectory(python) endif() + +if(ADIOS2_HAVE_Kokkos) +add_subdirectory(gpu-backend) +endif() diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index 8dbdb07b4b..dbbce8540c 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -278,17 +278,3 @@ gtest_add_tests_helper(StepsInSituGlobalArray MPI_ALLOW BP Engine.BP. .FileStrea gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .FileStream WORKING_DIRECTORY ${FS_DIR} EXTRA_ARGS "FileStream" ) - -if(ADIOS2_HAVE_Kokkos) - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") - - gtest_add_tests_helper(WriteReadKokkos MPI_ALLOW BP Engine.BP. .BP5 - WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" - ) - - foreach(tgt ${Test.Engine.BP.WriteReadKokkos-TARGETS}) - target_link_libraries(${tgt} Kokkos::kokkos) - endforeach() -endif() diff --git a/testing/adios2/gpu-backend/CMakeLists.txt b/testing/adios2/gpu-backend/CMakeLists.txt new file mode 100644 index 0000000000..7ea38b99e4 --- /dev/null +++ b/testing/adios2/gpu-backend/CMakeLists.txt @@ -0,0 +1,19 @@ +# ------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# ------------------------------------------------------------------------------# +include(ADIOSFunctions) + + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + if(DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + endif() + + gtest_add_tests_helper(WriteReadKokkos MPI_ALLOW BP Engine.BP. .BP5 + WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" + ) + + foreach(tgt ${Test.Engine.BP.WriteReadKokkos-TARGETS}) + target_link_libraries(${tgt} Kokkos::kokkos) + endforeach() diff --git a/testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp b/testing/adios2/gpu-backend/TestBPWriteReadKokkos.cpp similarity index 99% rename from testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp rename to testing/adios2/gpu-backend/TestBPWriteReadKokkos.cpp index 6c7b520750..ad9fda35cf 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadKokkos.cpp +++ b/testing/adios2/gpu-backend/TestBPWriteReadKokkos.cpp @@ -521,6 +521,7 @@ int main(int argc, char **argv) // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif + Kokkos::initialize(argc, argv); int result; ::testing::InitGoogleTest(&argc, argv); @@ -530,6 +531,7 @@ int main(int argc, char **argv) } result = RUN_ALL_TESTS(); + Kokkos::finalize(); #if ADIOS2_USE_MPI MPI_Finalize(); #endif From 6aed25a119f3e4a11c39013bebdba33091116340 Mon Sep 17 00:00:00 2001 From: Ana Gainaru Date: Fri, 11 Oct 2024 21:42:29 -0400 Subject: [PATCH 4/5] Update the rocm and Kokkos versions for the Frontier CI --- .gitlab/gitlab-ci-frontier.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/gitlab-ci-frontier.yml b/.gitlab/gitlab-ci-frontier.yml index abff467f37..c68377f7f0 100644 --- a/.gitlab/gitlab-ci-frontier.yml +++ b/.gitlab/gitlab-ci-frontier.yml @@ -101,7 +101,7 @@ stages: PrgEnv-gnu gcc-native/12 craype-accel-amd-gfx90a - rocm/5.4.3 + rocm/5.7.1 cmake git ninja @@ -111,10 +111,10 @@ stages: setup:frontier-kokkos-hip: variables: - KOKKOS_VER: 3.7.01 + KOKKOS_VER: 4.4.01 KOKKOS_OPTS: >- -DCMAKE_INSTALL_PREFIX:PATH=$Kokkos_DIR - -DCMAKE_CXX_COMPILER:FILEPATH=/opt/rocm-5.4.3/hip/bin/hipcc + -DCMAKE_CXX_COMPILER:FILEPATH=/opt/rocm-5.7.1/hip/bin/hipcc -DKokkos_ARCH_VEGA90A:BOOL=ON -DKokkos_ENABLE_HIP:BOOL=ON -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE:BOOL=OFF From b17c40c9fb7b5beab14138bd4ace3356ee863d92 Mon Sep 17 00:00:00 2001 From: anagainaru Date: Tue, 15 Oct 2024 13:21:30 -0400 Subject: [PATCH 5/5] Format fixes and cleaning Co-authored-by: Vicente Adolfo Bolea Sanchez --- testing/adios2/gpu-backend/CMakeLists.txt | 24 +++++++++---------- .../gpu-backend/TestBPWriteReadKokkos.cpp | 3 --- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/testing/adios2/gpu-backend/CMakeLists.txt b/testing/adios2/gpu-backend/CMakeLists.txt index 7ea38b99e4..e3270bc6a2 100644 --- a/testing/adios2/gpu-backend/CMakeLists.txt +++ b/testing/adios2/gpu-backend/CMakeLists.txt @@ -2,18 +2,16 @@ # Distributed under the OSI-approved Apache License, Version 2.0. See # accompanying file Copyright.txt for details. # ------------------------------------------------------------------------------# -include(ADIOSFunctions) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +if(DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") +endif() - set(CMAKE_CXX_STANDARD 17) - set(CMAKE_CXX_STANDARD_REQUIRED ON) - if(DEFINED Kokkos_CXX_COMPILER) - set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") - endif() +gtest_add_tests_helper(WriteReadKokkos MPI_ALLOW BP Engine.BP. .BP5 + WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" +) - gtest_add_tests_helper(WriteReadKokkos MPI_ALLOW BP Engine.BP. .BP5 - WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" - ) - - foreach(tgt ${Test.Engine.BP.WriteReadKokkos-TARGETS}) - target_link_libraries(${tgt} Kokkos::kokkos) - endforeach() +foreach(tgt ${Test.Engine.BP.WriteReadKokkos-TARGETS}) + target_link_libraries(${tgt} Kokkos::kokkos) +endforeach() diff --git a/testing/adios2/gpu-backend/TestBPWriteReadKokkos.cpp b/testing/adios2/gpu-backend/TestBPWriteReadKokkos.cpp index ad9fda35cf..a69915b593 100644 --- a/testing/adios2/gpu-backend/TestBPWriteReadKokkos.cpp +++ b/testing/adios2/gpu-backend/TestBPWriteReadKokkos.cpp @@ -367,9 +367,6 @@ bool compareSelection2D( if (b(x, y) != a(start_0 + x, start_1 + y)) { lmatch++; - // Add lines back when using Kokkos 4.2 - // Kokkos::printf(" Non-match at pos = (%d %d) : input = %f : output = %f\n", x, - // y, a(start_0 + x, start_1 + y), b(x, y)); } }, match);