run each tridiagonal solver test multiple times and take average of t…

…he metrics being recorded
NCAR · Jun 18, 2024 · c74f3f2 · c74f3f2
1 parent c2d6472
commit c74f3f2
Show file tree

Hide file tree

Showing 18 changed files with 301 additions and 266 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -155,7 +155,6 @@ if(PROJECT_IS_TOP_LEVEL AND TUVX_ENABLE_TESTS)
 
   enable_testing()
   add_subdirectory(test)
-  add_subdirectory(benchmark)
 endif()
 
 # ##############################################################################

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
diff --git a/cmake/benchmark_util.cmake b/cmake/benchmark_util.cmake
diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake
@@ -83,15 +83,7 @@ if(TUVX_ENABLE_TESTS)
       OFF
       CACHE BOOL "" FORCE)
 
-  # google benchmark
-
-  FetchContent_Declare(
-    googlebenchmark
-    GIT_REPOSITORY https://github.com/google/benchmark.git
-    GIT_TAG main) # need main for benchmark::benchmark
-
-  FetchContent_MakeAvailable(googletest googlebenchmark)
-
+  FetchContent_MakeAvailable(googletest)
   # don't run clang-tidy on google test
   set_target_properties(gtest PROPERTIES CXX_CLANG_TIDY "")
   set_target_properties(gtest_main PROPERTIES CXX_CLANG_TIDY "")

diff --git a/cmake/test_util.cmake b/cmake/test_util.cmake
@@ -1,32 +1,36 @@
-################################################################################
+# ##############################################################################
 # Utility functions for creating tests
 
 if(TUVX_ENABLE_MEMCHECK)
   find_program(MEMORYCHECK_COMMAND "valgrind")
 endif()
 
-################################################################################
+# ##############################################################################
 # impose that one test runs after another so that we can safely test in parallel
 
 function(add_test_dependency run_second run_first)
-    # add dependency between two tests
-    # https://stackoverflow.com/a/66931930/5217293
-    set_tests_properties(${run_first}  PROPERTIES FIXTURES_SETUP    f_${run_first})
-    set_tests_properties(${run_second} PROPERTIES FIXTURES_REQUIRED f_${run_first})
+  # add dependency between two tests
+  # https://stackoverflow.com/a/66931930/5217293
+  set_tests_properties(${run_first} PROPERTIES FIXTURES_SETUP f_${run_first})
+  set_tests_properties(${run_second} PROPERTIES FIXTURES_REQUIRED
+                                                f_${run_first})
 endfunction(add_test_dependency)
 
-################################################################################
+# ##############################################################################
 # build and add a standard test (one linked to the tuvx library)
 
 function(create_standard_test)
   set(prefix TEST)
   set(singleValues NAME WORKING_DIRECTORY)
   set(multiValues SOURCES)
   include(CMakeParseArguments)
-  cmake_parse_arguments(${prefix} " " "${singleValues}" "${multiValues}" ${ARGN})
+  cmake_parse_arguments(${prefix} " " "${singleValues}" "${multiValues}"
+                        ${ARGN})
   add_executable(test_${TEST_NAME} ${TEST_SOURCES})
   set_target_properties(test_${TEST_NAME} PROPERTIES LINKER_LANGUAGE Fortran)
-  target_link_libraries(test_${TEST_NAME} PUBLIC musica::tuvx tuvx_test_utils ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} GTest::gtest_main)
+  target_link_libraries(
+    test_${TEST_NAME} PUBLIC musica::tuvx tuvx_test_utils ${BLAS_LIBRARIES}
+                             ${LAPACK_LIBRARIES} GTest::gtest_main)
   if(TUVX_ENABLE_OPENMP)
     target_link_libraries(test_${TEST_NAME} PUBLIC OpenMP::OpenMP_Fortran)
   endif()
@@ -36,7 +40,7 @@ function(create_standard_test)
   add_tuvx_test(${TEST_NAME} test_${TEST_NAME} "" ${TEST_WORKING_DIRECTORY})
 endfunction(create_standard_test)
 
-################################################################################
+# ##############################################################################
 # build and add a standard test (one linked to the tuvx library)
 
 function(create_standard_cxx_test)
@@ -46,13 +50,15 @@ function(create_standard_cxx_test)
   set(multiValues SOURCES LIBRARIES)
 
   include(CMakeParseArguments)
-  cmake_parse_arguments(${prefix} "${optionalValues}" "${singleValues}" "${multiValues}" ${ARGN})
+  cmake_parse_arguments(${prefix} "${optionalValues}" "${singleValues}"
+                        "${multiValues}" ${ARGN})
 
   add_executable(test_${TEST_NAME} ${TEST_SOURCES})
   target_include_directories(test_${TEST_NAME} PUBLIC ${LAPACK_INCLUDE_DIRS})
-  target_link_libraries(test_${TEST_NAME} PUBLIC LAPACK::LAPACK musica::tuvx GTest::gtest_main)
+  target_link_libraries(test_${TEST_NAME} PUBLIC LAPACK::LAPACK musica::tuvx
+                                                 GTest::gtest_main)
   message(STATUS ${OpenBLAS_INCLUDE_DIRS})
-   
+
   # link additional libraries
   foreach(library ${TEST_LIBRARIES})
     target_link_libraries(test_${TEST_NAME} PUBLIC ${library})
@@ -62,52 +68,68 @@ function(create_standard_cxx_test)
     set(TEST_WORKING_DIRECTORY "${CMAKE_BINARY_DIR}")
   endif()
 
-  add_tuvx_test(${TEST_NAME} test_${TEST_NAME} "" ${TEST_WORKING_DIRECTORY} ${TEST_SKIP_MEMCHECK})
+  add_tuvx_test(${TEST_NAME} test_${TEST_NAME} "" ${TEST_WORKING_DIRECTORY}
+                ${TEST_SKIP_MEMCHECK})
 endfunction(create_standard_cxx_test)
 
-################################################################################
+# ##############################################################################
 # Add a test
 
 function(add_tuvx_test test_name test_binary test_args working_dir)
   if(TUVX_ENABLE_MPI)
-    add_test(NAME ${test_name}
+    add_test(
+      NAME ${test_name}
       COMMAND mpirun -v -np 2 ${CMAKE_BINARY_DIR}/${test_binary} ${test_args}
-             WORKING_DIRECTORY ${working_dir})
+      WORKING_DIRECTORY ${working_dir})
   else()
-    add_test(NAME ${test_name}
-             COMMAND ${test_binary} ${test_args}
-             WORKING_DIRECTORY ${working_dir})
+    add_test(
+      NAME ${test_name}
+      COMMAND ${test_binary} ${test_args}
+      WORKING_DIRECTORY ${working_dir})
   endif()
-  set(MEMORYCHECK_COMMAND_OPTIONS "--error-exitcode=1 --trace-children=yes --leak-check=full -s --gen-suppressions=all ${MEMCHECK_SUPPRESS}")
+  set(MEMORYCHECK_COMMAND_OPTIONS
+      "--error-exitcode=1 --trace-children=yes --leak-check=full -s --gen-suppressions=all ${MEMCHECK_SUPPRESS}"
+  )
   set(memcheck "${MEMORYCHECK_COMMAND} ${MEMORYCHECK_COMMAND_OPTIONS}")
   separate_arguments(memcheck)
-  if(TUVX_ENABLE_MPI AND MEMORYCHECK_COMMAND AND TUVX_ENABLE_MEMCHECK)
-    add_test(NAME memcheck_${test_name}
-      COMMAND mpirun -v -np 2 ${memcheck} ${CMAKE_BINARY_DIR}/${test_binary} ${test_args}
-             WORKING_DIRECTORY ${working_dir})
+  if(TUVX_ENABLE_MPI
+     AND MEMORYCHECK_COMMAND
+     AND TUVX_ENABLE_MEMCHECK)
+    add_test(
+      NAME memcheck_${test_name}
+      COMMAND mpirun -v -np 2 ${memcheck} ${CMAKE_BINARY_DIR}/${test_binary}
+              ${test_args}
+      WORKING_DIRECTORY ${working_dir})
   elseif(MEMORYCHECK_COMMAND AND TUVX_ENABLE_MEMCHECK)
-    add_test(NAME memcheck_${test_name}
-             COMMAND ${memcheck} ${CMAKE_BINARY_DIR}/${test_binary} ${test_args}
-             WORKING_DIRECTORY ${working_dir})
+    add_test(
+      NAME memcheck_${test_name}
+      COMMAND ${memcheck} ${CMAKE_BINARY_DIR}/${test_binary} ${test_args}
+      WORKING_DIRECTORY ${working_dir})
   endif()
 endfunction(add_tuvx_test)
 
-################################################################################
-# Setup regression tests. Add dependencies between each regression test and its 
+# ##############################################################################
+# Setup regression tests. Add dependencies between each regression test and its
 # memcheck test. Also add a dependence with any previous tests. Becuase TUV-x
 # outputs to the same location, concurrent runs of the standalone tool that
 # depend on the output must run in serial
 
 function(add_regression_test test_name command memcheck_command)
-  add_test(NAME ${test_name} COMMAND ${command} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+  add_test(
+    NAME ${test_name}
+    COMMAND ${command}
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
   if(MEMORYCHECK_COMMAND AND TUVX_ENABLE_MEMCHECK)
-    add_test(NAME memcheck_${test_name} COMMAND ${memcheck_command} WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+    add_test(
+      NAME memcheck_${test_name}
+      COMMAND ${memcheck_command}
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
   endif()
 
 endfunction(add_regression_test)
 
-################################################################################
+# ##############################################################################
 # Link tuv-x to a test and add it to the suite as a bash script
 
 macro(add_std_test_script test_name script_path)
@@ -119,4 +141,4 @@ macro(add_std_test_script test_name script_path)
   add_test(NAME ${test_name} COMMAND ${script_path})
 endmacro(add_std_test_script)
 
-################################################################################
+# ##############################################################################