From c45b9dfb7fa040d1c356b60b4bc35a288442042e Mon Sep 17 00:00:00 2001 From: "Martyrshot martyrshot@gmail.com" <> Date: Sat, 17 Feb 2024 00:26:18 -0600 Subject: [PATCH] change from using ninja and custom cmake target `run_test` to using cmake & ctest. Upon reading issue #1494 This PR aims to remove the need for ninja, and make tests platform agnostic. The new build process will work as follows: ``` cmake -B build cmake --build build --parallel ctest --test-dir build/tests --output-on-failure ``` Unfortunately, cmake does not have the nice pytest --ignore feature. So to ensure test_kat_all doesn't run by default, the `OQS_ENABLE_LONG_TESTS` variableis introduced. If you would like to run the long tests (currently only `test_kat_all`) you would do the following: ``` cmake -B build -DOQS_ENABLE_LONG_TESTS=on cmake --build build --parallel ctest --test-dir build/tests --output-on-failure ``` --- .CMake/cmake_uninstall.cmake.in | 16 ++--- .dsci.yml | 4 +- .travis.yml | 8 +-- CMakeLists.txt | 1 + CONFIGURE.md | 2 +- README.md | 19 +++--- tests/CMakeLists.txt | 115 +++++++++++++++++++++++++++++--- 7 files changed, 129 insertions(+), 36 deletions(-) diff --git a/.CMake/cmake_uninstall.cmake.in b/.CMake/cmake_uninstall.cmake.in index 3880d489e3..c4275a07e0 100644 --- a/.CMake/cmake_uninstall.cmake.in +++ b/.CMake/cmake_uninstall.cmake.in @@ -1,24 +1,22 @@ # As per https://gitlab.kitware.com/cmake/community/-/wikis/FAQ#can-i-do-make-uninstall-with-cmake - -if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt") - message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt") +if(NOT EXISTS "${CMAKE_BINARY_DIR}/install_manifest.txt") + message(FATAL_ERROR "Cannot find install manifest: ${CMAKE_BINARY_DIR}/install_manifest.txt") endif() -file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files) +file(READ "${CMAKE_BINARY_DIR}/install_manifest.txt" files) string(REGEX REPLACE "\n" ";" files "${files}") foreach(file ${files}) message(STATUS "Uninstalling $ENV{DESTDIR}${file}") if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") - exec_program( - "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" + execute_process( + COMMAND "${CMAKE_COMMAND}" -E remove "$ENV{DESTDIR}${file}" OUTPUT_VARIABLE rm_out - RETURN_VALUE rm_retval + RESULT_VARIABLE rm_retval ) if(NOT "${rm_retval}" STREQUAL 0) message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") endif() - else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") + else() message(STATUS "File $ENV{DESTDIR}${file} does not exist.") endif() endforeach() - diff --git a/.dsci.yml b/.dsci.yml index 3bbba9bed3..b2a25f48fc 100644 --- a/.dsci.yml +++ b/.dsci.yml @@ -3,8 +3,8 @@ jobs: env: PYTEST_ARGS: tests/test_code_conventions.py tests/test_kat.py cmds: - - uname -a && mkdir build && cd build && cmake -GNinja .. && ninja && cd .. && python3 -m pytest --numprocesses=auto --verbose $PYTEST_ARGS ; rm -rf build + - uname -a && cmake -B build -DCMAKE_C_COMPILER=gcc-11 -GNinja .. && cmake --build build --parallel && ctest --test-dir build/tests --output-on-failure --parallel $(nproc) -R test_code_conventions -R test_kat -E test_kat_all && ctest --test-dir build/tests --output-on-failure --parallel $(nproc) -R test_code_conventions ; cd .. && rm -rf build - name: Building and testing using gcc-11 on M1 cmds: - - uname -a && mkdir build && cd build && cmake -DCMAKE_C_COMPILER=gcc-11 -GNinja .. && ninja && ninja run_tests ; cd .. && rm -rf build + - uname -a && cmake -B build -DCMAKE_C_COMPILER=gcc-11 -GNinja .. && cmake --build build --parallel && ctest --test-dir build/tests --output-on-failure --parallel $(nproc) ; cd .. && rm -rf build diff --git a/.travis.yml b/.travis.yml index fd13edc301..24bd67e668 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,13 +9,13 @@ jobs: compiler: gcc if: NOT branch =~ /^ghactionsonly-/ script: - - mkdir build && cd build && cmake -GNinja .. && cmake -LA .. && ninja - - cd build & ninja run_tests + - cmake -B build && cmake --build build --parallel + - ctest --test-dir build/tests --output-on-failure --parallel $(nproc) - arch: s390x os: linux dist: focal compiler: gcc if: NOT branch =~ /^ghactionsonly-/ script: - - mkdir build && cd build && cmake -GNinja .. && cmake -LA .. && ninja - - cd build & ninja run_tests + - cmake -B build && cmake --build build --parallel + - ctest --test-dir build/tests --output-on-failure --parallel $(nproc) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1f070b3a7..daad6a6c98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,6 +187,7 @@ include_directories(${PROJECT_BINARY_DIR}/include) add_subdirectory(src) if(NOT ${OQS_BUILD_ONLY_LIB}) + enable_testing() add_subdirectory(tests) if (NOT CYGWIN) diff --git a/CONFIGURE.md b/CONFIGURE.md index 05049a8554..b86c0fe129 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -62,7 +62,7 @@ Selects algorithm set enabled. Possible values are "STD" selecting all algorithm ## OQS_BUILD_ONLY_LIB -Can be `ON` or `OFF`. When `ON`, only liboqs is built, and all the targets: `run_tests`, `gen_docs`, and `prettyprint` are excluded from the build system. +Can be `ON` or `OFF`. When `ON`, only liboqs is built, and all tests and the targets: `gen_docs`, and `prettyprint` are excluded from the build system. **Default**: `OFF`. diff --git a/README.md b/README.md index b6733fa2fa..8c866610ea 100644 --- a/README.md +++ b/README.md @@ -84,11 +84,11 @@ In order to optimize support effort, On Ubuntu: - sudo apt install astyle cmake gcc ninja-build libssl-dev python3-pytest python3-pytest-xdist unzip xsltproc doxygen graphviz python3-yaml valgrind + sudo apt install astyle cmake gcc libssl-dev python3-pytest python3-pytest-xdist unzip xsltproc doxygen graphviz python3-yaml valgrind On macOS, using a package manager of your choice (we've picked Homebrew): - brew install cmake ninja openssl@1.1 wget doxygen graphviz astyle valgrind + brew install cmake openssl@1.1 wget doxygen graphviz astyle valgrind pip3 install pytest pytest-xdist pyyaml Note that, if you want liboqs to use OpenSSL for various symmetric crypto algorithms (AES, SHA-2, etc.) then you must have OpenSSL installed (version 3.x recommended; EOL version 1.1.1 also still possible). @@ -100,9 +100,8 @@ In order to optimize support effort, and build: - mkdir build && cd build - cmake -GNinja .. - ninja + cmake -B build + cmake --build build --parallel Various `cmake` build options to customize the resultant artifacts are available and are [documented in CONFIGURE.md](CONFIGURE.md#options-for-configuring-liboqs-builds). All supported options are also listed in the `.CMake/alg-support.cmake` file, and can be viewed by running `cmake -LAH ..` in the `build` directory. @@ -125,24 +124,24 @@ The following instructions assume we are in `build`. The complete test suite can be run using - ninja run_tests + ctest --test-dir build/tests --parallel $(nproc) 4. To generate HTML documentation of the API, run: - ninja gen_docs + cmake --build build --target gen_docs Then open `docs/html/index.html` in your web browser. -4. `ninja install` can be run to install the built library and `include` files to a location of choice, which can be specified by passing the `-DCMAKE_INSTALL_PREFIX=` option to `cmake` at configure time. Alternatively, `ninja package` can be run to create an install package. +4. `cmake --install build` can be run to install the built library and `include` files to a location of choice, which can be specified by passing the `-DCMAKE_INSTALL_PREFIX=` option to `cmake` at configure time. Alternatively, `cmake --build build --target package` can be run to create an install package. -5. `ninja uninstall` can be run to remove all installation files. +5. `cmake --build build --target uninstall` can be run to remove all installation files. ### Windows Binaries can be generated using Visual Studio 2019 with the [CMake Tools](https://marketplace.visualstudio.com/items?itemName=ms-vscode.cmake-tools) extension installed. The same options as explained above for Linux/macOS can be used and build artifacts are generated in the specified `build` folders. -If you want to create Visual Studio build files, e.g., if not using `ninja`, be sure to _not_ pass the parameter `-GNinja` to the `cmake` command as exemplified above. You can then build all components using `msbuild`, e.g. as follows: `msbuild ALL_BUILD.vcxproj` and install all artifacts e.g. using this command `msbuild INSTALL.vcxproj`. +You can then build all components using `msbuild`, e.g. as follows: `msbuild ALL_BUILD.vcxproj` and install all artifacts e.g. using this command `msbuild INSTALL.vcxproj`. ### Cross compilation diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ff68438829..b7584c5900 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -119,13 +119,108 @@ if (CMAKE_GENERATOR MATCHES "Visual Studio") RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL "${CMAKE_BINARY_DIR}/tests") endif() -# TODO: Get CMake to find python. -# and set PATH variable in Windows -# for DLL builds. -add_custom_target( - run_tests - # skip long KAT tests - COMMAND ${CMAKE_COMMAND} -E env OQS_BUILD_DIR=${CMAKE_BINARY_DIR} ${PYTHON3_EXEC} -m pytest --verbose --numprocesses=auto --ignore=scripts/copy_from_upstream/repos --ignore=tests/test_kat_all.py - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - DEPENDS oqs dump_alg_info ${KEM_TESTS} ${SIG_TESTS} ${UNIX_TESTS} - USES_TERMINAL) +add_test( + NAME test_alg_info + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_alg_info.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_binary + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_binary.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_cmdline + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_cmdline.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_code_conventions + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_code_conventions.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_constant_time + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_constant_time.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_distbuild + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_distbuild.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_hash + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_hash.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_kat + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_kat.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +# test_kat_all takes a very long time. Disable by default. +if (OQS_ENABLE_LONG_TESTS) + add_test( + NAME test_kat_all + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_kat_all.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) + set_tests_properties(test_kat_all + PROPERTIES ENVIRONMENT "OQS_BUILD_DIR=${CMAKE_BINARY_DIR}" + ) +endif() +add_test( + NAME test_leaks + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_leaks.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_mem + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_mem.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +add_test( + NAME test_speed + COMMAND ${PYTHON3_EXEC} + -m pytest + tests/test_speed.py + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) +set_tests_properties( + test_alg_info + test_binary + test_cmdline + test_code_conventions + test_constant_time + test_distbuild + test_hash + test_kat + test_speed + test_leaks + test_mem + test_speed + PROPERTIES ENVIRONMENT "OQS_BUILD_DIR=${CMAKE_BINARY_DIR}" +)