change from using ninja and custom cmake target run_test to using c…

…make & ctest.

Upon reading issue #1494
This PR aims to remove the need for ninja, and make tests platform agnostic.

The new build process will work as follows:
```
cmake -B build
cmake --build build --parallel
ctest --test-dir build/tests --output-on-failure
```

Unfortunately, cmake does not have the nice pytest --ignore feature. So to ensure
test_kat_all doesn't run by default, the `OQS_ENABLE_LONG_TESTS` variableis introduced.
If you would like to run the long tests (currently only `test_kat_all`) you would do the
following:
```
cmake -B build -DOQS_ENABLE_LONG_TESTS=on
cmake --build build --parallel
ctest --test-dir build/tests --output-on-failure
```

.CMake/cmake_uninstall.cmake.in

@@ -1,24 +1,22 @@
 # As per https://gitlab.kitware.com/cmake/community/-/wikis/FAQ#can-i-do-make-uninstall-with-cmake
-
-if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
-  message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
+if(NOT EXISTS "${CMAKE_BINARY_DIR}/install_manifest.txt")
+  message(FATAL_ERROR "Cannot find install manifest: ${CMAKE_BINARY_DIR}/install_manifest.txt")
 endif()
 
-file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
+file(READ "${CMAKE_BINARY_DIR}/install_manifest.txt" files)
 string(REGEX REPLACE "\n" ";" files "${files}")
 foreach(file ${files})
   message(STATUS "Uninstalling $ENV{DESTDIR}${file}")
   if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
-    exec_program(
-      "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\""
+    execute_process(
+      COMMAND "${CMAKE_COMMAND}" -E remove "$ENV{DESTDIR}${file}"
       OUTPUT_VARIABLE rm_out
-      RETURN_VALUE rm_retval
+      RESULT_VARIABLE rm_retval
       )
     if(NOT "${rm_retval}" STREQUAL 0)
       message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
     endif()
-  else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
+  else()
     message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
   endif()
 endforeach()
-

.dsci.yml

@@ -3,8 +3,8 @@ jobs:
     env:
       PYTEST_ARGS: tests/test_code_conventions.py tests/test_kat.py
     cmds:
-      - uname -a && mkdir build && cd build && cmake -GNinja .. && ninja && cd .. && python3 -m pytest --numprocesses=auto --verbose $PYTEST_ARGS ; rm -rf build
+      - uname -a && cmake -B build -DCMAKE_C_COMPILER=gcc-11 -GNinja .. && cmake --build build --parallel && ctest --test-dir build/tests --output-on-failure --parallel $(nproc) -R test_code_conventions -R test_kat -E test_kat_all && ctest --test-dir build/tests --output-on-failure --parallel $(nproc) -R test_code_conventions ; cd .. && rm -rf build
   - name: Building and testing using gcc-11 on M1
     cmds:
-      - uname -a && mkdir build && cd build && cmake -DCMAKE_C_COMPILER=gcc-11 -GNinja .. && ninja && ninja run_tests ; cd .. && rm -rf build
+      - uname -a && cmake -B build -DCMAKE_C_COMPILER=gcc-11 -GNinja .. && cmake --build build --parallel && ctest --test-dir build/tests --output-on-failure --parallel $(nproc) ; cd .. && rm -rf build
 

.travis.yml

@@ -9,13 +9,13 @@ jobs:
       compiler: gcc
       if: NOT branch =~ /^ghactionsonly-/
       script:
-        - mkdir build && cd build && cmake -GNinja .. && cmake -LA .. && ninja
-        - cd build & ninja run_tests
+        - cmake -B build && cmake --build build --parallel
+        - ctest --test-dir build/tests --output-on-failure --parallel $(nproc)
     - arch: s390x
       os: linux
       dist: focal
       compiler: gcc
       if: NOT branch =~ /^ghactionsonly-/
       script:
-        - mkdir build && cd build && cmake -GNinja .. && cmake -LA .. && ninja
-        - cd build & ninja run_tests
+        - cmake -B build && cmake --build build --parallel
+        - ctest --test-dir build/tests --output-on-failure --parallel $(nproc)

CMakeLists.txt

@@ -187,6 +187,7 @@ include_directories(${PROJECT_BINARY_DIR}/include)
 add_subdirectory(src)
 
 if(NOT ${OQS_BUILD_ONLY_LIB})
+    enable_testing()
     add_subdirectory(tests)
 
     if (NOT CYGWIN)

CONFIGURE.md

@@ -62,7 +62,7 @@ Selects algorithm set enabled. Possible values are "STD" selecting all algorithm
 
 ## OQS_BUILD_ONLY_LIB
 
-Can be `ON` or `OFF`. When `ON`, only liboqs is built, and all the targets: `run_tests`, `gen_docs`, and `prettyprint` are excluded from the build system.
+Can be `ON` or `OFF`. When `ON`, only liboqs is built, and all tests and the targets: `gen_docs`, and `prettyprint` are excluded from the build system.
 
 **Default**: `OFF`.
 

README.md

@@ -84,11 +84,11 @@ In order to optimize support effort,
 
 	On Ubuntu:
 
-		 sudo apt install astyle cmake gcc ninja-build libssl-dev python3-pytest python3-pytest-xdist unzip xsltproc doxygen graphviz python3-yaml valgrind
+		 sudo apt install astyle cmake gcc libssl-dev python3-pytest python3-pytest-xdist unzip xsltproc doxygen graphviz python3-yaml valgrind
 
 	On macOS, using a package manager of your choice (we've picked Homebrew):
 
-		brew install cmake ninja [email protected] wget doxygen graphviz astyle valgrind
+		brew install cmake [email protected] wget doxygen graphviz astyle valgrind
 		pip3 install pytest pytest-xdist pyyaml
 
 	Note that, if you want liboqs to use OpenSSL for various symmetric crypto algorithms (AES, SHA-2, etc.) then you must have OpenSSL installed (version 3.x recommended; EOL version 1.1.1 also still possible).
@@ -100,9 +100,8 @@ In order to optimize support effort,
 
 	and build:
 
-		mkdir build && cd build
-		cmake -GNinja ..
-		ninja
+		cmake -B build
+		cmake --build build --parallel
 
 Various `cmake` build options to customize the resultant artifacts are available and are [documented in CONFIGURE.md](CONFIGURE.md#options-for-configuring-liboqs-builds). All supported options are also listed in the `.CMake/alg-support.cmake` file, and can be viewed by running `cmake -LAH ..` in the `build` directory.
 
@@ -125,24 +124,24 @@ The following instructions assume we are in `build`.
 
 	The complete test suite can be run using
 
-		ninja run_tests
+		ctest --test-dir build/tests --parallel $(nproc)
 
 4. To generate HTML documentation of the API, run:
 
-		ninja gen_docs
+		cmake --build build --target gen_docs
 
 	Then open `docs/html/index.html` in your web browser.
 
-4. `ninja install` can be run to install the built library and `include` files to a location of choice, which can be specified by passing the `-DCMAKE_INSTALL_PREFIX=<dir>` option to `cmake` at configure time. Alternatively, `ninja package` can be run to create an install package.
+4. `cmake --install build` can be run to install the built library and `include` files to a location of choice, which can be specified by passing the `-DCMAKE_INSTALL_PREFIX=<dir>` option to `cmake` at configure time. Alternatively, `cmake --build build --target package` can be run to create an install package.
 
-5. `ninja uninstall` can be run to remove all installation files.
+5. `cmake --build build --target uninstall` can be run to remove all installation files.
 
 
 ### Windows
 
 Binaries can be generated using Visual Studio 2019 with the [CMake Tools](https://marketplace.visualstudio.com/items?itemName=ms-vscode.cmake-tools) extension installed. The same options as explained above for Linux/macOS can be used and build artifacts are generated in the specified `build` folders.
 
-If you want to create Visual Studio build files, e.g., if not using `ninja`, be sure to _not_ pass the parameter `-GNinja` to the `cmake` command as exemplified above. You can then build all components using `msbuild`, e.g. as follows: `msbuild ALL_BUILD.vcxproj` and install all artifacts e.g. using this command `msbuild INSTALL.vcxproj`.
+You can then build all components using `msbuild`, e.g. as follows: `msbuild ALL_BUILD.vcxproj` and install all artifacts e.g. using this command `msbuild INSTALL.vcxproj`.
 
 
 ### Cross compilation

tests/CMakeLists.txt

@@ -119,13 +119,108 @@ if (CMAKE_GENERATOR MATCHES "Visual Studio")
         RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL     "${CMAKE_BINARY_DIR}/tests")
 endif()
 
-# TODO: Get CMake to find python.
-# and set PATH variable in Windows
-# for DLL builds.
-add_custom_target(
-    run_tests
-    # skip long KAT tests
-    COMMAND ${CMAKE_COMMAND} -E env OQS_BUILD_DIR=${CMAKE_BINARY_DIR} ${PYTHON3_EXEC} -m pytest --verbose --numprocesses=auto --ignore=scripts/copy_from_upstream/repos --ignore=tests/test_kat_all.py
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    DEPENDS oqs dump_alg_info ${KEM_TESTS} ${SIG_TESTS} ${UNIX_TESTS}
-    USES_TERMINAL)
+add_test(
+	NAME test_alg_info
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_alg_info.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_binary
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_binary.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_cmdline
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_cmdline.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_code_conventions
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_code_conventions.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_constant_time
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_constant_time.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_distbuild
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_distbuild.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_hash
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_hash.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_kat
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_kat.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+# test_kat_all takes a very long time. Disable by default.
+if (OQS_ENABLE_LONG_TESTS)
+	add_test(
+		NAME test_kat_all
+		COMMAND ${PYTHON3_EXEC}
+		-m pytest 
+		tests/test_kat_all.py
+		WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+	)
+	set_tests_properties(test_kat_all
+		PROPERTIES ENVIRONMENT "OQS_BUILD_DIR=${CMAKE_BINARY_DIR}"
+	)
+endif()
+add_test(
+	NAME test_leaks
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_leaks.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_mem
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_mem.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+add_test(
+	NAME test_speed
+	COMMAND ${PYTHON3_EXEC}
+	-m pytest 
+	tests/test_speed.py
+	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+set_tests_properties(
+	test_alg_info
+	test_binary
+	test_cmdline
+	test_code_conventions
+	test_constant_time
+	test_distbuild
+	test_hash
+	test_kat
+	test_speed
+	test_leaks
+	test_mem
+	test_speed
+	PROPERTIES ENVIRONMENT "OQS_BUILD_DIR=${CMAKE_BINARY_DIR}"
+)