From 90030a4ae4ae26b2e90c3e84cc11395e6b5d77a2 Mon Sep 17 00:00:00 2001 From: Marc Stevens Date: Tue, 22 Oct 2024 15:16:56 +0200 Subject: [PATCH] Add benchmarking for stateful hash based schemes: speed_sig_stfl (#1952) * Add speed_sig_stfl Signed-off-by: cr-marcstevens * Fix speed_sig_stfl.c: limit timing with max sig ops & provide required secure keystore with dummy keystore Signed-off-by: cr-marcstevens * Cleanup speed_sig_stfl.c Signed-off-by: cr-marcstevens * speed_sig_stfl: fix for LMS (secure store context must not be NULL), refresh key when out of sigs for sig benchmark Signed-off-by: cr-marcstevens * tests/speed_sig_stfl.c: astyle fix Signed-off-by: cr-marcstevens * tests/speed_sig_stfl.c: stfl sig benchmarks require intermittent resetting of secret key Signed-off-by: cr-marcstevens * speed_sig_stfl: add speed_sig_stfl to: README scripts/nogress.sh tests/test_speed.py Signed-off-by: cr-marcstevens * test_speed.py: limit testing of stfl sigs to parameters with 2^10 max sigs Signed-off-by: cr-marcstevens * scripts/nogress.sh: limit regression tests on stfl sigs to only algorithms with 2^10 max sigs Signed-off-by: cr-marcstevens * speed_sig_stfl.c: astyle fix Signed-off-by: cr-marcstevens * speed_sig_stfl: 1) fix use-after-free bug. 2) Simply return success if keygen and sign are not enabled. Signed-off-by: cr-marcstevens --------- Signed-off-by: cr-marcstevens --- README.md | 1 + scripts/noregress.sh | 39 +++++- tests/CMakeLists.txt | 6 +- tests/ds_benchmark.h | 16 +++ tests/speed_sig_stfl.c | 269 +++++++++++++++++++++++++++++++++++++++++ tests/test_speed.py | 11 ++ 6 files changed, 337 insertions(+), 5 deletions(-) create mode 100644 tests/speed_sig_stfl.c diff --git a/README.md b/README.md index 0d4ae69bb6..e10e718910 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ The following instructions assume we are in `build`. - `kat_sig_stfl`: Program for checking results against submitted KAT values using `tests/test_kat.py` - `speed_kem`: Benchmarking program for key encapsulation mechanisms; see `./speed_kem --help` for usage instructions - `speed_sig`: Benchmarking program for signature mechanisms; see `./speed_sig --help` for usage instructions + - `speed_sig_stfl`: Benchmarking program for stateful signature mechanisms; see `./speed_sig_stfl --help` for usage instructions - `example_kem`: Minimal runnable example showing the usage of the KEM API - `example_sig`: Minimal runnable example showing the usage of the signature API - `example_sig_stfl`: Minimal runnable example showing the usage of the stateful signature API diff --git a/scripts/noregress.sh b/scripts/noregress.sh index ff99959900..1217edfe66 100755 --- a/scripts/noregress.sh +++ b/scripts/noregress.sh @@ -15,7 +15,18 @@ fi # Approach: Check out $1 into tmp folder, build, run speed_kem|sig and compare results -mkdir tmp && cd tmp && git clone --depth 1 --branch $1 https://github.com/open-quantum-safe/liboqs && cd liboqs && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > ../../speed_kem.log && ./tests/speed_sig > ../../speed_sig.log && cd ../../.. +mkdir tmp && \ +cd tmp && \ +git clone --depth 1 --branch $1 https://github.com/open-quantum-safe/liboqs && \ +cd liboqs && \ +mkdir build && \ +cd build && \ +cmake $2 .. && \ +$MAKECMD && \ +./tests/speed_kem > ../../speed_kem.log && \ +./tests/speed_sig > ../../speed_sig.log && \ +./tests/speed_sig_stfl --limit10 > ../../speed_sig_stfl.log && \ +cd ../../.. if [ $? -ne 0 ]; then echo "Build and test of baseline $1 failed. Exiting." @@ -24,7 +35,13 @@ fi # transform results into JSON files for simple comparison -cd tmp && git clone --depth 1 https://github.com/open-quantum-safe/profiling.git && cd profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../speed_kem.log && python3 parse_liboqs_speed.py ../../../speed_sig.log && cd ../../../.. +cd tmp && \ +git clone --depth 1 https://github.com/open-quantum-safe/profiling.git && \ +cd profiling/perf/scripts && \ +python3 parse_liboqs_speed.py ../../../speed_kem.log && \ +python3 parse_liboqs_speed.py ../../../speed_sig.log && \ +python3 parse_liboqs_speed.py ../../../speed_sig_stfl.log && \ +cd ../../../.. if [ $? -ne 0 ]; then echo "Failure converting results. Exiting." @@ -32,7 +49,19 @@ if [ $? -ne 0 ]; then fi # obtain current base speed results -rm -rf build && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > speed_kem.log && ./tests/speed_sig > speed_sig.log && cd ../tmp/profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../../build/speed_kem.log && python3 parse_liboqs_speed.py ../../../../build/speed_sig.log && cd ../../../.. +rm -rf build && \ +mkdir build && \ +cd build && \ +cmake $2 .. && \ +$MAKECMD && \ +./tests/speed_kem > speed_kem.log && \ +./tests/speed_sig > speed_sig.log && \ +./tests/speed_sig_stfl --limit10 > speed_sig_stfl.log && \ +cd ../tmp/profiling/perf/scripts && \ +python3 parse_liboqs_speed.py ../../../../build/speed_kem.log && \ +python3 parse_liboqs_speed.py ../../../../build/speed_sig.log && \ +python3 parse_liboqs_speed.py ../../../../build/speed_sig_stfl.log && \ +cd ../../../.. if [ $? -ne 0 ]; then echo "Failure creating current results. Exiting." @@ -40,4 +69,6 @@ if [ $? -ne 0 ]; then fi # now compare results using old/tmp runs as baseline (for list of algorithms) -python3 scripts/noregress.py tmp/speed_kem.json build/speed_kem.json && python3 scripts/noregress.py tmp/speed_sig.json build/speed_sig.json +python3 scripts/noregress.py tmp/speed_kem.json build/speed_kem.json && \ +python3 scripts/noregress.py tmp/speed_sig.json build/speed_sig.json && \ +python3 scripts/noregress.py tmp/speed_sig_stfl.json build/speed_sig_stfl.json diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a03e784d36..ad7677ed24 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -138,6 +138,7 @@ target_link_libraries(test_sig_mem PRIVATE ${TEST_DEPS}) add_executable(speed_sig speed_sig.c) target_link_libraries(speed_sig PRIVATE ${TEST_DEPS}) + set(SIG_TESTS example_sig kat_sig test_sig test_sig_mem speed_sig vectors_sig) # SIG_STFL API tests @@ -148,7 +149,10 @@ else () target_link_libraries(test_sig_stfl PRIVATE ${TEST_DEPS}) endif() -set(SIG_STFL_TESTS kat_sig_stfl test_sig_stfl) +add_executable(speed_sig_stfl speed_sig_stfl.c) +target_link_libraries(speed_sig_stfl PRIVATE ${TEST_DEPS}) + +set(SIG_STFL_TESTS kat_sig_stfl test_sig_stfl speed_sig_stfl) add_executable(dump_alg_info dump_alg_info.c) target_link_libraries(dump_alg_info PRIVATE ${TEST_DEPS}) diff --git a/tests/ds_benchmark.h b/tests/ds_benchmark.h index b69e8c4bcc..5fd8ff5ac8 100644 --- a/tests/ds_benchmark.h +++ b/tests/ds_benchmark.h @@ -289,4 +289,20 @@ static void _bench_init_perfcounters(void) { PRINT_TIMER_AVG(op_name) \ } +#define TIME_OPERATION_SECONDS_MAXIT(op, op_name, secs, maxit, refresh) \ + { \ + DEFINE_TIMER_VARIABLES \ + INITIALIZE_TIMER \ + uint64_t _bench_time_goal_usecs = 1000000 * secs; \ + while (_bench_time_cumulative < _bench_time_goal_usecs) { \ + for (unsigned long long i = 0; i < (maxit) && _bench_time_cumulative < _bench_time_goal_usecs; i++) { \ + START_TIMER { op; } \ + STOP_TIMER \ + } \ + if (_bench_time_cumulative < _bench_time_goal_usecs) { refresh; } \ + } \ + FINALIZE_TIMER \ + PRINT_TIMER_AVG(op_name) \ + } + #endif diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c new file mode 100644 index 0000000000..ac09ca7bb2 --- /dev/null +++ b/tests/speed_sig_stfl.c @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: MIT + +#include +#include +#include +#include +#include + +#include + +#if defined(OQS_USE_RASPBERRY_PI) +#define _OQS_RASPBERRY_PI +#endif +#if defined(OQS_SPEED_USE_ARM_PMU) +#define SPEED_USE_ARM_PMU +#endif +#include "ds_benchmark.h" +#include "system_info.c" + +OQS_STATUS dummy_secure_storage(uint8_t *sk_buf, size_t sk_buf_len, void *context) { + // suppress unused parameter warning + (void)(sk_buf); + (void)(sk_buf_len); + (void)(context); + return OQS_SUCCESS; +} + +// reset secret key: some schemes fail to create a new secret key over a previous secret key +OQS_SIG_STFL_SECRET_KEY *reset_secret_key(OQS_SIG_STFL *sig, OQS_SIG_STFL_SECRET_KEY *secret_key) { + OQS_SIG_STFL_SECRET_KEY_free(secret_key); + secret_key = OQS_SIG_STFL_SECRET_KEY_new(sig->method_name); + OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, secret_key); + return secret_key; +} + +static void fullcycle(OQS_SIG_STFL *sig, uint8_t *public_key, OQS_SIG_STFL_SECRET_KEY *secret_key, uint8_t *signature, size_t signature_len, uint8_t *message, size_t message_len) { + if (OQS_SIG_STFL_keypair(sig, public_key, secret_key) != OQS_SUCCESS) { + printf("keygen error. Exiting.\n"); + exit(-1); + } + if (OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key) != OQS_SUCCESS) { + printf("sign error. Exiting.\n"); + exit(-1); + } + if (OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key) != OQS_SUCCESS) { + printf("verify error. Exiting.\n"); + exit(-1); + } +} + +static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, bool printInfo, bool doFullCycle) { + OQS_SIG_STFL *sig = NULL; + uint8_t *public_key = NULL; + OQS_SIG_STFL_SECRET_KEY *secret_key = NULL; + uint8_t *message = NULL; + uint8_t *signature = NULL; + size_t message_len = 50; + size_t signature_len = 0; + OQS_STATUS ret = OQS_ERROR; + + // if keygen and signing is disabled then we can't benchmark and we simply return OQS_SUCCESS +#ifndef OQS_ALLOW_XMSS_KEY_AND_SIG_GEN + if (strstr(method_name, "XMSS") != NULL) { + printf("XMSS keygen and signing is not enabled.\n"); + return OQS_SUCCESS; + } +#endif +#ifndef OQS_ALLOW_LMS_KEY_AND_SIG_GEN + if (strstr(method_name, "LMS") != NULL) { + printf("LMS keygen and signing is not enabled.\n"); + return OQS_SUCCESS; + } +#endif + + sig = OQS_SIG_STFL_new(method_name); + if (sig == NULL) { + return OQS_SUCCESS; + } + + secret_key = OQS_SIG_STFL_SECRET_KEY_new(sig->method_name); + if (secret_key == NULL) { + fprintf(stderr, "ERROR: OQS_SIG_STFL_SECRET_KEY_new failed\n"); + goto err; + } + // for LMS context must not be NULL + OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, secret_key); + + public_key = malloc(sig->length_public_key); + message = malloc(message_len); + signature = malloc(sig->length_signature); + + if ((public_key == NULL) || (secret_key == NULL) || (message == NULL) || (signature == NULL)) { + fprintf(stderr, "ERROR: malloc failed\n"); + goto err; + } + + OQS_randombytes(message, message_len); + + printf("%-36s | %10s | %14s | %15s | %10s | %25s | %10s\n", sig->method_name, "", "", "", "", "", ""); + if (!doFullCycle) { + // benchmark keygen: need to reset secret key between calls + OQS_STATUS status = 0; + TIME_OPERATION_SECONDS_MAXIT({ status = OQS_SIG_STFL_keypair(sig, public_key, secret_key); }, "keypair", duration, 1, { + if (status != OQS_SUCCESS) { + printf("keygen error. Exiting.\n"); + exit(-1); + } + secret_key = reset_secret_key(sig, secret_key); + }) + // benchmark sign: need to generate new secret key after available signatures have been exhausted + unsigned long long max_sigs; + OQS_SIG_STFL_sigs_total(sig, &max_sigs, secret_key); + TIME_OPERATION_SECONDS_MAXIT({ status = OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key); }, "sign", duration, max_sigs, { + if (status != OQS_SUCCESS) { + printf("sign error. Exiting.\n"); + exit(-1); + } + OQS_SIG_STFL_keypair(sig, public_key, secret_key); + }) + // benchmark verification + TIME_OPERATION_SECONDS({ OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key); }, "verify", duration) + } else { + // benchmark fullcycle: need to reset secret key between calls + TIME_OPERATION_SECONDS_MAXIT({ fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len); }, "fullcycle", duration, 1, { secret_key = reset_secret_key(sig, secret_key); }) + } + + if (printInfo) { + printf("public key bytes: %zu, secret key bytes: %zu, signature bytes: %zu\n", sig->length_public_key, sig->length_secret_key, sig->length_signature); + if (signature_len != sig->length_signature) { + printf(" Actual signature length returned (%zu) less than declared maximum signature length (%zu)\n", signature_len, sig->length_signature); + } + } + + ret = OQS_SUCCESS; + goto cleanup; + +err: + ret = OQS_ERROR; + +cleanup: + if (sig != NULL) { + OQS_SIG_STFL_SECRET_KEY_free(secret_key); + } + OQS_MEM_insecure_free(public_key); + OQS_MEM_insecure_free(signature); + OQS_MEM_insecure_free(message); + OQS_SIG_STFL_free(sig); + + return ret; +} + +static OQS_STATUS printAlgs(void) { + for (size_t i = 0; i < OQS_SIG_STFL_algs_length; i++) { + OQS_SIG_STFL *sig = OQS_SIG_STFL_new(OQS_SIG_STFL_alg_identifier(i)); + if (sig == NULL) { + printf("%s (disabled)\n", OQS_SIG_STFL_alg_identifier(i)); + } else { + printf("%s\n", OQS_SIG_STFL_alg_identifier(i)); + } + OQS_SIG_STFL_free(sig); + } + return OQS_SUCCESS; +} + +int main(int argc, char **argv) { + + int ret = EXIT_SUCCESS; + OQS_STATUS rc; + + bool printUsage = false; + uint64_t duration = 3; + bool printSigInfo = false; + bool doFullCycle = false; + bool onlyMaxSigs10 = false; + + OQS_SIG_STFL *single_sig = NULL; + + OQS_init(); + OQS_randombytes_switch_algorithm(OQS_RAND_alg_openssl); + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--algs") == 0) { + rc = printAlgs(); + if (rc == OQS_SUCCESS) { + OQS_destroy(); + return EXIT_SUCCESS; + } else { + OQS_destroy(); + return EXIT_FAILURE; + } + } else if ((strcmp(argv[i], "--duration") == 0) || (strcmp(argv[i], "-d") == 0)) { + if (i < argc - 1) { + duration = (uint64_t)strtol(argv[i + 1], NULL, 10); + if (duration > 0) { + i += 1; + continue; + } + } + } else if ((strcmp(argv[i], "--help") == 0) || (strcmp(argv[i], "-h") == 0)) { + printUsage = true; + break; + } else if ((strcmp(argv[i], "--info") == 0) || (strcmp(argv[i], "-i") == 0)) { + printSigInfo = true; + continue; + } else if ((strcmp(argv[i], "--fullcycle") == 0) || (strcmp(argv[i], "-f") == 0)) { + doFullCycle = true; + continue; + } else if ((strcmp(argv[i], "--limit10") == 0) || (strcmp(argv[i], "-l") == 0)) { + onlyMaxSigs10 = true; + continue; + } else { + single_sig = OQS_SIG_STFL_new(argv[i]); + if (single_sig == NULL) { + printUsage = true; + break; + } + } + } + + if (printUsage) { + fprintf(stderr, "Usage: speed_sig_stfl \n"); + fprintf(stderr, "\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "--algs Print supported algorithms and terminate\n"); + fprintf(stderr, "--duration n\n"); + fprintf(stderr, " -d n Run each speed test for approximately n seconds, default n=3\n"); + fprintf(stderr, "--help\n"); + fprintf(stderr, " -h Print usage\n"); + fprintf(stderr, "--info\n"); + fprintf(stderr, " -i Print info (sizes, security level) about each SIG\n"); + fprintf(stderr, "--fullcycle\n"); + fprintf(stderr, " -f Test full keygen-sign-verify cycle of each SIG\n"); + fprintf(stderr, "--limit10 Test only algorithms with 2^10 max signatures\n"); + fprintf(stderr, " -l\n"); + fprintf(stderr, "\n"); + fprintf(stderr, " Only run the specified SIG method; must be one of the algorithms output by --algs\n"); + OQS_destroy(); + return EXIT_FAILURE; + } + + print_system_info(); + + printf("Speed test\n"); + printf("==========\n"); + + PRINT_TIMER_HEADER + if (single_sig != NULL) { + rc = sig_speed_wrapper(single_sig->method_name, duration, printSigInfo, doFullCycle); + if (rc != OQS_SUCCESS) { + ret = EXIT_FAILURE; + } + OQS_SIG_STFL_free(single_sig); + + } else { + for (size_t i = 0; i < OQS_SIG_STFL_algs_length; i++) { + if (onlyMaxSigs10 > 0 && strstr(OQS_SIG_STFL_alg_identifier(i), "_10") == NULL && strstr(OQS_SIG_STFL_alg_identifier(i), "H10") == NULL) { + continue; + } + rc = sig_speed_wrapper(OQS_SIG_STFL_alg_identifier(i), duration, printSigInfo, doFullCycle); + if (rc != OQS_SUCCESS) { + ret = EXIT_FAILURE; + } + } + } + PRINT_TIMER_FOOTER + OQS_destroy(); + + return ret; +} diff --git a/tests/test_speed.py b/tests/test_speed.py index a99bbc9499..22be199d64 100644 --- a/tests/test_speed.py +++ b/tests/test_speed.py @@ -20,6 +20,17 @@ def test_sig(sig_name): if not(helpers.is_sig_enabled_by_name(sig_name)): pytest.skip('Not enabled') helpers.run_subprocess( [helpers.path_to_executable('speed_sig'), sig_name, "-f"]) +@helpers.filtered_test +@pytest.mark.parametrize('sig_stfl_name', helpers.available_sig_stfls_by_name()) +def test_sig(sig_stfl_name): + kats = helpers.get_kats("sig_stfl") + if not(helpers.is_sig_stfl_enabled_by_name(sig_stfl_name)): + pytest.skip('Not enabled') + elif sig_stfl_name.find("_10")==-1 and sig_stfl_name.find("H10")==-1: + pytest.skip('Test skipped') + else: + helpers.run_subprocess( [helpers.path_to_executable('speed_sig_stfl'), sig_stfl_name, "-f"]) + if __name__ == "__main__": import sys pytest.main(sys.argv)