From 757e3a50e13b4ca360e15b6d64e87528ffd3a88c Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Tue, 15 Oct 2024 17:08:54 +0200 Subject: [PATCH 01/11] Add speed_sig_stfl Signed-off-by: cr-marcstevens --- tests/CMakeLists.txt | 6 +- tests/speed_sig_stfl.c | 206 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 tests/speed_sig_stfl.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eb297a804..07e6a5e66 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -123,6 +123,7 @@ target_link_libraries(test_sig_mem PRIVATE ${TEST_DEPS}) add_executable(speed_sig speed_sig.c) target_link_libraries(speed_sig PRIVATE ${TEST_DEPS}) + set(SIG_TESTS example_sig kat_sig test_sig test_sig_mem speed_sig vectors_sig) # SIG_STFL API tests @@ -133,7 +134,10 @@ else () target_link_libraries(test_sig_stfl PRIVATE ${TEST_DEPS}) endif() -set(SIG_STFL_TESTS kat_sig_stfl test_sig_stfl) +add_executable(speed_sig_stfl speed_sig_stfl.c) +target_link_libraries(speed_sig_stfl PRIVATE ${TEST_DEPS}) + +set(SIG_STFL_TESTS kat_sig_stfl test_sig_stfl speed_sig_stfl) add_executable(dump_alg_info dump_alg_info.c) target_link_libraries(dump_alg_info PRIVATE ${TEST_DEPS}) diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c new file mode 100644 index 000000000..d9e23cfa6 --- /dev/null +++ b/tests/speed_sig_stfl.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: MIT + +#include +#include +#include +#include +#include + +#include + +#if defined(OQS_USE_RASPBERRY_PI) +#define _OQS_RASPBERRY_PI +#endif +#if defined(OQS_SPEED_USE_ARM_PMU) +#define SPEED_USE_ARM_PMU +#endif +#include "ds_benchmark.h" +#include "system_info.c" + +static void fullcycle(OQS_SIG_STFL *sig, uint8_t *public_key, OQS_SIG_STFL_SECRET_KEY *secret_key, uint8_t *signature, size_t signature_len, uint8_t *message, size_t message_len) { + if (OQS_SIG_STFL_keypair(sig, public_key, secret_key) != OQS_SUCCESS) { + printf("keygen error. Exiting.\n"); + exit(-1); + } + if (OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key) != OQS_SUCCESS) { + printf("sign error. Exiting.\n"); + exit(-1); + } + if (OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key) != OQS_SUCCESS) { + printf("verify error. Exiting.\n"); + exit(-1); + } +} + +static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, bool printInfo, bool doFullCycle) { + + OQS_SIG_STFL *sig = NULL; + uint8_t *public_key = NULL; + OQS_SIG_STFL_SECRET_KEY *secret_key = NULL; + uint8_t *message = NULL; + uint8_t *signature = NULL; + size_t message_len = 50; + size_t signature_len = 0; + OQS_STATUS ret = OQS_ERROR; + + sig = OQS_SIG_STFL_new(method_name); + if (sig == NULL) { + return OQS_SUCCESS; + } + + public_key = malloc(sig->length_public_key); + secret_key = OQS_SIG_STFL_SECRET_KEY_new(sig->method_name); + message = malloc(message_len); + signature = malloc(sig->length_signature); + + if ((public_key == NULL) || (secret_key == NULL) || (message == NULL) || (signature == NULL)) { + fprintf(stderr, "ERROR: malloc failed\n"); + goto err; + } + + OQS_randombytes(message, message_len); + + printf("%-36s | %10s | %14s | %15s | %10s | %25s | %10s\n", sig->method_name, "", "", "", "", "", ""); + if (!doFullCycle) { + TIME_OPERATION_SECONDS(OQS_SIG_STFL_keypair(sig, public_key, secret_key), "keypair", duration) + TIME_OPERATION_SECONDS(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration) + TIME_OPERATION_SECONDS(OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key), "verify", duration) + } else { + TIME_OPERATION_SECONDS(fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len), "fullcycle", duration) + } + + + if (printInfo) { + printf("public key bytes: %zu, secret key bytes: %zu, signature bytes: %zu\n", sig->length_public_key, sig->length_secret_key, sig->length_signature); + if (signature_len != sig->length_signature) { + printf(" Actual signature length returned (%zu) less than declared maximum signature length (%zu)\n", signature_len, sig->length_signature); + } + } + + ret = OQS_SUCCESS; + goto cleanup; + +err: + ret = OQS_ERROR; + +cleanup: + if (sig != NULL) { + OQS_SIG_STFL_SECRET_KEY_free(secret_key); + } + OQS_MEM_insecure_free(public_key); + OQS_MEM_insecure_free(signature); + OQS_MEM_insecure_free(message); + OQS_SIG_STFL_free(sig); + + return ret; +} + +static OQS_STATUS printAlgs(void) { + for (size_t i = 0; i < OQS_SIG_STFL_algs_length; i++) { + OQS_SIG_STFL *sig = OQS_SIG_STFL_new(OQS_SIG_STFL_alg_identifier(i)); + if (sig == NULL) { + printf("%s (disabled)\n", OQS_SIG_STFL_alg_identifier(i)); + } else { + printf("%s\n", OQS_SIG_STFL_alg_identifier(i)); + } + OQS_SIG_STFL_free(sig); + } + return OQS_SUCCESS; +} + +int main(int argc, char **argv) { + + int ret = EXIT_SUCCESS; + OQS_STATUS rc; + + bool printUsage = false; + uint64_t duration = 3; + bool printSigInfo = false; + bool doFullCycle = false; + + OQS_SIG_STFL *single_sig = NULL; + + OQS_init(); + OQS_randombytes_switch_algorithm(OQS_RAND_alg_openssl); + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--algs") == 0) { + rc = printAlgs(); + if (rc == OQS_SUCCESS) { + OQS_destroy(); + return EXIT_SUCCESS; + } else { + OQS_destroy(); + return EXIT_FAILURE; + } + } else if ((strcmp(argv[i], "--duration") == 0) || (strcmp(argv[i], "-d") == 0)) { + if (i < argc - 1) { + duration = (uint64_t)strtol(argv[i + 1], NULL, 10); + if (duration > 0) { + i += 1; + continue; + } + } + } else if ((strcmp(argv[i], "--help") == 0) || (strcmp(argv[i], "-h") == 0)) { + printUsage = true; + break; + } else if ((strcmp(argv[i], "--info") == 0) || (strcmp(argv[i], "-i") == 0)) { + printSigInfo = true; + continue; + } else if ((strcmp(argv[i], "--fullcycle") == 0) || (strcmp(argv[i], "-f") == 0)) { + doFullCycle = true; + continue; + } else { + single_sig = OQS_SIG_STFL_new(argv[i]); + if (single_sig == NULL) { + printUsage = true; + break; + } + } + } + + if (printUsage) { + fprintf(stderr, "Usage: speed_sig \n"); + fprintf(stderr, "\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "--algs Print supported algorithms and terminate\n"); + fprintf(stderr, "--duration n\n"); + fprintf(stderr, " -d n Run each speed test for approximately n seconds, default n=3\n"); + fprintf(stderr, "--help\n"); + fprintf(stderr, " -h Print usage\n"); + fprintf(stderr, "--info\n"); + fprintf(stderr, " -i Print info (sizes, security level) about each SIG\n"); + fprintf(stderr, "--fullcycle\n"); + fprintf(stderr, " -f Test full keygen-sign-verify cycle of each SIG\n"); + fprintf(stderr, "\n"); + fprintf(stderr, " Only run the specified SIG method; must be one of the algorithms output by --algs\n"); + OQS_destroy(); + return EXIT_FAILURE; + } + + print_system_info(); + + printf("Speed test\n"); + printf("==========\n"); + + PRINT_TIMER_HEADER + if (single_sig != NULL) { + rc = sig_speed_wrapper(single_sig->method_name, duration, printSigInfo, doFullCycle); + if (rc != OQS_SUCCESS) { + ret = EXIT_FAILURE; + } + OQS_SIG_STFL_free(single_sig); + + } else { + for (size_t i = 0; i < OQS_SIG_STFL_algs_length; i++) { + rc = sig_speed_wrapper(OQS_SIG_STFL_alg_identifier(i), duration, printSigInfo, doFullCycle); + if (rc != OQS_SUCCESS) { + ret = EXIT_FAILURE; + } + } + } + PRINT_TIMER_FOOTER + OQS_destroy(); + + return ret; +} From 1962edd1a287211ed55c13226bbfe9cf46e82a36 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Wed, 16 Oct 2024 09:51:21 +0200 Subject: [PATCH 02/11] Fix speed_sig_stfl.c: limit timing with max sig ops & provide required secure keystore with dummy keystore Signed-off-by: cr-marcstevens --- tests/ds_benchmark.h | 13 +++++++++++++ tests/speed_sig_stfl.c | 19 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/ds_benchmark.h b/tests/ds_benchmark.h index b69e8c4bc..a28d915a9 100644 --- a/tests/ds_benchmark.h +++ b/tests/ds_benchmark.h @@ -289,4 +289,17 @@ static void _bench_init_perfcounters(void) { PRINT_TIMER_AVG(op_name) \ } +#define TIME_OPERATION_SECONDS_MAXIT(op, op_name, secs, maxit) \ + { \ + DEFINE_TIMER_VARIABLES \ + INITIALIZE_TIMER \ + uint64_t _bench_time_goal_usecs = 1000000 * secs; \ + for (unsigned long long i = 0; i < (maxit) && _bench_time_cumulative < _bench_time_goal_usecs; i++) { \ + START_TIMER { op; } \ + STOP_TIMER \ + } \ + FINALIZE_TIMER \ + PRINT_TIMER_AVG(op_name) \ + } + #endif diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index d9e23cfa6..8ff6dfcaf 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -17,6 +17,11 @@ #include "ds_benchmark.h" #include "system_info.c" +OQS_STATUS dummy_secure_storage(uint8_t *sk_buf, size_t sk_buf_len, void *context) +{ + return OQS_SUCCESS; +} + static void fullcycle(OQS_SIG_STFL *sig, uint8_t *public_key, OQS_SIG_STFL_SECRET_KEY *secret_key, uint8_t *signature, size_t signature_len, uint8_t *message, size_t message_len) { if (OQS_SIG_STFL_keypair(sig, public_key, secret_key) != OQS_SUCCESS) { printf("keygen error. Exiting.\n"); @@ -48,8 +53,16 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, return OQS_SUCCESS; } - public_key = malloc(sig->length_public_key); secret_key = OQS_SIG_STFL_SECRET_KEY_new(sig->method_name); + if (secret_key == NULL) { + fprintf(stderr, "ERROR: OQS_SIG_STFL_SECRET_KEY_new failed\n"); + goto err; + } + OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, NULL); + unsigned long long max_sigs; + OQS_SIG_STFL_sigs_remaining(sig, &max_sigs, secret_key); + + public_key = malloc(sig->length_public_key); message = malloc(message_len); signature = malloc(sig->length_signature); @@ -63,7 +76,9 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, printf("%-36s | %10s | %14s | %15s | %10s | %25s | %10s\n", sig->method_name, "", "", "", "", "", ""); if (!doFullCycle) { TIME_OPERATION_SECONDS(OQS_SIG_STFL_keypair(sig, public_key, secret_key), "keypair", duration) - TIME_OPERATION_SECONDS(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration) + unsigned long long max_sigs; + OQS_SIG_STFL_sigs_total(sig, &max_sigs, secret_key); + TIME_OPERATION_SECONDS_MAXIT(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration, max_sigs) TIME_OPERATION_SECONDS(OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key), "verify", duration) } else { TIME_OPERATION_SECONDS(fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len), "fullcycle", duration) From 940eb59d3d28f61eea517200a8e77a8dfef75127 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Wed, 16 Oct 2024 09:55:55 +0200 Subject: [PATCH 03/11] Cleanup speed_sig_stfl.c Signed-off-by: cr-marcstevens --- tests/speed_sig_stfl.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index 8ff6dfcaf..4989ff4c7 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -59,8 +59,6 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, goto err; } OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, NULL); - unsigned long long max_sigs; - OQS_SIG_STFL_sigs_remaining(sig, &max_sigs, secret_key); public_key = malloc(sig->length_public_key); message = malloc(message_len); From ed04287bd39807d965d25bde6ce3ebc9b2c16157 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Wed, 16 Oct 2024 13:59:17 +0200 Subject: [PATCH 04/11] speed_sig_stfl: fix for LMS (secure store context must not be NULL), refresh key when out of sigs for sig benchmark Signed-off-by: cr-marcstevens --- tests/ds_benchmark.h | 25 ++++++++++++++----------- tests/speed_sig_stfl.c | 6 ++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/ds_benchmark.h b/tests/ds_benchmark.h index a28d915a9..5fd8ff5ac 100644 --- a/tests/ds_benchmark.h +++ b/tests/ds_benchmark.h @@ -289,17 +289,20 @@ static void _bench_init_perfcounters(void) { PRINT_TIMER_AVG(op_name) \ } -#define TIME_OPERATION_SECONDS_MAXIT(op, op_name, secs, maxit) \ - { \ - DEFINE_TIMER_VARIABLES \ - INITIALIZE_TIMER \ - uint64_t _bench_time_goal_usecs = 1000000 * secs; \ - for (unsigned long long i = 0; i < (maxit) && _bench_time_cumulative < _bench_time_goal_usecs; i++) { \ - START_TIMER { op; } \ - STOP_TIMER \ - } \ - FINALIZE_TIMER \ - PRINT_TIMER_AVG(op_name) \ +#define TIME_OPERATION_SECONDS_MAXIT(op, op_name, secs, maxit, refresh) \ + { \ + DEFINE_TIMER_VARIABLES \ + INITIALIZE_TIMER \ + uint64_t _bench_time_goal_usecs = 1000000 * secs; \ + while (_bench_time_cumulative < _bench_time_goal_usecs) { \ + for (unsigned long long i = 0; i < (maxit) && _bench_time_cumulative < _bench_time_goal_usecs; i++) { \ + START_TIMER { op; } \ + STOP_TIMER \ + } \ + if (_bench_time_cumulative < _bench_time_goal_usecs) { refresh; } \ + } \ + FINALIZE_TIMER \ + PRINT_TIMER_AVG(op_name) \ } #endif diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index 4989ff4c7..235eb7371 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -58,7 +58,8 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, fprintf(stderr, "ERROR: OQS_SIG_STFL_SECRET_KEY_new failed\n"); goto err; } - OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, NULL); + // for LMS context must not be NULL + OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, secret_key); public_key = malloc(sig->length_public_key); message = malloc(message_len); @@ -76,7 +77,8 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, TIME_OPERATION_SECONDS(OQS_SIG_STFL_keypair(sig, public_key, secret_key), "keypair", duration) unsigned long long max_sigs; OQS_SIG_STFL_sigs_total(sig, &max_sigs, secret_key); - TIME_OPERATION_SECONDS_MAXIT(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration, max_sigs) + TIME_OPERATION_SECONDS_MAXIT(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration, + max_sigs, OQS_SIG_STFL_keypair(sig, public_key, secret_key) ) TIME_OPERATION_SECONDS(OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key), "verify", duration) } else { TIME_OPERATION_SECONDS(fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len), "fullcycle", duration) From f9dd42b5f9a61f0fe06ffc031799c2084b6e7777 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Wed, 16 Oct 2024 15:32:08 +0200 Subject: [PATCH 05/11] tests/speed_sig_stfl.c: astyle fix Signed-off-by: cr-marcstevens --- tests/speed_sig_stfl.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index 235eb7371..25bf524cd 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -17,8 +17,7 @@ #include "ds_benchmark.h" #include "system_info.c" -OQS_STATUS dummy_secure_storage(uint8_t *sk_buf, size_t sk_buf_len, void *context) -{ +OQS_STATUS dummy_secure_storage(uint8_t *sk_buf, size_t sk_buf_len, void *context) { return OQS_SUCCESS; } @@ -77,8 +76,8 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, TIME_OPERATION_SECONDS(OQS_SIG_STFL_keypair(sig, public_key, secret_key), "keypair", duration) unsigned long long max_sigs; OQS_SIG_STFL_sigs_total(sig, &max_sigs, secret_key); - TIME_OPERATION_SECONDS_MAXIT(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration, - max_sigs, OQS_SIG_STFL_keypair(sig, public_key, secret_key) ) + TIME_OPERATION_SECONDS_MAXIT(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration, + max_sigs, OQS_SIG_STFL_keypair(sig, public_key, secret_key) ) TIME_OPERATION_SECONDS(OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key), "verify", duration) } else { TIME_OPERATION_SECONDS(fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len), "fullcycle", duration) From 9a5c3fb186921e45d065197284e8ce44ebab8cb8 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Thu, 17 Oct 2024 09:54:58 +0200 Subject: [PATCH 06/11] tests/speed_sig_stfl.c: stfl sig benchmarks require intermittent resetting of secret key Signed-off-by: cr-marcstevens --- tests/speed_sig_stfl.c | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index 25bf524cd..865eff8c9 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -18,9 +18,20 @@ #include "system_info.c" OQS_STATUS dummy_secure_storage(uint8_t *sk_buf, size_t sk_buf_len, void *context) { + // suppress unused parameter warning + (void)(sk_buf); + (void)(sk_buf_len); + (void)(context); return OQS_SUCCESS; } +// reset secret key: some schemes fail to create a new secret key over a previous secret key +void reset_secret_key(OQS_SIG_STFL *sig, OQS_SIG_STFL_SECRET_KEY *secret_key) { + OQS_SIG_STFL_SECRET_KEY_free(secret_key); + secret_key = OQS_SIG_STFL_SECRET_KEY_new(sig->method_name); + OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, secret_key); +} + static void fullcycle(OQS_SIG_STFL *sig, uint8_t *public_key, OQS_SIG_STFL_SECRET_KEY *secret_key, uint8_t *signature, size_t signature_len, uint8_t *message, size_t message_len) { if (OQS_SIG_STFL_keypair(sig, public_key, secret_key) != OQS_SUCCESS) { printf("keygen error. Exiting.\n"); @@ -73,17 +84,32 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, printf("%-36s | %10s | %14s | %15s | %10s | %25s | %10s\n", sig->method_name, "", "", "", "", "", ""); if (!doFullCycle) { - TIME_OPERATION_SECONDS(OQS_SIG_STFL_keypair(sig, public_key, secret_key), "keypair", duration) + // benchmark keygen: need to reset secret key between calls + OQS_STATUS status = 0; + TIME_OPERATION_SECONDS_MAXIT({ status = OQS_SIG_STFL_keypair(sig, public_key, secret_key); }, "keypair", duration, 1, { + if (status != OQS_SUCCESS) { + printf("keygen error. Exiting.\n"); + exit(-1); + } + reset_secret_key(sig, secret_key); + }) + // benchmark sign: need to generate new secret key after available signatures have been exhausted unsigned long long max_sigs; OQS_SIG_STFL_sigs_total(sig, &max_sigs, secret_key); - TIME_OPERATION_SECONDS_MAXIT(OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key), "sign", duration, - max_sigs, OQS_SIG_STFL_keypair(sig, public_key, secret_key) ) - TIME_OPERATION_SECONDS(OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key), "verify", duration) + TIME_OPERATION_SECONDS_MAXIT({ status = OQS_SIG_STFL_sign(sig, signature, &signature_len, message, message_len, secret_key); }, "sign", duration, max_sigs, { + if (status != OQS_SUCCESS) { + printf("sign error. Exiting.\n"); + exit(-1); + } + OQS_SIG_STFL_keypair(sig, public_key, secret_key); + }) + // benchmark verification + TIME_OPERATION_SECONDS({ OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key); }, "verify", duration) } else { - TIME_OPERATION_SECONDS(fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len), "fullcycle", duration) + // benchmark fullcycle: need to reset secret key between calls + TIME_OPERATION_SECONDS_MAXIT({ fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len); }, "fullcycle", duration, 1, { reset_secret_key(sig, secret_key); }) } - if (printInfo) { printf("public key bytes: %zu, secret key bytes: %zu, signature bytes: %zu\n", sig->length_public_key, sig->length_secret_key, sig->length_signature); if (signature_len != sig->length_signature) { From aae3f410971823ba731b4d2470552a9ba102a2c5 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Thu, 17 Oct 2024 10:24:18 +0200 Subject: [PATCH 07/11] speed_sig_stfl: add speed_sig_stfl to: README scripts/nogress.sh tests/test_speed.py Signed-off-by: cr-marcstevens --- README.md | 1 + scripts/noregress.sh | 8 ++++---- tests/speed_sig_stfl.c | 2 +- tests/test_speed.py | 7 +++++++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0d4ae69bb..e10e71891 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ The following instructions assume we are in `build`. - `kat_sig_stfl`: Program for checking results against submitted KAT values using `tests/test_kat.py` - `speed_kem`: Benchmarking program for key encapsulation mechanisms; see `./speed_kem --help` for usage instructions - `speed_sig`: Benchmarking program for signature mechanisms; see `./speed_sig --help` for usage instructions + - `speed_sig_stfl`: Benchmarking program for stateful signature mechanisms; see `./speed_sig_stfl --help` for usage instructions - `example_kem`: Minimal runnable example showing the usage of the KEM API - `example_sig`: Minimal runnable example showing the usage of the signature API - `example_sig_stfl`: Minimal runnable example showing the usage of the stateful signature API diff --git a/scripts/noregress.sh b/scripts/noregress.sh index ff9995990..418e8243e 100755 --- a/scripts/noregress.sh +++ b/scripts/noregress.sh @@ -15,7 +15,7 @@ fi # Approach: Check out $1 into tmp folder, build, run speed_kem|sig and compare results -mkdir tmp && cd tmp && git clone --depth 1 --branch $1 https://github.com/open-quantum-safe/liboqs && cd liboqs && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > ../../speed_kem.log && ./tests/speed_sig > ../../speed_sig.log && cd ../../.. +mkdir tmp && cd tmp && git clone --depth 1 --branch $1 https://github.com/open-quantum-safe/liboqs && cd liboqs && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > ../../speed_kem.log && ./tests/speed_sig > ../../speed_sig.log && ./tests/speed_sig_stfl > ../../speed_sig_stfl.log && cd ../../.. if [ $? -ne 0 ]; then echo "Build and test of baseline $1 failed. Exiting." @@ -24,7 +24,7 @@ fi # transform results into JSON files for simple comparison -cd tmp && git clone --depth 1 https://github.com/open-quantum-safe/profiling.git && cd profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../speed_kem.log && python3 parse_liboqs_speed.py ../../../speed_sig.log && cd ../../../.. +cd tmp && git clone --depth 1 https://github.com/open-quantum-safe/profiling.git && cd profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../speed_kem.log && python3 parse_liboqs_speed.py ../../../speed_sig.log && python3 parse_liboqs_speed.py ../../../speed_sig_stfl.log && cd ../../../.. if [ $? -ne 0 ]; then echo "Failure converting results. Exiting." @@ -32,7 +32,7 @@ if [ $? -ne 0 ]; then fi # obtain current base speed results -rm -rf build && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > speed_kem.log && ./tests/speed_sig > speed_sig.log && cd ../tmp/profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../../build/speed_kem.log && python3 parse_liboqs_speed.py ../../../../build/speed_sig.log && cd ../../../.. +rm -rf build && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > speed_kem.log && ./tests/speed_sig > speed_sig.log && ./tests/speed_sig_stfl > speed_sig_stfl.log && cd ../tmp/profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../../build/speed_kem.log && python3 parse_liboqs_speed.py ../../../../build/speed_sig.log && python3 parse_liboqs_speed.py ../../../../build/speed_sig_stfl.log && cd ../../../.. if [ $? -ne 0 ]; then echo "Failure creating current results. Exiting." @@ -40,4 +40,4 @@ if [ $? -ne 0 ]; then fi # now compare results using old/tmp runs as baseline (for list of algorithms) -python3 scripts/noregress.py tmp/speed_kem.json build/speed_kem.json && python3 scripts/noregress.py tmp/speed_sig.json build/speed_sig.json +python3 scripts/noregress.py tmp/speed_kem.json build/speed_kem.json && python3 scripts/noregress.py tmp/speed_sig.json build/speed_sig.json && python3 scripts/noregress.py tmp/speed_sig_stfl.json build/speed_sig_stfl.json diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index 865eff8c9..5252c891c 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -200,7 +200,7 @@ int main(int argc, char **argv) { } if (printUsage) { - fprintf(stderr, "Usage: speed_sig \n"); + fprintf(stderr, "Usage: speed_sig_stfl \n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); fprintf(stderr, "--algs Print supported algorithms and terminate\n"); diff --git a/tests/test_speed.py b/tests/test_speed.py index a99bbc949..b41412e22 100644 --- a/tests/test_speed.py +++ b/tests/test_speed.py @@ -20,6 +20,13 @@ def test_sig(sig_name): if not(helpers.is_sig_enabled_by_name(sig_name)): pytest.skip('Not enabled') helpers.run_subprocess( [helpers.path_to_executable('speed_sig'), sig_name, "-f"]) +@helpers.filtered_test +@pytest.mark.parametrize('sig_stfl_name', helpers.available_sig_stfls_by_name()) +def test_sig(sig_stfl_name): + kats = helpers.get_kats("sig_stfl") + if not(helpers.is_sig_stfl_enabled_by_name(sig_stfl_name)): pytest.skip('Not enabled') + helpers.run_subprocess( [helpers.path_to_executable('speed_sig_stfl'), sig_stfl_name, "-f"]) + if __name__ == "__main__": import sys pytest.main(sys.argv) From fbe7107833d874df91821e7249a4116da9ee5be3 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Sat, 19 Oct 2024 17:39:52 +0200 Subject: [PATCH 08/11] test_speed.py: limit testing of stfl sigs to parameters with 2^10 max sigs Signed-off-by: cr-marcstevens --- tests/test_speed.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_speed.py b/tests/test_speed.py index b41412e22..22be199d6 100644 --- a/tests/test_speed.py +++ b/tests/test_speed.py @@ -24,8 +24,12 @@ def test_sig(sig_name): @pytest.mark.parametrize('sig_stfl_name', helpers.available_sig_stfls_by_name()) def test_sig(sig_stfl_name): kats = helpers.get_kats("sig_stfl") - if not(helpers.is_sig_stfl_enabled_by_name(sig_stfl_name)): pytest.skip('Not enabled') - helpers.run_subprocess( [helpers.path_to_executable('speed_sig_stfl'), sig_stfl_name, "-f"]) + if not(helpers.is_sig_stfl_enabled_by_name(sig_stfl_name)): + pytest.skip('Not enabled') + elif sig_stfl_name.find("_10")==-1 and sig_stfl_name.find("H10")==-1: + pytest.skip('Test skipped') + else: + helpers.run_subprocess( [helpers.path_to_executable('speed_sig_stfl'), sig_stfl_name, "-f"]) if __name__ == "__main__": import sys From 57a4468cf5f2a3c4ca6330dc8020141e345eea69 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Sat, 19 Oct 2024 17:56:12 +0200 Subject: [PATCH 09/11] scripts/nogress.sh: limit regression tests on stfl sigs to only algorithms with 2^10 max sigs Signed-off-by: cr-marcstevens --- scripts/noregress.sh | 39 +++++++++++++++++++++++++++++++++++---- tests/speed_sig_stfl.c | 8 ++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/scripts/noregress.sh b/scripts/noregress.sh index 418e8243e..1217edfe6 100755 --- a/scripts/noregress.sh +++ b/scripts/noregress.sh @@ -15,7 +15,18 @@ fi # Approach: Check out $1 into tmp folder, build, run speed_kem|sig and compare results -mkdir tmp && cd tmp && git clone --depth 1 --branch $1 https://github.com/open-quantum-safe/liboqs && cd liboqs && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > ../../speed_kem.log && ./tests/speed_sig > ../../speed_sig.log && ./tests/speed_sig_stfl > ../../speed_sig_stfl.log && cd ../../.. +mkdir tmp && \ +cd tmp && \ +git clone --depth 1 --branch $1 https://github.com/open-quantum-safe/liboqs && \ +cd liboqs && \ +mkdir build && \ +cd build && \ +cmake $2 .. && \ +$MAKECMD && \ +./tests/speed_kem > ../../speed_kem.log && \ +./tests/speed_sig > ../../speed_sig.log && \ +./tests/speed_sig_stfl --limit10 > ../../speed_sig_stfl.log && \ +cd ../../.. if [ $? -ne 0 ]; then echo "Build and test of baseline $1 failed. Exiting." @@ -24,7 +35,13 @@ fi # transform results into JSON files for simple comparison -cd tmp && git clone --depth 1 https://github.com/open-quantum-safe/profiling.git && cd profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../speed_kem.log && python3 parse_liboqs_speed.py ../../../speed_sig.log && python3 parse_liboqs_speed.py ../../../speed_sig_stfl.log && cd ../../../.. +cd tmp && \ +git clone --depth 1 https://github.com/open-quantum-safe/profiling.git && \ +cd profiling/perf/scripts && \ +python3 parse_liboqs_speed.py ../../../speed_kem.log && \ +python3 parse_liboqs_speed.py ../../../speed_sig.log && \ +python3 parse_liboqs_speed.py ../../../speed_sig_stfl.log && \ +cd ../../../.. if [ $? -ne 0 ]; then echo "Failure converting results. Exiting." @@ -32,7 +49,19 @@ if [ $? -ne 0 ]; then fi # obtain current base speed results -rm -rf build && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > speed_kem.log && ./tests/speed_sig > speed_sig.log && ./tests/speed_sig_stfl > speed_sig_stfl.log && cd ../tmp/profiling/perf/scripts && python3 parse_liboqs_speed.py ../../../../build/speed_kem.log && python3 parse_liboqs_speed.py ../../../../build/speed_sig.log && python3 parse_liboqs_speed.py ../../../../build/speed_sig_stfl.log && cd ../../../.. +rm -rf build && \ +mkdir build && \ +cd build && \ +cmake $2 .. && \ +$MAKECMD && \ +./tests/speed_kem > speed_kem.log && \ +./tests/speed_sig > speed_sig.log && \ +./tests/speed_sig_stfl --limit10 > speed_sig_stfl.log && \ +cd ../tmp/profiling/perf/scripts && \ +python3 parse_liboqs_speed.py ../../../../build/speed_kem.log && \ +python3 parse_liboqs_speed.py ../../../../build/speed_sig.log && \ +python3 parse_liboqs_speed.py ../../../../build/speed_sig_stfl.log && \ +cd ../../../.. if [ $? -ne 0 ]; then echo "Failure creating current results. Exiting." @@ -40,4 +69,6 @@ if [ $? -ne 0 ]; then fi # now compare results using old/tmp runs as baseline (for list of algorithms) -python3 scripts/noregress.py tmp/speed_kem.json build/speed_kem.json && python3 scripts/noregress.py tmp/speed_sig.json build/speed_sig.json && python3 scripts/noregress.py tmp/speed_sig_stfl.json build/speed_sig_stfl.json +python3 scripts/noregress.py tmp/speed_kem.json build/speed_kem.json && \ +python3 scripts/noregress.py tmp/speed_sig.json build/speed_sig.json && \ +python3 scripts/noregress.py tmp/speed_sig_stfl.json build/speed_sig_stfl.json diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index 5252c891c..a4fdf1754 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -157,6 +157,7 @@ int main(int argc, char **argv) { uint64_t duration = 3; bool printSigInfo = false; bool doFullCycle = false; + bool onlyMaxSigs10 = false; OQS_SIG_STFL *single_sig = NULL; @@ -190,6 +191,9 @@ int main(int argc, char **argv) { } else if ((strcmp(argv[i], "--fullcycle") == 0) || (strcmp(argv[i], "-f") == 0)) { doFullCycle = true; continue; + } else if ((strcmp(argv[i], "--limit10") == 0) || (strcmp(argv[i], "-l") == 0)) { + onlyMaxSigs10 = true; + continue; } else { single_sig = OQS_SIG_STFL_new(argv[i]); if (single_sig == NULL) { @@ -212,6 +216,8 @@ int main(int argc, char **argv) { fprintf(stderr, " -i Print info (sizes, security level) about each SIG\n"); fprintf(stderr, "--fullcycle\n"); fprintf(stderr, " -f Test full keygen-sign-verify cycle of each SIG\n"); + fprintf(stderr, "--limit10 Test only algorithms with 2^10 max signatures\n"); + fprintf(stderr, " -l\n"); fprintf(stderr, "\n"); fprintf(stderr, " Only run the specified SIG method; must be one of the algorithms output by --algs\n"); OQS_destroy(); @@ -233,6 +239,8 @@ int main(int argc, char **argv) { } else { for (size_t i = 0; i < OQS_SIG_STFL_algs_length; i++) { + if (onlyMaxSigs10 > 0 && strstr(OQS_SIG_STFL_alg_identifier(i),"_10")==NULL && strstr(OQS_SIG_STFL_alg_identifier(i),"H10")==NULL) + continue; rc = sig_speed_wrapper(OQS_SIG_STFL_alg_identifier(i), duration, printSigInfo, doFullCycle); if (rc != OQS_SUCCESS) { ret = EXIT_FAILURE; From fdc76be74e810bb612f1c52bff4915daf86f2a76 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Sat, 19 Oct 2024 18:39:35 +0200 Subject: [PATCH 10/11] speed_sig_stfl.c: astyle fix Signed-off-by: cr-marcstevens --- tests/speed_sig_stfl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index a4fdf1754..b29da122c 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -239,8 +239,9 @@ int main(int argc, char **argv) { } else { for (size_t i = 0; i < OQS_SIG_STFL_algs_length; i++) { - if (onlyMaxSigs10 > 0 && strstr(OQS_SIG_STFL_alg_identifier(i),"_10")==NULL && strstr(OQS_SIG_STFL_alg_identifier(i),"H10")==NULL) + if (onlyMaxSigs10 > 0 && strstr(OQS_SIG_STFL_alg_identifier(i), "_10") == NULL && strstr(OQS_SIG_STFL_alg_identifier(i), "H10") == NULL) { continue; + } rc = sig_speed_wrapper(OQS_SIG_STFL_alg_identifier(i), duration, printSigInfo, doFullCycle); if (rc != OQS_SUCCESS) { ret = EXIT_FAILURE; From eb76b7739b36debd044c9fbaafd8a5e5293d45b3 Mon Sep 17 00:00:00 2001 From: cr-marcstevens Date: Sun, 20 Oct 2024 23:59:29 +0200 Subject: [PATCH 11/11] speed_sig_stfl: 1) fix use-after-free bug. 2) Simply return success if keygen and sign are not enabled. Signed-off-by: cr-marcstevens --- tests/speed_sig_stfl.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/speed_sig_stfl.c b/tests/speed_sig_stfl.c index b29da122c..ac09ca7bb 100644 --- a/tests/speed_sig_stfl.c +++ b/tests/speed_sig_stfl.c @@ -26,10 +26,11 @@ OQS_STATUS dummy_secure_storage(uint8_t *sk_buf, size_t sk_buf_len, void *contex } // reset secret key: some schemes fail to create a new secret key over a previous secret key -void reset_secret_key(OQS_SIG_STFL *sig, OQS_SIG_STFL_SECRET_KEY *secret_key) { +OQS_SIG_STFL_SECRET_KEY *reset_secret_key(OQS_SIG_STFL *sig, OQS_SIG_STFL_SECRET_KEY *secret_key) { OQS_SIG_STFL_SECRET_KEY_free(secret_key); secret_key = OQS_SIG_STFL_SECRET_KEY_new(sig->method_name); OQS_SIG_STFL_SECRET_KEY_SET_store_cb(secret_key, &dummy_secure_storage, secret_key); + return secret_key; } static void fullcycle(OQS_SIG_STFL *sig, uint8_t *public_key, OQS_SIG_STFL_SECRET_KEY *secret_key, uint8_t *signature, size_t signature_len, uint8_t *message, size_t message_len) { @@ -48,7 +49,6 @@ static void fullcycle(OQS_SIG_STFL *sig, uint8_t *public_key, OQS_SIG_STFL_SECRE } static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, bool printInfo, bool doFullCycle) { - OQS_SIG_STFL *sig = NULL; uint8_t *public_key = NULL; OQS_SIG_STFL_SECRET_KEY *secret_key = NULL; @@ -58,6 +58,20 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, size_t signature_len = 0; OQS_STATUS ret = OQS_ERROR; + // if keygen and signing is disabled then we can't benchmark and we simply return OQS_SUCCESS +#ifndef OQS_ALLOW_XMSS_KEY_AND_SIG_GEN + if (strstr(method_name, "XMSS") != NULL) { + printf("XMSS keygen and signing is not enabled.\n"); + return OQS_SUCCESS; + } +#endif +#ifndef OQS_ALLOW_LMS_KEY_AND_SIG_GEN + if (strstr(method_name, "LMS") != NULL) { + printf("LMS keygen and signing is not enabled.\n"); + return OQS_SUCCESS; + } +#endif + sig = OQS_SIG_STFL_new(method_name); if (sig == NULL) { return OQS_SUCCESS; @@ -91,7 +105,7 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, printf("keygen error. Exiting.\n"); exit(-1); } - reset_secret_key(sig, secret_key); + secret_key = reset_secret_key(sig, secret_key); }) // benchmark sign: need to generate new secret key after available signatures have been exhausted unsigned long long max_sigs; @@ -107,7 +121,7 @@ static OQS_STATUS sig_speed_wrapper(const char *method_name, uint64_t duration, TIME_OPERATION_SECONDS({ OQS_SIG_STFL_verify(sig, message, message_len, signature, signature_len, public_key); }, "verify", duration) } else { // benchmark fullcycle: need to reset secret key between calls - TIME_OPERATION_SECONDS_MAXIT({ fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len); }, "fullcycle", duration, 1, { reset_secret_key(sig, secret_key); }) + TIME_OPERATION_SECONDS_MAXIT({ fullcycle(sig, public_key, secret_key, signature, signature_len, message, message_len); }, "fullcycle", duration, 1, { secret_key = reset_secret_key(sig, secret_key); }) } if (printInfo) {