From 9cf92621da566b2a5610c21018cf8b0b5884fb75 Mon Sep 17 00:00:00 2001 From: ericeberry Date: Mon, 30 Jan 2023 14:42:07 -0800 Subject: [PATCH] Issue 28: Fix integration issue with OpenSSL 1.1.1l (#29) --- include/sec_security_svp.h | 8 ++++++ src/sec_adapter_engine.c | 53 +++++++++++++++++--------------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/include/sec_security_svp.h b/include/sec_security_svp.h index 6acc9ed..b1af5c9 100644 --- a/include/sec_security_svp.h +++ b/include/sec_security_svp.h @@ -24,6 +24,10 @@ #include "sec_security.h" #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct svp_processor_buffer_struct { Sec_ProcessorHandle* processorHandle; sa_svp_buffer svp_buffer; @@ -49,4 +53,8 @@ Sec_Result SecOpaqueBuffer_Create(Sec_OpaqueBufferHandle** opaqueBufferHandle, v sa_svp_buffer get_svp_buffer(Sec_ProcessorHandle* processorHandle, Sec_OpaqueBufferHandle* opaqueBufferHandle); void release_svp_buffer(Sec_ProcessorHandle* processorHandle, Sec_OpaqueBufferHandle* opaqueBufferHandle); +#ifdef __cplusplus +} +#endif + #endif // SEC_SECURITY_SVP_H diff --git a/src/sec_adapter_engine.c b/src/sec_adapter_engine.c index 50b7aff..3abaab2 100644 --- a/src/sec_adapter_engine.c +++ b/src/sec_adapter_engine.c @@ -1,5 +1,5 @@ /** -* Copyright 2020-2022 Comcast Cable Communications Management, LLC +* Copyright 2020-2023 Comcast Cable Communications Management, LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,8 @@ static SEC_BOOL g_sec_openssl_inited = SEC_FALSE; static RSA_METHOD* rsa_method = NULL; #endif +static ENGINE* engine = NULL; + static void Sec_ShutdownOpenSSL() { #if OPENSSL_VERSION_NUMBER >= 0x10100000L if (rsa_method != NULL) { @@ -36,11 +38,10 @@ static void Sec_ShutdownOpenSSL() { } #endif - ENGINE* engine = ENGINE_by_id(ENGINE_ID); if (engine != NULL) { - ENGINE_remove(engine); ENGINE_finish(engine); ENGINE_free(engine); + engine = NULL; } } @@ -198,7 +199,7 @@ static RSA_METHOD g_sec_openssl_rsamethod = { #endif static void ENGINE_load_securityapi(void) { - ENGINE* engine = ENGINE_new(); + engine = ENGINE_new(); if (engine == NULL) { SEC_LOG_ERROR("ENGINE_new failed"); return; @@ -206,30 +207,40 @@ static void ENGINE_load_securityapi(void) { if (!ENGINE_set_id(engine, ENGINE_ID)) { ENGINE_free(engine); + engine = NULL; return; } if (!ENGINE_set_name(engine, "SecurityApi engine")) { ENGINE_free(engine); + engine = NULL; return; } if (!ENGINE_init(engine)) { ENGINE_free(engine); + engine = NULL; return; } #if OPENSSL_VERSION_NUMBER < 0x10100000L if (!ENGINE_set_RSA(engine, &g_sec_openssl_rsamethod)) { #else + if (rsa_method == NULL) { + rsa_method = RSA_meth_new("securityapi RSA method", RSA_METHOD_FLAG_NO_CHECK | RSA_FLAG_EXT_PKEY); + RSA_meth_set_pub_enc(rsa_method, Sec_OpenSSLPubEncrypt); + RSA_meth_set_priv_dec(rsa_method, Sec_OpenSSLPrivDecrypt); + RSA_meth_set_sign(rsa_method, Sec_OpenSSLPrivSign); + RSA_meth_set_verify(rsa_method, Sec_OpenSSLPubVerify); + } + if (!ENGINE_set_RSA(engine, rsa_method)) { #endif - ENGINE_remove(engine); + ENGINE_finish(engine); ENGINE_free(engine); + engine = NULL; return; } - ENGINE_add(engine); - ENGINE_free(engine); ERR_clear_error(); } @@ -239,16 +250,7 @@ void Sec_InitOpenSSL() { pthread_mutex_lock(&init_openssl_mutex); if (g_sec_openssl_inited != SEC_TRUE) { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - if (rsa_method == NULL) { - rsa_method = RSA_meth_new("securityapi RSA method", RSA_METHOD_FLAG_NO_CHECK | RSA_FLAG_EXT_PKEY); - RSA_meth_set_pub_enc(rsa_method, Sec_OpenSSLPubEncrypt); - RSA_meth_set_priv_dec(rsa_method, Sec_OpenSSLPrivDecrypt); - RSA_meth_set_sign(rsa_method, Sec_OpenSSLPrivSign); - RSA_meth_set_verify(rsa_method, Sec_OpenSSLPubVerify); - } - -#else +#if OPENSSL_VERSION_NUMBER < 0x10100000L ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); OpenSSL_add_all_ciphers(); @@ -257,7 +259,6 @@ void Sec_InitOpenSSL() { ENGINE_load_builtin_engines(); ENGINE_register_all_complete(); - ENGINE_load_securityapi(); if (atexit(Sec_ShutdownOpenSSL) != 0) { SEC_LOG_ERROR("atexit failed"); @@ -267,6 +268,10 @@ void Sec_InitOpenSSL() { g_sec_openssl_inited = SEC_TRUE; } + if (engine == NULL) { + ENGINE_load_securityapi(); + } + pthread_mutex_unlock(&init_openssl_mutex); } @@ -278,23 +283,19 @@ void Sec_PrintOpenSSLVersion() { RSA* SecKey_ToEngineRSA(Sec_KeyHandle* keyHandle) { Sec_RSARawPublicKey pubKey; RSA* rsa = NULL; - ENGINE* engine = NULL; - engine = ENGINE_by_id(ENGINE_ID); if (engine == NULL) { - SEC_LOG_ERROR("ENGINE_by_id failed"); + SEC_LOG_ERROR("engine not initialized"); return NULL; } if (SEC_RESULT_SUCCESS != SecKey_ExtractRSAPublicKey(keyHandle, &pubKey)) { - ENGINE_free(engine); SEC_LOG_ERROR("SecKey_ExtractRSAPublicKey failed"); return NULL; } rsa = RSA_new_method(engine); if (rsa == NULL) { - ENGINE_free(engine); SEC_LOG_ERROR("RSA_new_method failed"); return NULL; } @@ -308,30 +309,25 @@ RSA* SecKey_ToEngineRSA(Sec_KeyHandle* keyHandle) { #endif RSA_set_app_data(rsa, keyHandle); - ENGINE_free(engine); return rsa; } RSA* SecKey_ToEngineRSAWithCert(Sec_KeyHandle* keyHandle, Sec_CertificateHandle* certificateHandle) { Sec_RSARawPublicKey pubKey; RSA* rsa = NULL; - ENGINE* engine = NULL; - engine = ENGINE_by_id(ENGINE_ID); if (engine == NULL) { SEC_LOG_ERROR("ENGINE_by_id failed"); return NULL; } if (SEC_RESULT_SUCCESS != SecCertificate_ExtractRSAPublicKey(certificateHandle, &pubKey)) { - ENGINE_free(engine); SEC_LOG_ERROR("SecKey_ExtractRSAPublicKey failed"); return NULL; } rsa = RSA_new_method(engine); if (rsa == NULL) { - ENGINE_free(engine); SEC_LOG_ERROR("RSA_new_method failed"); return NULL; } @@ -345,7 +341,6 @@ RSA* SecKey_ToEngineRSAWithCert(Sec_KeyHandle* keyHandle, Sec_CertificateHandle* #endif RSA_set_app_data(rsa, keyHandle); - ENGINE_free(engine); return rsa; }