PS-8389 feature: Align PS Encryption UDFs functionality with new feat…

…ures from MySQL 8.0.30 Enterprise Encryption Component (openssl111 evp_pkey)

https://jira.percona.com/browse/PS-8389

'evp_pkey' class reworked to support OpenSSL 1.1.1. We no longer use
'OSSL_PARAM's to operate on 'EVP_PKEY' as they become available only
starting from OpenSSL 3.0.0.

Fixed exception error message in 'decrypt_with_rsa_public_key()' /
'decrypt_with_rsa_private_key()' functions.

Modified 'digest_table.inc' MTR include file. For OpenSSL 1.1.1 we use the
following digests only for 'create_digest()' checks but not for
'asymmetric_sign()':
- 'sha512-224'
- 'sha512-256'
- 'whirlpool'
- 'sm3'
- 'blake2b512'
- 'blake2s256'
- 'shake128'
- 'shake256'
For OpenSSL 3.x.y we use the following digests only for 'create_digest()'
checks but not for 'asymmetric_sign()':
- 'sha1'
- 'md5-sha1'
This change was necessary only for Oracle Linux 9 platform - for some
reason its default OpenSSL 3.0.7 has additional restriction that these
digests cannot be used in 'RSA' 'EVP_PKEY' signing operations.

Checks for default padding scheme in 'asymmetric_encrypt()' /
'asymmetric_decrypt()' removed from the
'component_encryption_udf.xsa_sanity' MTR test case as they are covered in
'component_encryption_udf.legacy_padding_scheme'.

'component_encryption_udf.legacy_padding_scheme' MTR test case modified
to take into account that decrypting messages with different padding scheme
may either fail with an error or succeed but produce garbage (depending on
random bytes inserted into the encrypted message).

extra/opensslpp/include_private/opensslpp/bio.hpp

@@ -31,7 +31,7 @@ class bio final {
 
  public:
   bio();
-  bio(std::string_view buffer);
+  explicit bio(std::string_view buffer);
 
   ~bio() noexcept = default;
 

extra/opensslpp/src/opensslpp/digest_context.cpp

@@ -68,6 +68,8 @@ void digest_context::swap(digest_context &obj) noexcept {
 }
 
 std::size_t digest_context::get_size_in_bytes() const noexcept {
+  assert(!is_empty());
+
   return EVP_MD_CTX_size(digest_context_accessor::get_impl(*this));
 }
 

extra/opensslpp/src/opensslpp/evp_pkey.cpp

@@ -18,39 +18,81 @@
 #include <memory>
 #include <vector>
 
-#include <openssl/core_names.h>
+#include <openssl/evp.h>
 #include <openssl/pem.h>
 
 #include <opensslpp/evp_pkey.hpp>
 
-#include <opensslpp/big_number.hpp>
 #include <opensslpp/core_error.hpp>
 #include <opensslpp/operation_cancelled_error.hpp>
+#include <opensslpp/rsa_key.hpp>
 
-#include "opensslpp/big_number_accessor.hpp"
 #include "opensslpp/bio.hpp"
 #include "opensslpp/bio_accessor.hpp"
 #include "opensslpp/evp_pkey_accessor.hpp"
 #include "opensslpp/evp_pkey_algorithm_conversions.hpp"
-#include "opensslpp/key_generation_cancellation_context.hpp"
-#include "opensslpp/key_generation_cancellation_context_accessor.hpp"
+#include "opensslpp/rsa_key_accessor.hpp"
+
+namespace {
+
+void duplicate_evp_pkey(opensslpp::evp_pkey &dest,
+                        const opensslpp::evp_pkey &source, bool public_only) {
+  assert(!source.is_empty());
+
+  auto *casted_impl{
+      opensslpp::evp_pkey_accessor::get_impl_const_casted(source)};
+  unsigned char *der_raw{nullptr};
+  const auto serializer{public_only ? &i2d_PublicKey : &i2d_PrivateKey};
+  auto der_length{(*serializer)(casted_impl, &der_raw)};
+  if (der_length < 0) {
+    opensslpp::core_error::raise_with_error_string(
+        "cannot serialize EVP_PKEY to DER format");
+  }
+
+  struct ossl_deleter {
+    void operator()(void *ptr) const noexcept {
+      if (ptr != nullptr) {
+        OPENSSL_free(ptr);
+      }
+    }
+  };
+  using buffer_ptr = std::unique_ptr<unsigned char, ossl_deleter>;
+  buffer_ptr der{der_raw};
+
+  const unsigned char *der_ptr{der_raw};
+  const auto deserializer{public_only ? &d2i_PublicKey : &d2i_PrivateKey};
+  opensslpp::evp_pkey_accessor::set_impl(
+      dest, (*deserializer)(EVP_PKEY_base_id(casted_impl), nullptr, &der_ptr,
+                            der_length));
+  if (dest.is_empty()) {
+    opensslpp::core_error::raise_with_error_string(
+        "cannot deserialize EVP_PKEY from DER format");
+  }
+}
+
+}  // anonymous namespace
 
 namespace opensslpp {
 
 void evp_pkey::evp_pkey_deleter::operator()(void *evp_pkey) const noexcept {
   if (evp_pkey != nullptr) EVP_PKEY_free(static_cast<EVP_PKEY *>(evp_pkey));
 }
 
-evp_pkey::evp_pkey(const evp_pkey &obj)
-    :  // due to a bug in openssl interface, EVP_PKEY_dup() expects
-       // non-const parameter while it does not do any modifications with the
-       // object - it just performs duplication via ASN1_item_i2d/ASN1_item_d2i
-       // conversions
-      impl_{obj.is_empty()
-                ? nullptr
-                : EVP_PKEY_dup(evp_pkey_accessor::get_impl_const_casted(obj))} {
-  if (!obj.is_empty() && is_empty())
-    throw core_error{"cannot duplicate EVP_PKEY key"};
+evp_pkey::evp_pkey(const evp_pkey &obj) : impl_{} {
+  if (!obj.is_empty()) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    // due to a bug in openssl interface, EVP_PKEY_dup() expects
+    // non-const parameter while it does not do any modifications with the
+    // object - it just performs duplication via ASN1_item_i2d/ASN1_item_d2i
+    // conversions
+    impl_.reset(EVP_PKEY_dup(evp_pkey_accessor::get_impl_const_casted(obj)));
+    if (!impl_) {
+      throw core_error{"cannot duplicate EVP_PKEY key"};
+    }
+#else
+    duplicate_evp_pkey(*this, obj, !obj.is_private());
+#endif
+  }
 }
 
 evp_pkey &evp_pkey::operator=(const evp_pkey &obj) {
@@ -63,60 +105,39 @@ void evp_pkey::swap(evp_pkey &obj) noexcept { impl_.swap(obj.impl_); }
 
 evp_pkey_algorithm evp_pkey::get_algorithm() const noexcept {
   assert(!is_empty());
-  auto native_algorithm{
-      EVP_PKEY_get_base_id(evp_pkey_accessor::get_impl(*this))};
+  auto native_algorithm{EVP_PKEY_base_id(evp_pkey_accessor::get_impl(*this))};
   return native_algorithm_to_evp_pkey_algorithm(native_algorithm);
 }
 
 bool evp_pkey::is_private() const noexcept {
   assert(!is_empty());
 
-  OSSL_PARAM params[] = {
-      OSSL_PARAM_construct_BN(OSSL_PKEY_PARAM_RSA_D, nullptr, 0),
-      OSSL_PARAM_construct_end()};
-
-  return EVP_PKEY_get_params(evp_pkey_accessor::get_impl(*this), params) != 0;
+  // TODO: implement checks for other algorithms
+  const auto *native_rsa{
+      EVP_PKEY_get0_RSA(evp_pkey_accessor::get_impl_const_casted(*this))};
+  assert(native_rsa != nullptr);
+  rsa_key underlying_key;
+  rsa_key_accessor::set_impl(underlying_key, const_cast<RSA *>(native_rsa));
+  const auto res{underlying_key.is_private()};
+  rsa_key_accessor::release(underlying_key);
+  return res;
 }
 
 std::size_t evp_pkey::get_size_in_bits() const noexcept {
   assert(!is_empty());
-  return EVP_PKEY_get_bits(evp_pkey_accessor::get_impl(*this));
+  return EVP_PKEY_bits(evp_pkey_accessor::get_impl(*this));
 }
 
 std::size_t evp_pkey::get_size_in_bytes() const noexcept {
   assert(!is_empty());
-  return EVP_PKEY_get_size(evp_pkey_accessor::get_impl(*this));
+  return EVP_PKEY_size(evp_pkey_accessor::get_impl(*this));
 }
 
 evp_pkey evp_pkey::derive_public_key() const {
   assert(!is_empty());
 
-  const auto *casted_impl{evp_pkey_accessor::get_impl(*this)};
-  unsigned char *der_raw{nullptr};
-  auto der_length{i2d_PublicKey(casted_impl, &der_raw)};
-  if (der_length < 0) {
-    core_error::raise_with_error_string(
-        "cannot serialize EVP_PKEY to DER format");
-  }
-
-  struct ossl_deleter {
-    void operator()(void *ptr) const noexcept {
-      if (ptr != nullptr) {
-        OPENSSL_free(ptr);
-      }
-    }
-  };
-  using buffer_ptr = std::unique_ptr<unsigned char, ossl_deleter>;
-  buffer_ptr der{der_raw};
-
-  const unsigned char *der_ptr{der_raw};
   evp_pkey res{};
-  evp_pkey_accessor::set_impl(
-      res, d2i_PublicKey(EVP_PKEY_get_base_id(casted_impl), nullptr, &der_ptr,
-                         der_length));
-  if (res.is_empty()) {
-    core_error::raise_with_error_string("cannot derive public EVP_PKEY");
-  }
+  duplicate_evp_pkey(res, *this, true);
 
   return res;
 }
@@ -128,6 +149,7 @@ class evp_pkey_keygen_ctx {
       : impl_{EVP_PKEY_CTX_new_id(id, nullptr)},
         cancellation_callback_{&cancellation_callback},
         cancelled_{false} {
+    assert(id == EVP_PKEY_RSA);
     if (!impl_) {
       throw core_error{"cannot create EVP_PKEY context for key generation"};
     }
@@ -138,11 +160,8 @@ class evp_pkey_keygen_ctx {
   }
 
   evp_pkey generate(std::size_t bits) {
-    auto local_bits{bits};
-    OSSL_PARAM params[] = {
-        OSSL_PARAM_construct_size_t(OSSL_PKEY_PARAM_BITS, &local_bits),
-        OSSL_PARAM_construct_end()};
-    if (EVP_PKEY_CTX_set_params(impl_.get(), params) <= 0) {
+    // TODO: implement setting bit length for other algorithms
+    if (EVP_PKEY_CTX_set_rsa_keygen_bits(impl_.get(), bits) <= 0) {
       throw core_error{"cannot set EVP_PKEY context key generation parameters"};
     }
 
@@ -214,9 +233,10 @@ std::string evp_pkey::export_private_pem(const evp_pkey &key) {
     throw core_error{"EVP_PKEY does not have private components"};
 
   auto sink = bio{};
-  const int r = PEM_write_bio_PrivateKey(bio_accessor::get_impl(sink),
-                                         evp_pkey_accessor::get_impl(key),
-                                         nullptr, nullptr, 0, nullptr, nullptr);
+  const int r =
+      PEM_write_bio_PrivateKey(bio_accessor::get_impl(sink),
+                               evp_pkey_accessor::get_impl_const_casted(key),
+                               nullptr, nullptr, 0, nullptr, nullptr);
 
   if (r <= 0)
     core_error::raise_with_error_string(
@@ -230,8 +250,9 @@ std::string evp_pkey::export_public_pem(const evp_pkey &key) {
   assert(!key.is_empty());
 
   auto sink = bio{};
-  const int r = PEM_write_bio_PUBKEY(bio_accessor::get_impl(sink),
-                                     evp_pkey_accessor::get_impl(key));
+  const int r =
+      PEM_write_bio_PUBKEY(bio_accessor::get_impl(sink),
+                           evp_pkey_accessor::get_impl_const_casted(key));
   if (r == 0)
     core_error::raise_with_error_string(
         "cannot export EVP_PKEY key to PEM PUBLIC KEY");

extra/opensslpp/src/opensslpp/rsa_encrypt_decrypt_operations.cpp

@@ -109,7 +109,7 @@ std::string decrypt_with_rsa_public_key(std::string_view input,
       rsa_encryption_padding_to_native_padding(padding));
   if (enc_status == -1)
     core_error::raise_with_error_string(
-        "cannot encrypt data block with the specified public RSA key");
+        "cannot decrypt data block with the specified public RSA key");
 
   assert(enc_status >= 0);
   res.resize(static_cast<std::size_t>(enc_status));
@@ -137,7 +137,7 @@ std::string decrypt_with_rsa_private_key(std::string_view input,
       rsa_encryption_padding_to_native_padding(padding));
   if (enc_status == -1)
     core_error::raise_with_error_string(
-        "cannot encrypt data block with the specified private RSA key");
+        "cannot decrypt data block with the specified private RSA key");
 
   assert(enc_status >= 0);
   res.resize(static_cast<std::size_t>(enc_status));

mysql-test/suite/component_encryption_udf/include/digest_table.inc

@@ -81,20 +81,29 @@ if (`select "$ssl_ver" like "OpenSSL 1.1.1%"`)
     ('md4'       , NULL),
     ('md5-sha1'  , NULL),
     ('ripemd160' , NULL),
-    ('sha512-224', NULL),
-    ('sha512-256', NULL),
-    ('whirlpool' , NULL),
-    ('sm3'       , NULL),
-    ('blake2b512', NULL),
-    ('blake2s256', NULL),
     ('sha3-224'  , NULL),
     ('sha3-256'  , NULL),
     ('sha3-384'  , NULL),
-    ('sha3-512'  , NULL),
-    ('shake128'  , NULL),
-    ('shake256'  , NULL)
+    ('sha3-512'  , NULL)
   ;
 
+  # the following digests can only be used in create_digest() and not in
+  # asymmetric_sign() as these digests do not have corresponding asn1
+  # object identifiers
+  if ($include_digests_with_no_ans1_ids)
+  {
+    INSERT INTO digest_type(digest_name, builtin_template) VALUES
+      ('sha512-224', NULL),
+      ('sha512-256', NULL),
+      ('whirlpool' , NULL),
+      ('sm3'       , NULL),
+      ('blake2b512', NULL),
+      ('blake2s256', NULL),
+      ('shake128'  , NULL),
+      ('shake256'  , NULL)
+    ;
+  }
+
   --enable_query_log
   --enable_result_log
 }
@@ -108,14 +117,12 @@ if (`select "$ssl_ver" like "OpenSSL 3.%"`)
 
   INSERT INTO digest_type(digest_name, builtin_template) VALUES
     ('md5'       , 'MD5(@message)'),
-    ('sha1'      , 'SHA(@message)'),
 
     ('sha224'    , 'SHA2(@message, 224)'),
     ('sha256'    , 'SHA2(@message, 256)'),
     ('sha384'    , 'SHA2(@message, 384)'),
     ('sha512'    , 'SHA2(@message, 512)'),
 
-    ('md5-sha1'  , NULL),
     ('sha512-224', NULL),
     ('sha512-256', NULL),
     ('sha3-224'  , NULL),
@@ -127,9 +134,14 @@ if (`select "$ssl_ver" like "OpenSSL 3.%"`)
   # BLAKE2B512, BLAKE2S256, SHAKE128, SHAKE256 and SM3 can only be used in
   # create_digest() and not in asymmetric_sign() as these digests do not have
   # corresponding asn1 object identifiers
+
+  # for some reason on Oracle Linux 9 (with default OpenSSL 3.0.7) 'sha1' and
+  # 'md5-sha1' cannot be used in RSA EVP_PKEY signing operations
   if ($include_digests_with_no_ans1_ids)
   {
     INSERT INTO digest_type(digest_name, builtin_template) VALUES
+      ('sha1'      , 'SHA(@message)'),
+      ('md5-sha1'  , NULL),
       ('sm3'       , NULL),
       ('blake2b512', NULL),
       ('blake2s256', NULL),

mysql-test/suite/component_encryption_udf/r/legacy_padding_scheme.result

@@ -20,17 +20,21 @@ SET @encrypted_message_pkcs1_imlicit = asymmetric_encrypt(@algorithm, @message,
 SET GLOBAL encryption_udf.legacy_padding_scheme = OFF;
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_oaep_imlicit, @priv_key, 'oaep');
 include/assert.inc [decrypting @encrypted_message_oaep_imlicit with 'oaep' scheme specified explicitly must succeed]
+SET @decrypted_message = '';
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_pkcs1_imlicit, @priv_key, 'oaep');
-ERROR HY000: asymmetric_decrypt<string> UDF failed; cannot encrypt data block with the specified private RSA key
+include/assert.inc [decrypting @encrypted_message_pkcs1_imlicit with 'oaep' scheme specified explicitly must fail]
+SET @decrypted_message = '';
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_oaep_imlicit, @priv_key, 'pkcs1');
 include/assert.inc [decrypting @encrypted_message_oaep_imlicit with 'pkcs1' scheme specified explicitly must fail]
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_pkcs1_imlicit, @priv_key, 'pkcs1');
 include/assert.inc [decrypting @encrypted_message_pkcs1_imlicit with 'pkcs1' scheme specified explicitly must succeed]
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_oaep_explicit, @priv_key);
 include/assert.inc [decrypting @encrypted_message_oaep_explicit with 'oaep' scheme specified implicitly must succeed]
+SET @decrypted_message = '';
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_pkcs1_explicit, @priv_key);
-ERROR HY000: asymmetric_decrypt<string> UDF failed; cannot encrypt data block with the specified private RSA key
+include/assert.inc [decrypting @encrypted_message_pkcs1_explicit with 'oaep' scheme specified implicitly must fail]
 SET GLOBAL encryption_udf.legacy_padding_scheme = ON;
+SET @decrypted_message = '';
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_oaep_explicit, @priv_key);
 include/assert.inc [decrypting @encrypted_message_oaep_explicit with 'pkcs1' scheme specified implicitly must fail]
 SET @decrypted_message = asymmetric_decrypt(@algorithm, @encrypted_message_pkcs1_explicit, @priv_key);

mysql-test/suite/component_encryption_udf/r/xsa_sanity.result

@@ -95,21 +95,6 @@ ERROR HY000: asymmetric_decrypt<string> UDF failed; decrypting with RSA public k
 SELECT asymmetric_decrypt(@algorithm, @message, @priv, 'oaep', '');
 ERROR HY000: Can't initialize function 'asymmetric_decrypt'; Function requires three or four arguments
 
-** checking that by default 'asymmetric_encrypt()' / 'asymmetric_decrypt()'
-** functions use PKCS1 OAEP padding
-SET @oaep_encrypted_message = asymmetric_encrypt(@algorithm, @message, @pub);
-include/assert.inc [[oaep encryption] specifying "no" padding explicitly in asymmetric_decrypt() must succeed but produce garbage]
-include/assert.inc [[oaep encryption] specifying "pkcs1" padding explicitly in asymmetric_decrypt() must succeed but produce garbage]
-include/assert.inc [[oaep encryption] specifying "oaep" padding explicitly in asymmetric_decrypt() must succeed]
-include/assert.inc [[oaep encryption] not specifying padding explicitly in asymmetric_decrypt() must succeed]
-SET @pkcs1_encrypted_message = asymmetric_encrypt(@algorithm, @message, @pub, 'pkcs1');
-include/assert.inc [[pkcs1 encryption] specifying "no" padding explicitly in asymmetric_decrypt() must succeed but produce garbage]
-include/assert.inc [[pkcs1 encryption] specifying "pkcs1" padding explicitly in asymmetric_decrypt() must succeed]
-SELECT asymmetric_decrypt(@algorithm, @pkcs1_encrypted_message, @priv, 'oaep');
-ERROR HY000: asymmetric_decrypt<string> UDF failed; cannot encrypt data block with the specified private RSA key
-SELECT asymmetric_decrypt(@algorithm, @pkcs1_encrypted_message, @priv);
-ERROR HY000: asymmetric_decrypt<string> UDF failed; cannot encrypt data block with the specified private RSA key
-
 ** creating sample digest
 SET @message_digest = create_digest(@digest_type, @message);