Harden RSA public encrypt

Check the that the indicated output buffer length is large enough.

Fix EVP_SealInit() to initialise the output buffer length to the RSA
modulus length, not the input KEK length.

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28536)

diff --git a/CHANGES.md b/CHANGES.md
index 491cc94f6982c46a8af0b08eafd159067570a405..4a68af1435a1e5ff04bff24b44886221e12a0cb0 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -30,7 +30,18 @@ breaking changes, and mappings for the large list of deprecated functions.
 
 ### Changes between 3.0.17 and 3.0.18 [xx XXX xxxx]
 
- * none yet
+ * Hardened the provider implementation of the RSA public key "encrypt"
+   operation to add a missing check that the caller-indicated output buffer
+   size is at least as large as the byte count of the RSA modulus.  The issue
+   was reported by Arash Ale Ebrahim from SYSPWN.
+
+   This operation is typically invoked via `EVP_PKEY_encrypt(3)`.  Callers that
+   in fact provide a sufficiently large buffer, but fail to correctly indicate
+   its size may now encounter unexpected errors.  In applications that attempt
+   RSA public encryption into a buffer that is too small, an out-of-bounds
+   write is now avoided and an error is reported instead.
+
+   *Viktor Dukhovni*
 
 ### Changes between 3.0.16 and 3.0.17 [1 Jul 2025]
 

diff --git a/crypto/evp/p_seal.c b/crypto/evp/p_seal.c
index 475082d431161d89f186e6b3810e7f0a584b068f..426b0ee9f20c7de36a7c901a3c208e1342d69188 100644 (file)
--- a/crypto/evp/p_seal.c
+++ b/crypto/evp/p_seal.c
@@ -56,6 +56,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
 
     for (i = 0; i < npubk; i++) {
         size_t keylen = len;
+        size_t outlen = EVP_PKEY_get_size(pubk[i]);
 
         pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pubk[i], NULL);
         if (pctx == NULL) {
@@ -64,9 +65,9 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
         }
 
         if (EVP_PKEY_encrypt_init(pctx) <= 0
-            || EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
+            || EVP_PKEY_encrypt(pctx, ek[i], &outlen, key, keylen) <= 0)
             goto err;
-        ekl[i] = (int)keylen;
+        ekl[i] = (int)outlen;
         EVP_PKEY_CTX_free(pctx);
     }
     pctx = NULL;

diff --git a/providers/implementations/asymciphers/rsa_enc.c b/providers/implementations/asymciphers/rsa_enc.c
index c8921acd6e615145e7c2a84e0a785408ec020b99..f4ca96ebea5c8dfb312e036887d95c853979aba9 100644 (file)
--- a/providers/implementations/asymciphers/rsa_enc.c
+++ b/providers/implementations/asymciphers/rsa_enc.c
@@ -136,22 +136,27 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
                        size_t outsize, const unsigned char *in, size_t inlen)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
+    size_t len = RSA_size(prsactx->rsa);
     int ret;
 
     if (!ossl_prov_is_running())
         return 0;
 
-    if (out == NULL) {
-        size_t len = RSA_size(prsactx->rsa);
+    if (len == 0) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+        return 0;
+    }
 
-        if (len == 0) {
-            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
-            return 0;
-        }
+    if (out == NULL) {
         *outlen = len;
         return 1;
     }
 
+    if (outsize < len) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
+        return 0;
+    }
+
     if (prsactx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
         int rsasize = RSA_size(prsactx->rsa);
         unsigned char *tbuf;