From: Viktor Dukhovni <openssl-users@dukhovni.org>
Date: Thu, 11 Sep 2025 08:50:44 +0000 (+1000)
Subject: Harden RSA public encrypt
X-Git-Tag: openssl-3.5.3~5
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0b776b5fcfa6461f3394b72d11bfceab41d07ed9;p=thirdparty%2Fopenssl.git

Harden RSA public encrypt

Check the that the indicated output buffer length is large enough.

Fix EVP_SealInit() to initialise the output buffer length to the RSA
modulus length, not the input KEK length.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28517)

(cherry picked from commit f34b1ad17ba37a76a96e53be3813d88bf2b329f2)
---

diff --git a/CHANGES.md b/CHANGES.md
index cd7867cd148..ef7e2dfa687 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -30,6 +30,19 @@ OpenSSL 3.5
 
 ### Changes between 3.5.2 and 3.5.3 [xx XXX xxxx]
 
+ * Hardened the provider implementation of the RSA public key "encrypt"
+   operation to add a missing check that the caller-indicated output buffer
+   size is at least as large as the byte count of the RSA modulus.  The issue
+   was reported by Arash Ale Ebrahim from SYSPWN.
+
+   This operation is typically invoked via `EVP_PKEY_encrypt(3)`.  Callers that
+   in fact provide a sufficiently large buffer, but fail to correctly indicate
+   its size may now encounter unexpected errors.  In applications that attempt
+   RSA public encryption into a buffer that is too small, an out-of-bounds
+   write is now avoided and an error is reported instead.
+
+   *Viktor Dukhovni*
+
  * Added PCT for key import for SLH-DSA when in FIPS mode.
 
    *Dr Paul Dale*
diff --git a/crypto/evp/p_seal.c b/crypto/evp/p_seal.c
index 94c8462ab45..99c640cc73e 100644
--- a/crypto/evp/p_seal.c
+++ b/crypto/evp/p_seal.c
@@ -55,6 +55,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
 
     for (i = 0; i < npubk; i++) {
         size_t keylen = len;
+        size_t outlen = EVP_PKEY_get_size(pubk[i]);
 
         pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pubk[i], NULL);
         if (pctx == NULL) {
@@ -63,9 +64,9 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
         }
 
         if (EVP_PKEY_encrypt_init(pctx) <= 0
-            || EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
+            || EVP_PKEY_encrypt(pctx, ek[i], &outlen, key, keylen) <= 0)
             goto err;
-        ekl[i] = (int)keylen;
+        ekl[i] = (int)outlen;
         EVP_PKEY_CTX_free(pctx);
     }
     pctx = NULL;
diff --git a/providers/implementations/asymciphers/rsa_enc.c b/providers/implementations/asymciphers/rsa_enc.c
index 6ee127caff8..00a00de7412 100644
--- a/providers/implementations/asymciphers/rsa_enc.c
+++ b/providers/implementations/asymciphers/rsa_enc.c
@@ -151,6 +151,7 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
                        size_t outsize, const unsigned char *in, size_t inlen)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
+    size_t len = RSA_size(prsactx->rsa);
     int ret;
 
     if (!ossl_prov_is_running())
@@ -168,17 +169,21 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
     }
 #endif
 
-    if (out == NULL) {
-        size_t len = RSA_size(prsactx->rsa);
+    if (len == 0) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+        return 0;
+    }
 
-        if (len == 0) {
-            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
-            return 0;
-        }
+    if (out == NULL) {
         *outlen = len;
         return 1;
     }
 
+    if (outsize < len) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
+        return 0;
+    }
+
     if (prsactx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
         int rsasize = RSA_size(prsactx->rsa);
         unsigned char *tbuf;