Harden RSA public encrypt

Check the that the indicated output buffer length is large enough.

Fix EVP_SealInit() to initialise the output buffer length to the RSA
modulus length, not the input KEK length.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28517)

(cherry picked from commit f34b1ad17ba37a76a96e53be3813d88bf2b329f2)

diff --git a/CHANGES.md b/CHANGES.md
index 1549f97befc36c4c12e7a69ff5ede0d3f188369e..76118704e613d739c92f171e651f6bee3b84f342 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -29,7 +29,18 @@ OpenSSL 3.4
 
 ### Changes between 3.4.2 and 3.4.3 [xx XXX xxxx]
 
- * none yet
+ * Hardened the provider implementation of the RSA public key "encrypt"
+   operation to add a missing check that the caller-indicated output buffer
+   size is at least as large as the byte count of the RSA modulus.  The issue
+   was reported by Arash Ale Ebrahim from SYSPWN.
+
+   This operation is typically invoked via `EVP_PKEY_encrypt(3)`.  Callers that
+   in fact provide a sufficiently large buffer, but fail to correctly indicate
+   its size may now encounter unexpected errors.  In applications that attempt
+   RSA public encryption into a buffer that is too small, an out-of-bounds
+   write is now avoided and an error is reported instead.
+
+   *Viktor Dukhovni*
 
 ### Changes between 3.4.1 and 3.4.2 [1 Jul 2025]
 

diff --git a/crypto/evp/p_seal.c b/crypto/evp/p_seal.c
index 94c8462ab457081e4b6cb5605845fd94f71e0646..99c640cc73ea74a3fb67779284d3bf53de48e661 100644 (file)
--- a/crypto/evp/p_seal.c
+++ b/crypto/evp/p_seal.c
@@ -55,6 +55,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
 
     for (i = 0; i < npubk; i++) {
         size_t keylen = len;
+        size_t outlen = EVP_PKEY_get_size(pubk[i]);
 
         pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pubk[i], NULL);
         if (pctx == NULL) {
@@ -63,9 +64,9 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
         }
 
         if (EVP_PKEY_encrypt_init(pctx) <= 0
-            || EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
+            || EVP_PKEY_encrypt(pctx, ek[i], &outlen, key, keylen) <= 0)
             goto err;
-        ekl[i] = (int)keylen;
+        ekl[i] = (int)outlen;
         EVP_PKEY_CTX_free(pctx);
     }
     pctx = NULL;

diff --git a/providers/implementations/asymciphers/rsa_enc.c b/providers/implementations/asymciphers/rsa_enc.c
index 3e7fd0f488ea7d9e7207ccb0acc0c841256e09d6..b6e2da98f8d14bb10fa2f534995722b11a135d24 100644 (file)
--- a/providers/implementations/asymciphers/rsa_enc.c
+++ b/providers/implementations/asymciphers/rsa_enc.c
@@ -151,6 +151,7 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
                        size_t outsize, const unsigned char *in, size_t inlen)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
+    size_t len = RSA_size(prsactx->rsa);
     int ret;
 
     if (!ossl_prov_is_running())
@@ -168,17 +169,21 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
     }
 #endif
 
-    if (out == NULL) {
-        size_t len = RSA_size(prsactx->rsa);
+    if (len == 0) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+        return 0;
+    }
 
-        if (len == 0) {
-            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
-            return 0;
-        }
+    if (out == NULL) {
         *outlen = len;
         return 1;
     }
 
+    if (outsize < len) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
+        return 0;
+    }
+
     if (prsactx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
         int rsasize = RSA_size(prsactx->rsa);
         unsigned char *tbuf;