Harden RSA public encrypt

Check the that the indicated output buffer length is large enough.

Fix EVP_SealInit() to initialise the output buffer length to the RSA
modulus length, not the input KEK length.

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28539)

diff --git a/CHANGES.md b/CHANGES.md
index 59fc513239abc92038befc0b90f1ed3e0c7669ce..20f2178c79ff41d1b9d9180711ea39b2fe641b06 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -28,7 +28,18 @@ OpenSSL 3.3
 
 ### Changes between 3.3.4 and 3.3.5 [xx XXX xxxx]
 
- * none yet
+ * Hardened the provider implementation of the RSA public key "encrypt"
+   operation to add a missing check that the caller-indicated output buffer
+   size is at least as large as the byte count of the RSA modulus.  The issue
+   was reported by Arash Ale Ebrahim from SYSPWN.
+
+   This operation is typically invoked via `EVP_PKEY_encrypt(3)`.  Callers that
+   in fact provide a sufficiently large buffer, but fail to correctly indicate
+   its size may now encounter unexpected errors.  In applications that attempt
+   RSA public encryption into a buffer that is too small, an out-of-bounds
+   write is now avoided and an error is reported instead.
+
+   *Viktor Dukhovni*
 
 ### Changes between 3.3.3 and 3.3.4 [1 Jul 2025]
 

diff --git a/crypto/evp/p_seal.c b/crypto/evp/p_seal.c
index 94c8462ab457081e4b6cb5605845fd94f71e0646..99c640cc73ea74a3fb67779284d3bf53de48e661 100644 (file)
--- a/crypto/evp/p_seal.c
+++ b/crypto/evp/p_seal.c
@@ -55,6 +55,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
 
     for (i = 0; i < npubk; i++) {
         size_t keylen = len;
+        size_t outlen = EVP_PKEY_get_size(pubk[i]);
 
         pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pubk[i], NULL);
         if (pctx == NULL) {
@@ -63,9 +64,9 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
         }
 
         if (EVP_PKEY_encrypt_init(pctx) <= 0
-            || EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
+            || EVP_PKEY_encrypt(pctx, ek[i], &outlen, key, keylen) <= 0)
             goto err;
-        ekl[i] = (int)keylen;
+        ekl[i] = (int)outlen;
         EVP_PKEY_CTX_free(pctx);
     }
     pctx = NULL;

diff --git a/providers/implementations/asymciphers/rsa_enc.c b/providers/implementations/asymciphers/rsa_enc.c
index 71bfa344d4864a002e056623c7d04166569ab3cb..fc882025cb787400413dd252ec225595a5126fbc 100644 (file)
--- a/providers/implementations/asymciphers/rsa_enc.c
+++ b/providers/implementations/asymciphers/rsa_enc.c
@@ -139,22 +139,27 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
                        size_t outsize, const unsigned char *in, size_t inlen)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
+    size_t len = RSA_size(prsactx->rsa);
     int ret;
 
     if (!ossl_prov_is_running())
         return 0;
 
-    if (out == NULL) {
-        size_t len = RSA_size(prsactx->rsa);
+    if (len == 0) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+        return 0;
+    }
 
-        if (len == 0) {
-            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
-            return 0;
-        }
+    if (out == NULL) {
         *outlen = len;
         return 1;
     }
 
+    if (outsize < len) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
+        return 0;
+    }
+
     if (prsactx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
         int rsasize = RSA_size(prsactx->rsa);
         unsigned char *tbuf;