Harden RSA public encrypt

Check the that the indicated output buffer length is large enough.

Fix EVP_SealInit() to initialise the output buffer length to the RSA
modulus length, not the input KEK length.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28517)

diff --git a/CHANGES.md b/CHANGES.md
index 9c452570b5d1534974c8c8175baba224ec3f0a74..45a66f3dcec220583047fdc0c3d6aed5165b2d22 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -46,6 +46,19 @@ OpenSSL 3.6
 
 ### Changes between 3.5 and 3.6 [xx XXX xxxx]
 
+ * Hardened the provider implementation of the RSA public key "encrypt"
+   operation to add a missing check that the caller-indicated output buffer
+   size is at least as large as the byte count of the RSA modulus.  The issue
+   was reported by Arash Ale Ebrahim from SYSPWN.
+
+   This operation is typically invoked via `EVP_PKEY_encrypt(3)`.  Callers that
+   in fact provide a sufficiently large buffer, but fail to correctly indicate
+   its size may now encounter unexpected errors.  In applications that attempt
+   RSA public encryption into a buffer that is too small, an out-of-bounds
+   write is now avoided and an error is reported instead.
+
+   *Viktor Dukhovni*
+
  * Added support for EVP_SKEY opaque symmetric key objects to the key
    derivation and key exchange provider methods. Added `EVP_KDF_CTX_set_SKEY()`,
    `EVP_KDF_derive_SKEY()`, and `EVP_PKEY_derive_SKEY()` functions.

diff --git a/crypto/evp/p_seal.c b/crypto/evp/p_seal.c
index 94c8462ab457081e4b6cb5605845fd94f71e0646..99c640cc73ea74a3fb67779284d3bf53de48e661 100644 (file)
--- a/crypto/evp/p_seal.c
+++ b/crypto/evp/p_seal.c
@@ -55,6 +55,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
 
     for (i = 0; i < npubk; i++) {
         size_t keylen = len;
+        size_t outlen = EVP_PKEY_get_size(pubk[i]);
 
         pctx = EVP_PKEY_CTX_new_from_pkey(libctx, pubk[i], NULL);
         if (pctx == NULL) {
@@ -63,9 +64,9 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
         }
 
         if (EVP_PKEY_encrypt_init(pctx) <= 0
-            || EVP_PKEY_encrypt(pctx, ek[i], &keylen, key, keylen) <= 0)
+            || EVP_PKEY_encrypt(pctx, ek[i], &outlen, key, keylen) <= 0)
             goto err;
-        ekl[i] = (int)keylen;
+        ekl[i] = (int)outlen;
         EVP_PKEY_CTX_free(pctx);
     }
     pctx = NULL;

diff --git a/providers/implementations/asymciphers/rsa_enc.c.in b/providers/implementations/asymciphers/rsa_enc.c.in
index a165f5210983400d16f3d12473b97e5345ec72bd..6d1e97d934de549de8f47e2317c437dff8329471 100644 (file)
--- a/providers/implementations/asymciphers/rsa_enc.c.in
+++ b/providers/implementations/asymciphers/rsa_enc.c.in
@@ -155,6 +155,7 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
                        size_t outsize, const unsigned char *in, size_t inlen)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
+    size_t len = RSA_size(prsactx->rsa);
     int ret;
 
     if (!ossl_prov_is_running())
@@ -172,17 +173,21 @@ static int rsa_encrypt(void *vprsactx, unsigned char *out, size_t *outlen,
     }
 #endif
 
-    if (out == NULL) {
-        size_t len = RSA_size(prsactx->rsa);
+    if (len == 0) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+        return 0;
+    }
 
-        if (len == 0) {
-            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
-            return 0;
-        }
+    if (out == NULL) {
         *outlen = len;
         return 1;
     }
 
+    if (outsize < len) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
+        return 0;
+    }
+
     if (prsactx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
         int rsasize = RSA_size(prsactx->rsa);
         unsigned char *tbuf;