Fix buffer overflow when generating large RSA keys in FIPS mode.

A pairwise test runs only in FIPS mode.
An assumption about the size of the 'to' buffer passed to
RSA_private_decrypt() was incorrect. It needs to be up to RSA_size()
bytes long - so a fixed buffer of 256 bytes was not large enough.
An exiting malloc has increased in size to allocate buffer space for
both the encrypt and decrypt buffer.

The existing test used 2080 bits which was not quite large enough to
trigger the issue. A test using 3072 bits has been added.

Reported by Mark Powers from Acumen.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15447)

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 07a3a7800ee474066e2fc9ad4d96db1f0323f4ca..ac64483e6a2ffe3057e4976cd65caa76b60bffd8 100644 (file)
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -479,7 +479,7 @@ static int rsa_keygen_pairwise_test(RSA *rsa, OSSL_CALLBACK *cb, void *cbarg)
     unsigned int ciphertxt_len;
     unsigned char *ciphertxt = NULL;
     const unsigned char plaintxt[16] = {0};
-    unsigned char decoded[256];
+    unsigned char *decoded = NULL;
     unsigned int decoded_len;
     unsigned int plaintxt_len = (unsigned int)sizeof(plaintxt_len);
     int padding = RSA_PKCS1_PADDING;
@@ -492,9 +492,14 @@ static int rsa_keygen_pairwise_test(RSA *rsa, OSSL_CALLBACK *cb, void *cbarg)
                            OSSL_SELF_TEST_DESC_PCT_RSA_PKCS1);
 
     ciphertxt_len = RSA_size(rsa);
-    ciphertxt = OPENSSL_zalloc(ciphertxt_len);
+    /*
+     * RSA_private_encrypt() and RSA_private_decrypt() requires the 'to'
+     * parameter to be a maximum of RSA_size() - allocate space for both.
+     */
+    ciphertxt = OPENSSL_zalloc(ciphertxt_len * 2);
     if (ciphertxt == NULL)
         goto err;
+    decoded = ciphertxt + ciphertxt_len;
 
     ciphertxt_len = RSA_public_encrypt(plaintxt_len, plaintxt, ciphertxt, rsa,
                                        padding);

diff --git a/test/recipes/15-test_genrsa.t b/test/recipes/15-test_genrsa.t
index 95390c5ff4007f77ff3dc11711f5ca3a9f0839f3..501d3a100f6e4832e34be2bc0e7a4f2b5a5eaf2e 100644 (file)
--- a/test/recipes/15-test_genrsa.t
+++ b/test/recipes/15-test_genrsa.t
@@ -24,7 +24,7 @@ use lib bldtop_dir('.');
 my $no_fips = disabled('fips') || ($ENV{NO_FIPS} // 0);
 
 plan tests =>
-    ($no_fips ? 0 : 1)          # Extra FIPS related test
+    ($no_fips ? 0 : 2)          # Extra FIPS related test
     + 13;
 
 # We want to know that an absurdly small number of bits isn't support
@@ -131,4 +131,10 @@ unless ($no_fips) {
                '-pkeyopt', 'bits:2080',
                '-out', 'genrsatest2080.pem'])),
        "Generating RSA key with > 2048 bits and < 3072 bits");
+    ok(run(app(['openssl', 'genpkey',
+                @prov,
+               '-algorithm', 'RSA',
+               '-pkeyopt', 'bits:3072',
+               '-out', 'genrsatest3072.pem'])),
+       "Generating RSA key with 3072 bits");
 }