From: Viktor Dukhovni <openssl-users@dukhovni.org>
Date: Sat, 13 Sep 2025 02:52:42 +0000 (+1000)
Subject: Test failure of rsa_encrypt when buffer too short
X-Git-Tag: openssl-3.2.6~35
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=db266724813fecdc4348fad4fd0f2f7424f577c0;p=thirdparty%2Fopenssl.git

Test failure of rsa_encrypt when buffer too short

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28538)
---

diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 35917fd4428..c27a990a520 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -3144,6 +3144,48 @@ static int test_RSA_OAEP_set_null_label(void)
     return ret;
 }
 
+static int test_RSA_encrypt(void)
+{
+    int ret = 0;
+    EVP_PKEY *pkey = NULL;
+    EVP_PKEY_CTX *pctx = NULL;
+    unsigned char *cbuf = NULL, *pbuf = NULL;
+    size_t clen = 0, plen = 0;
+
+    if (!TEST_ptr(pkey = load_example_rsa_key())
+        || !TEST_ptr(pctx = EVP_PKEY_CTX_new_from_pkey(testctx,
+                                                       pkey, testpropq))
+        || !TEST_int_gt(EVP_PKEY_encrypt_init(pctx), 0)
+        || !TEST_int_gt(EVP_PKEY_encrypt(pctx, cbuf, &clen, kMsg, sizeof(kMsg)), 0)
+        || !TEST_ptr(cbuf = OPENSSL_malloc(clen))
+        || !TEST_int_gt(EVP_PKEY_encrypt(pctx, cbuf, &clen, kMsg, sizeof(kMsg)), 0))
+        goto done;
+
+    /* Require failure when the output buffer is too small */
+    plen = clen - 1;
+    if (!TEST_int_le(EVP_PKEY_encrypt(pctx, cbuf, &plen, kMsg, sizeof(kMsg)), 0))
+        goto done;
+    /* flush error stack */
+    TEST_openssl_errors();
+
+    /* Check decryption of encrypted result */
+    if (!TEST_int_gt(EVP_PKEY_decrypt_init(pctx), 0)
+        || !TEST_int_gt(EVP_PKEY_decrypt(pctx, pbuf, &plen, cbuf, clen), 0)
+        || !TEST_ptr(pbuf = OPENSSL_malloc(plen))
+        || !TEST_int_gt(EVP_PKEY_decrypt(pctx, pbuf, &plen, cbuf, clen), 0)
+        || !TEST_mem_eq(pbuf, plen, kMsg, sizeof(kMsg))
+        || !TEST_int_gt(EVP_PKEY_encrypt_init(pctx), 0))
+        goto done;
+
+    ret = 1;
+done:
+    EVP_PKEY_CTX_free(pctx);
+    EVP_PKEY_free(pkey);
+    OPENSSL_free(cbuf);
+    OPENSSL_free(pbuf);
+    return ret;
+}
+
 #if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305)
 static int test_decrypt_null_chunks(void)
 {
@@ -5680,6 +5722,7 @@ int setup_tests(void)
     ADD_TEST(test_RSA_get_set_params);
     ADD_TEST(test_RSA_OAEP_set_get_params);
     ADD_TEST(test_RSA_OAEP_set_null_label);
+    ADD_TEST(test_RSA_encrypt);
 #if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305)
     ADD_TEST(test_decrypt_null_chunks);
 #endif