Test failure of rsa_encrypt when buffer too short

Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28539)

diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 5889ff8a4e9d8ec45ec39927829b8d9a26506898..ac7517c4eab929f8978e89b816701526955d86f2 100644 (file)
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -3208,6 +3208,48 @@ err:
 }
 #endif
 
+static int test_RSA_encrypt(void)
+{
+    int ret = 0;
+    EVP_PKEY *pkey = NULL;
+    EVP_PKEY_CTX *pctx = NULL;
+    unsigned char *cbuf = NULL, *pbuf = NULL;
+    size_t clen = 0, plen = 0;
+
+    if (!TEST_ptr(pkey = load_example_rsa_key())
+        || !TEST_ptr(pctx = EVP_PKEY_CTX_new_from_pkey(testctx,
+                                                       pkey, testpropq))
+        || !TEST_int_gt(EVP_PKEY_encrypt_init(pctx), 0)
+        || !TEST_int_gt(EVP_PKEY_encrypt(pctx, cbuf, &clen, kMsg, sizeof(kMsg)), 0)
+        || !TEST_ptr(cbuf = OPENSSL_malloc(clen))
+        || !TEST_int_gt(EVP_PKEY_encrypt(pctx, cbuf, &clen, kMsg, sizeof(kMsg)), 0))
+        goto done;
+
+    /* Require failure when the output buffer is too small */
+    plen = clen - 1;
+    if (!TEST_int_le(EVP_PKEY_encrypt(pctx, cbuf, &plen, kMsg, sizeof(kMsg)), 0))
+        goto done;
+    /* flush error stack */
+    TEST_openssl_errors();
+
+    /* Check decryption of encrypted result */
+    if (!TEST_int_gt(EVP_PKEY_decrypt_init(pctx), 0)
+        || !TEST_int_gt(EVP_PKEY_decrypt(pctx, pbuf, &plen, cbuf, clen), 0)
+        || !TEST_ptr(pbuf = OPENSSL_malloc(plen))
+        || !TEST_int_gt(EVP_PKEY_decrypt(pctx, pbuf, &plen, cbuf, clen), 0)
+        || !TEST_mem_eq(pbuf, plen, kMsg, sizeof(kMsg))
+        || !TEST_int_gt(EVP_PKEY_encrypt_init(pctx), 0))
+        goto done;
+
+    ret = 1;
+done:
+    EVP_PKEY_CTX_free(pctx);
+    EVP_PKEY_free(pkey);
+    OPENSSL_free(cbuf);
+    OPENSSL_free(pbuf);
+    return ret;
+}
+
 #if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305)
 static int test_decrypt_null_chunks(void)
 {
@@ -5753,6 +5795,7 @@ int setup_tests(void)
     ADD_TEST(test_RSA_get_set_params);
     ADD_TEST(test_RSA_OAEP_set_get_params);
     ADD_TEST(test_RSA_OAEP_set_null_label);
+    ADD_TEST(test_RSA_encrypt);
 #ifndef OPENSSL_NO_DEPRECATED_3_0
     ADD_TEST(test_RSA_legacy);
 #endif