From 6c6a91b1436aeec332a2482ae1321936ac987e2e Mon Sep 17 00:00:00 2001 From: Viktor Dukhovni Date: Sat, 13 Sep 2025 12:52:42 +1000 Subject: [PATCH] Test failure of rsa_encrypt when buffer too short Reviewed-by: Neil Horman Reviewed-by: Tomas Mraz (Merged from https://github.com/openssl/openssl/pull/28539) --- test/evp_extra_test.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c index 5889ff8a4e9..ac7517c4eab 100644 --- a/test/evp_extra_test.c +++ b/test/evp_extra_test.c @@ -3208,6 +3208,48 @@ err: } #endif +static int test_RSA_encrypt(void) +{ + int ret = 0; + EVP_PKEY *pkey = NULL; + EVP_PKEY_CTX *pctx = NULL; + unsigned char *cbuf = NULL, *pbuf = NULL; + size_t clen = 0, plen = 0; + + if (!TEST_ptr(pkey = load_example_rsa_key()) + || !TEST_ptr(pctx = EVP_PKEY_CTX_new_from_pkey(testctx, + pkey, testpropq)) + || !TEST_int_gt(EVP_PKEY_encrypt_init(pctx), 0) + || !TEST_int_gt(EVP_PKEY_encrypt(pctx, cbuf, &clen, kMsg, sizeof(kMsg)), 0) + || !TEST_ptr(cbuf = OPENSSL_malloc(clen)) + || !TEST_int_gt(EVP_PKEY_encrypt(pctx, cbuf, &clen, kMsg, sizeof(kMsg)), 0)) + goto done; + + /* Require failure when the output buffer is too small */ + plen = clen - 1; + if (!TEST_int_le(EVP_PKEY_encrypt(pctx, cbuf, &plen, kMsg, sizeof(kMsg)), 0)) + goto done; + /* flush error stack */ + TEST_openssl_errors(); + + /* Check decryption of encrypted result */ + if (!TEST_int_gt(EVP_PKEY_decrypt_init(pctx), 0) + || !TEST_int_gt(EVP_PKEY_decrypt(pctx, pbuf, &plen, cbuf, clen), 0) + || !TEST_ptr(pbuf = OPENSSL_malloc(plen)) + || !TEST_int_gt(EVP_PKEY_decrypt(pctx, pbuf, &plen, cbuf, clen), 0) + || !TEST_mem_eq(pbuf, plen, kMsg, sizeof(kMsg)) + || !TEST_int_gt(EVP_PKEY_encrypt_init(pctx), 0)) + goto done; + + ret = 1; +done: + EVP_PKEY_CTX_free(pctx); + EVP_PKEY_free(pkey); + OPENSSL_free(cbuf); + OPENSSL_free(pbuf); + return ret; +} + #if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305) static int test_decrypt_null_chunks(void) { @@ -5753,6 +5795,7 @@ int setup_tests(void) ADD_TEST(test_RSA_get_set_params); ADD_TEST(test_RSA_OAEP_set_get_params); ADD_TEST(test_RSA_OAEP_set_null_label); + ADD_TEST(test_RSA_encrypt); #ifndef OPENSSL_NO_DEPRECATED_3_0 ADD_TEST(test_RSA_legacy); #endif -- 2.47.3