rsa_sig: reject short buffers in verify_recover

The RSA PKCS#1 verify-recover provider path did not validate routsize
before passing the caller buffer to ossl_rsa_verify().

The X9.31 verify-recover path already rejects undersized output buffers,
but the PKCS#1 path could proceed with too little output space and rely
on the lower layer to write the recovered digest.

Check the expected digest size before calling ossl_rsa_verify() and
return PROV_R_OUTPUT_BUFFER_TOO_SMALL when the caller-provided buffer is
too small.

Add a regression test that covers both successful recovery with a
properly sized buffer and failure with a 1-byte output buffer, while
also checking that the short buffer is left unchanged.

Co-authored-by: Kushal <72255307+Kushalkhemka@users.noreply.github.com>
Co-authored-by: Mayank <175295782+mayank-jangid-moon@users.noreply.github.com>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
MergeDate: Wed May 27 11:46:40 2026
(Merged from https://github.com/openssl/openssl/pull/30917)

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index a4c5a90741ca977acde4b947cb83a973f698c365..43f648e2d4268e9837e22a49989cc734e806d58f 100644 (file)
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -988,8 +988,19 @@ static int rsa_verify_recover(void *vprsactx,
             break;
 
         case RSA_PKCS1_PADDING: {
+            int mdsize = EVP_MD_get_size(prsactx->md);
             size_t sltmp;
 
+            if (mdsize <= 0) {
+                ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST_LENGTH);
+                return 0;
+            }
+            if (routsize < (size_t)mdsize) {
+                ERR_raise_data(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL,
+                    "buffer size is %d, should be %d",
+                    routsize, mdsize);
+                return 0;
+            }
             ret = ossl_rsa_verify(prsactx->mdnid, NULL, 0, rout, &sltmp,
                 sig, siglen, prsactx->rsa);
             if (ret <= 0) {

diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index f4bfeae2a1d6346a6c23f187fb8d17986b13db3b..0f805f6cec71ac8eeb06a26ce9cd802147e568aa 100644 (file)
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -4036,6 +4036,82 @@ err:
     return ret;
 }
 
+static int test_RSA_verify_recover_rejects_short_buffer(void)
+{
+    int ret = 0;
+    int recovered_cap = 0;
+    EVP_PKEY *pkey = NULL;
+    EVP_PKEY_CTX *sign_ctx = NULL, *verify_ctx = NULL;
+    unsigned char *sig = NULL, *recovered = NULL;
+    size_t sig_len = 0, recovered_len = 0;
+    unsigned long err = 0;
+    unsigned char shortbuf[] = { 0xa5, 0x5a };
+    const unsigned char shortbuf_expected[] = { 0xa5, 0x5a };
+    unsigned char digest[32];
+    size_t i;
+
+    for (i = 0; i < sizeof(digest); i++)
+        digest[i] = (unsigned char)i;
+
+    if (!TEST_ptr(pkey = load_example_rsa_key())
+        || !TEST_ptr(sign_ctx = EVP_PKEY_CTX_new_from_pkey(testctx, pkey, NULL))
+        || !TEST_int_gt(EVP_PKEY_sign_init(sign_ctx), 0)
+        || !TEST_int_gt(EVP_PKEY_CTX_set_rsa_padding(sign_ctx,
+                            RSA_PKCS1_PADDING),
+            0)
+        || !TEST_int_gt(EVP_PKEY_CTX_set_signature_md(sign_ctx, EVP_sha256()),
+            0)
+        || !TEST_int_gt(EVP_PKEY_sign(sign_ctx, NULL, &sig_len, digest,
+                            sizeof(digest)),
+            0)
+        || !TEST_ptr(sig = OPENSSL_malloc(sig_len))
+        || !TEST_int_gt(EVP_PKEY_sign(sign_ctx, sig, &sig_len, digest,
+                            sizeof(digest)),
+            0)
+        || !TEST_int_gt(recovered_cap = EVP_PKEY_get_size(pkey), 0)
+        || !TEST_ptr(recovered = OPENSSL_malloc(recovered_cap))
+        || !TEST_ptr(verify_ctx = EVP_PKEY_CTX_new_from_pkey(testctx, pkey,
+                         NULL))
+        || !TEST_int_gt(EVP_PKEY_verify_recover_init(verify_ctx), 0)
+        || !TEST_int_gt(EVP_PKEY_CTX_set_rsa_padding(verify_ctx,
+                            RSA_PKCS1_PADDING),
+            0)
+        || !TEST_int_gt(EVP_PKEY_CTX_set_signature_md(verify_ctx, EVP_sha256()),
+            0))
+        goto done;
+
+    recovered_len = (size_t)recovered_cap;
+    if (!TEST_int_gt(EVP_PKEY_verify_recover(verify_ctx, recovered,
+                         &recovered_len, sig, sig_len),
+            0)
+        || !TEST_size_t_eq(recovered_len, sizeof(digest))
+        || !TEST_mem_eq(recovered, recovered_len, digest, sizeof(digest)))
+        goto done;
+
+    ERR_clear_error();
+    recovered_len = 1;
+    if (!TEST_int_le(EVP_PKEY_verify_recover(verify_ctx, shortbuf,
+                         &recovered_len, sig, sig_len),
+            0))
+        goto done;
+
+    err = ERR_peek_error();
+    if (!TEST_int_eq(ERR_GET_LIB(err), ERR_LIB_PROV)
+        || !TEST_int_eq(ERR_GET_REASON(err), PROV_R_OUTPUT_BUFFER_TOO_SMALL)
+        || !TEST_mem_eq(shortbuf, sizeof(shortbuf), shortbuf_expected,
+            sizeof(shortbuf_expected)))
+        goto done;
+
+    ret = 1;
+done:
+    EVP_PKEY_CTX_free(sign_ctx);
+    EVP_PKEY_CTX_free(verify_ctx);
+    EVP_PKEY_free(pkey);
+    OPENSSL_free(sig);
+    OPENSSL_free(recovered);
+    return ret;
+}
+
 static int test_RSA_encrypt(void)
 {
     int ret = 0;
@@ -7958,6 +8034,7 @@ int setup_tests(void)
     ADD_TEST(test_RSA_get_set_params);
     ADD_TEST(test_RSA_OAEP_set_get_params);
     ADD_TEST(test_RSA_OAEP_set_null_label);
+    ADD_TEST(test_RSA_verify_recover_rejects_short_buffer);
     ADD_TEST(test_RSA_encrypt);
 #ifndef OPENSSL_NO_DEPRECATED_3_0
     ADD_TEST(test_RSA_legacy);