rsa_kem: test RSA_public_encrypt() result in RSASVE

RSA_public_encrypt() returns the number of bytes written on success and
-1 on failure.

Add regression coverage in evp_extra_test using custom low-level RSA
methods to exercise the provider/legacy boundary. The new tests verify
that encapsulation fails when RSA_public_encrypt() returns:

  * -1, which is the documented failure result, and
  * a short positive length, which is also invalid for RSASVE with
    RSA_NO_PADDING because the ciphertext must be exactly nlen bytes.

Signed-off-by: Nikola Pajkovsky <nikolap@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.foundation>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
MergeDate: Mon Apr  6 19:45:39 2026
(cherry picked from commit 4c92661c45b6af78a901ee97db6f29a2ce90ae29)

diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 3dba83c273906005d2fbaae95e7adce007b707ac..51843a9f8189eef7f9643867fc0ec2bafe99e220 100644 (file)
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -6385,6 +6385,8 @@ end:
 #ifndef OPENSSL_NO_DEPRECATED_3_0
 
 static int sign_hits = 0;
+static int encap_hits = 0;
+static int flen_ne_ret_hits = 0;
 
 static int do_sign_with_method(EVP_PKEY *pkey)
 {
@@ -6427,6 +6429,28 @@ static int tst_rsa_priv_enc(int flen, const unsigned char *from, unsigned char *
     return orig_rsa_priv_enc(flen, from, to, rsa, padding);
 }
 
+static int tst_rsa_pub_enc(int flen, const unsigned char *from, unsigned char *to,
+    RSA *rsa, int padding)
+{
+    const char *marker = RSA_get_ex_data(rsa, rsa_ex_idx);
+
+    if (marker == NULL || strcmp(marker, "kem-test") != 0)
+        return 0;
+    encap_hits++;
+    return -1;
+}
+
+static int tst_rsa_pub_enc_flen_ne_ret(int flen, const unsigned char *from, unsigned char *to,
+    RSA *rsa, int padding)
+{
+    const char *marker = RSA_get_ex_data(rsa, rsa_ex_idx);
+
+    if (marker == NULL || strcmp(marker, "kem-test-flen-ne-ret") != 0)
+        return 0;
+    flen_ne_ret_hits++;
+    return flen - 1;
+}
+
 /* Test that a low level RSA method still gets used even with a provider */
 static int test_low_level_rsa_method(void)
 {
@@ -6483,6 +6507,86 @@ err:
     return testresult;
 }
 
+static int test_low_level_rsa_kem_public_encrypt_failure(int idx)
+{
+    RSA *rsa = NULL;
+    const RSA_METHOD *def = RSA_get_default_method();
+    RSA_METHOD *method = RSA_meth_dup(def);
+    EVP_PKEY *pkey = NULL;
+    EVP_PKEY_CTX *ctx = NULL;
+    unsigned char *ct = NULL;
+    unsigned char *secret = NULL;
+    size_t ctlen = 0, secretlen = 0;
+    int testresult = 0;
+
+    if (nullprov != NULL) {
+        testresult = TEST_skip("Test does not support a non-default library context");
+        goto err;
+    }
+
+    if (!TEST_ptr(method)
+        || !TEST_ptr(pkey = load_example_rsa_key()))
+        goto err;
+
+    rsa_ex_idx = RSA_get_ex_new_index(0, NULL, NULL, NULL, NULL);
+    if (!TEST_int_ne(rsa_ex_idx, -1) || !TEST_ptr(rsa = EVP_PKEY_get1_RSA(pkey)))
+        goto err;
+
+    switch (idx) {
+    case 0:
+        if (!TEST_true(RSA_set_ex_data(rsa, rsa_ex_idx, (void *)"kem-test"))
+            || !TEST_true(RSA_meth_set_pub_enc(method, tst_rsa_pub_enc)))
+            goto err;
+        break;
+    case 1:
+        if (!TEST_true(RSA_set_ex_data(rsa, rsa_ex_idx, (void *)"kem-test-flen-ne-ret"))
+            || !TEST_true(RSA_meth_set_pub_enc(method, tst_rsa_pub_enc_flen_ne_ret)))
+            goto err;
+        break;
+    default:
+        goto err;
+    };
+    if (!TEST_true(RSA_set_method(rsa, method))
+        || !TEST_int_gt(EVP_PKEY_assign_RSA(pkey, rsa), 0))
+        goto err;
+    rsa = NULL;
+
+    if (!TEST_ptr(ctx = EVP_PKEY_CTX_new_from_pkey(testctx, pkey, NULL))
+        || !TEST_int_eq(EVP_PKEY_encapsulate_init(ctx, NULL), 1)
+        || !TEST_int_eq(EVP_PKEY_CTX_set_kem_op(ctx, "RSASVE"), 1)
+        || !TEST_int_eq(EVP_PKEY_encapsulate(ctx, NULL, &ctlen, NULL, &secretlen), 1)
+        || !TEST_ptr(ct = OPENSSL_malloc(ctlen))
+        || !TEST_ptr(secret = OPENSSL_malloc(secretlen)))
+        goto err;
+
+    encap_hits = flen_ne_ret_hits = 0;
+    if (!TEST_int_eq(EVP_PKEY_encapsulate(ctx, ct, &ctlen, secret, &secretlen), 0))
+        goto err;
+
+    switch (idx) {
+    case 0:
+        if (!TEST_int_eq(encap_hits, 1))
+            goto err;
+        break;
+    case 1:
+        if (!TEST_int_eq(flen_ne_ret_hits, 1))
+            goto err;
+        break;
+    default:
+        goto err;
+    }
+    testresult = 1;
+
+err:
+    OPENSSL_free(secret);
+    OPENSSL_free(ct);
+    EVP_PKEY_CTX_free(ctx);
+    RSA_free(rsa);
+    EVP_PKEY_free(pkey);
+    RSA_meth_free(method);
+    return testresult;
+}
+
 #ifndef OPENSSL_NO_DSA
 static int dsa_ex_idx = -1;
 
@@ -6921,6 +7025,7 @@ int setup_tests(void)
 
 #ifndef OPENSSL_NO_DEPRECATED_3_0
     ADD_TEST(test_low_level_rsa_method);
+    ADD_ALL_TESTS(test_low_level_rsa_kem_public_encrypt_failure, 2);
 #ifndef OPENSSL_NO_DSA
     ADD_TEST(test_low_level_dsa_method);
 #endif