Reject clearly degenerate RSASVE parameters.

- A public exponent <= 1
- Ciphertext <= 1 or equal to n-1, where n is the public modulus.

Reviewed-by: Nikola Pajkovsky <nikolap@openssl.org>
Reviewed-by: Bob Beck <beck@openssl.org>
MergeDate: Mon Aug 10 06:27:45 2026
(Merged from https://github.com/openssl/openssl/pull/32124)

diff --git a/crypto/rsa/rsa_ossl.c b/crypto/rsa/rsa_ossl.c
index 674ee4b10d755008a9657419a8ce3f11af8d3a0a..8a3deae581d84f3e669f3ff484001e1e2067788e 100644 (file)
--- a/crypto/rsa/rsa_ossl.c
+++ b/crypto/rsa/rsa_ossl.c
@@ -564,7 +564,6 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
     if (BN_bin2bn(from, (int)flen, f) == NULL)
         goto err;
 
-#ifdef FIPS_MODULE
     /*
      * See SP800-56Br2, section 7.1.2.1
      * RSADP: 1 < f < (n – 1)
@@ -585,9 +584,7 @@ static int rsa_ossl_private_decrypt(int flen, const unsigned char *from,
             ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
             goto err;
         }
-    } else
-#endif
-    {
+    } else {
         if (BN_ucmp(f, rsa->n) >= 0) {
             ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
             goto err;

diff --git a/providers/implementations/kem/rsa_kem.c b/providers/implementations/kem/rsa_kem.c
index ab28a3a1a2808dff4160d2280cfe954d46eed305..729c5ac6be84b3ded7aedf3e2fa5cfb952e04a1b 100644 (file)
--- a/providers/implementations/kem/rsa_kem.c
+++ b/providers/implementations/kem/rsa_kem.c
@@ -140,6 +140,7 @@ static int rsakem_init(void *vprsactx, void *vrsa,
     const char *desc)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
+    const BIGNUM *e = NULL;
     int protect = 0;
 
     if (!ossl_prov_is_running())
@@ -155,6 +156,18 @@ static int rsakem_init(void *vprsactx, void *vrsa,
     RSA_free(prsactx->rsa);
     prsactx->rsa = vrsa;
 
+    /*
+     * Reject the trivial public exponent e <= 1. The FIPS module enforces the
+     * full SP 800-56B §6.4.1.1 constraints via ossl_fips_ind_rsa_key_check()
+     * below; non-FIPS callers wanting the complete §6.4.2 vetting can use
+     * EVP_PKEY_public_check().
+     */
+    RSA_get0_key(prsactx->rsa, NULL, &e, NULL);
+    if (e == NULL || BN_cmp(e, BN_value_one()) <= 0) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+        return 0;
+    }
+
     OSSL_FIPS_IND_SET_APPROVED(prsactx)
     if (!rsakem_set_ctx_params(prsactx, params))
         return 0;

diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 14eaa2a37f07321fa0f5a3a1edf5cbc44f7d0f3c..177eef4b186dbe0989b09ce46321366bd0bc0cb0 100644 (file)
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -8833,6 +8833,135 @@ end:
     return testresult;
 }
 
+/*
+ * RSASVE (SP 800-56B 7.2) must reject mathematically degenerate inputs:
+ * a public exponent e <= 1, and a ciphertext c in {0, 1, n - 1}.  Outside
+ * the FIPS module these were previously accepted; the checks now apply to
+ * every build, so exercise them in the default provider.
+ */
+
+/*
+ * With e <= 1 the RSA public operation is the identity (or worse), so
+ * encapsulation setup must reject the key with PROV_R_INVALID_KEY.  idx
+ * selects the exponent: 0 or 1.
+ */
+static int test_rsasve_degenerate_exponent(int idx)
+{
+    EVP_PKEY *rsakey = NULL;
+    EVP_PKEY *pubkey = NULL;
+    EVP_PKEY_CTX *genctx = NULL;
+    EVP_PKEY_CTX *ctx = NULL;
+    OSSL_PARAM_BLD *bld = NULL;
+    OSSL_PARAM *params = NULL;
+    BIGNUM *n = NULL;
+    BIGNUM *e = NULL;
+    int testresult = 0;
+
+    /* Borrow a real modulus; only the exponent is degenerate. */
+    if (!TEST_ptr(rsakey = load_example_rsa_key())
+        || !TEST_true(EVP_PKEY_get_bn_param(rsakey, OSSL_PKEY_PARAM_RSA_N, &n)))
+        goto err;
+
+    if (!TEST_ptr(e = BN_new())
+        || !TEST_true(BN_set_word(e, (BN_ULONG)idx))) /* idx is 0 or 1 */
+        goto err;
+
+    if (!TEST_ptr(bld = OSSL_PARAM_BLD_new())
+        || !TEST_true(OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_RSA_N, n))
+        || !TEST_true(OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_RSA_E, e))
+        || !TEST_ptr(params = OSSL_PARAM_BLD_to_param(bld)))
+        goto err;
+
+    if (!TEST_ptr(genctx = EVP_PKEY_CTX_new_from_name(testctx, "RSA", NULL))
+        || !TEST_int_gt(EVP_PKEY_fromdata_init(genctx), 0)
+        || !TEST_int_gt(EVP_PKEY_fromdata(genctx, &pubkey, EVP_PKEY_PUBLIC_KEY,
+                            params),
+            0))
+        goto err;
+
+    ERR_clear_error();
+    if (!TEST_ptr(ctx = EVP_PKEY_CTX_new_from_pkey(testctx, pubkey, NULL))
+        || !TEST_int_eq(EVP_PKEY_encapsulate_init(ctx, NULL), 0)
+        || !TEST_int_eq(ERR_GET_REASON(ERR_get_error()), PROV_R_INVALID_KEY))
+        goto err;
+
+    testresult = 1;
+err:
+    EVP_PKEY_CTX_free(ctx);
+    EVP_PKEY_CTX_free(genctx);
+    EVP_PKEY_free(pubkey);
+    EVP_PKEY_free(rsakey);
+    OSSL_PARAM_free(params);
+    OSSL_PARAM_BLD_free(bld);
+    BN_free(e);
+    BN_free(n);
+    return testresult;
+}
+
+/*
+ * A ciphertext c in {0, 1, n - 1} is a fixed point or trivial case of RSADP,
+ * so RSASVE recovery must reject it.  idx selects the ciphertext: 0, 1, or
+ * n - 1.  The ciphertext length must equal the modulus length.
+ */
+static int test_rsasve_degenerate_ciphertext(int idx)
+{
+    EVP_PKEY *rsakey = NULL;
+    EVP_PKEY_CTX *ctx = NULL;
+    BIGNUM *n = NULL;
+    unsigned char *ct = NULL;
+    unsigned char *secret = NULL;
+    size_t ctlen = 0;
+    size_t secretlen = 0;
+    int expected_reason = 0;
+    int testresult = 0;
+
+    if (!TEST_ptr(rsakey = load_example_rsa_key())
+        || !TEST_true(EVP_PKEY_get_bn_param(rsakey, OSSL_PKEY_PARAM_RSA_N, &n)))
+        goto err;
+
+    ctlen = secretlen = (size_t)EVP_PKEY_get_size(rsakey);
+    if (!TEST_ptr(ct = OPENSSL_zalloc(ctlen))
+        || !TEST_ptr(secret = OPENSSL_malloc(secretlen)))
+        goto err;
+
+    switch (idx) {
+    case 0: /* c = 0 */
+        expected_reason = RSA_R_DATA_TOO_SMALL;
+        break;
+    case 1: /* c = 1 */
+        ct[ctlen - 1] = 1;
+        expected_reason = RSA_R_DATA_TOO_SMALL;
+        break;
+    case 2: /* c = n - 1 */
+        if (!TEST_true(BN_sub_word(n, 1))
+            || !TEST_int_eq(BN_bn2binpad(n, ct, (int)ctlen), (int)ctlen))
+            goto err;
+        expected_reason = RSA_R_DATA_TOO_LARGE_FOR_MODULUS;
+        break;
+    default:
+        goto err;
+    }
+
+    if (!TEST_ptr(ctx = EVP_PKEY_CTX_new_from_pkey(testctx, rsakey, NULL))
+        || !TEST_int_eq(EVP_PKEY_decapsulate_init(ctx, NULL), 1)
+        || !TEST_int_eq(EVP_PKEY_CTX_set_kem_op(ctx, "RSASVE"), 1))
+        goto err;
+
+    ERR_clear_error();
+    if (!TEST_int_eq(EVP_PKEY_decapsulate(ctx, secret, &secretlen, ct, ctlen), 0)
+        || !TEST_int_eq(ERR_GET_REASON(ERR_get_error()), expected_reason))
+        goto err;
+
+    testresult = 1;
+err:
+    OPENSSL_free(secret);
+    OPENSSL_free(ct);
+    EVP_PKEY_CTX_free(ctx);
+    EVP_PKEY_free(rsakey);
+    BN_free(n);
+    return testresult;
+}
+
 #ifndef OPENSSL_NO_DEPRECATED_3_0
 
 static int sign_hits = 0;
@@ -9492,6 +9621,9 @@ int setup_tests(void)
 
     ADD_TEST(test_evp_cipher_pipeline);
 
+    ADD_ALL_TESTS(test_rsasve_degenerate_exponent, 2);
+    ADD_ALL_TESTS(test_rsasve_degenerate_ciphertext, 3);
+
 #ifndef OPENSSL_NO_ML_KEM
     ADD_ALL_TESTS(test_ml_kem_seed_only, 2);
 #endif