FIPS: Make keygen PCT failures recoverable

Key generation pairwise consistency tests (PCT) no longer cause the
FIPS provider to enter a non recoverable error state.

Originally I considered that the pairwise tests should never fail, so a
failure was considered as a fatal error. Unfortunately this is not true,
since the RSA pairwise test was changed to call
rsa_ossl_public_encrypt(). This function can return -1 based on the
values of n and e, resulting in a DOS for bad inputs.
Under NIST FIPS 186-5, the public exponent (e) must be an odd positive
integer greater than (e > 65536) and less than 2^256. The OpenSSL code
however returns an error if e > 2^64 when n > 3072.
(This check was added to prevent a CVE DOS).

While FIPS 140-3 mandates that a module must transition into an error
state upon self-test failures, a PCT is classified as a conditional
self-test, not a pre-operational self-test
(like an integrity test or a Known Answer Test)
The cryptographic module is only required to immediately discard the
faulty key pair and output a local error. The module is allowed to
immediately attempt generating a new key pair using fresh entropy,
without needing a full system reboot or module reset.

Detected by Oracle during Jipher testing.

Reviewed-by: Simo Sorce <simo@redhat.com>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
MergeDate: Thu Jun 11 16:03:42 2026
(Merged from https://github.com/openssl/openssl/pull/31359)

diff --git a/crypto/dsa/dsa_key.c b/crypto/dsa/dsa_key.c
index aa69c3eea8fb815f96ebf7ce08a3a2ac4cd3526d..a07f866a218a7c39576be6ac63b6f32b261c4ff8 100644 (file)
--- a/crypto/dsa/dsa_key.c
+++ b/crypto/dsa/dsa_key.c
@@ -198,7 +198,6 @@ static int dsa_keygen(DSA *dsa)
         ok = dsa_keygen_pairwise_test(dsa, cb, cbarg)
             && dsa_keygen_knownanswer_test(dsa, ctx, cb, cbarg);
         if (!ok) {
-            ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
             BN_free(dsa->pub_key);
             BN_clear_free(dsa->priv_key);
             dsa->pub_key = NULL;

diff --git a/crypto/ec/ec_key.c b/crypto/ec/ec_key.c
index 48e08e7de47a46e9adfb0273740e6002699871a7..8deaa3b93068f80570f1e16976a214776a12bf69 100644 (file)
--- a/crypto/ec/ec_key.c
+++ b/crypto/ec/ec_key.c
@@ -366,7 +366,6 @@ static int ec_generate_key(EC_KEY *eckey, int pairwise_test)
 err:
     /* Step (9): If there is an error return an invalid keypair. */
     if (!ok) {
-        ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
         BN_clear(eckey->priv_key);
         if (eckey->pub_key != NULL)
             EC_POINT_set_to_infinity(group, eckey->pub_key);

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 9e053edb7c2021ef0754f0351db65b5a29c1b53a..df44f50a768d623bf8ab75d46e21ef853d9c8479 100644 (file)
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -645,7 +645,6 @@ static int rsa_keygen(OSSL_LIB_CTX *libctx, RSA *rsa, int bits, int primes,
         OSSL_SELF_TEST_get_callback(libctx, &stcb, &stcbarg);
         ok = rsa_keygen_pairwise_test(rsa, stcb, stcbarg);
         if (!ok) {
-            ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
             /* Clear intermediate results */
             BN_clear_free(rsa->d);
             BN_clear_free(rsa->p);

diff --git a/providers/implementations/keymgmt/dh_kmgmt.c b/providers/implementations/keymgmt/dh_kmgmt.c
index b090695eda4033073979cf9e1021f42307d0298a..8901f14e316c1bb3cc28f861eae0ec4eae5db1b6 100644 (file)
--- a/providers/implementations/keymgmt/dh_kmgmt.c
+++ b/providers/implementations/keymgmt/dh_kmgmt.c
@@ -843,10 +843,8 @@ static void *dh_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg)
 #ifdef FIPS_MODULE
         if (!ossl_fips_self_testing()) {
             ret = ossl_dh_check_pairwise(dh, 0);
-            if (ret <= 0) {
-                ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
+            if (ret <= 0)
                 goto end;
-            }
         }
 #endif /* FIPS_MODULE */
     }

diff --git a/providers/implementations/keymgmt/ec_kmgmt.c b/providers/implementations/keymgmt/ec_kmgmt.c
index d203a0c171b8759dee56ba4ed1d603f0ae1208d2..03c7ea352909b613fe8434c52b45db38a16cd301 100644 (file)
--- a/providers/implementations/keymgmt/ec_kmgmt.c
+++ b/providers/implementations/keymgmt/ec_kmgmt.c
@@ -1326,12 +1326,10 @@ static void *ec_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg)
 
         ret = bnctx != NULL && ossl_ec_key_pairwise_check(ec, bnctx);
         BN_CTX_free(bnctx);
-        if (ret <= 0)
-            ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
     }
 #endif /* FIPS_MODULE */
 
-    if (ret)
+    if (ret > 0)
         return ec;
 err:
     /* Something went wrong, throw the key away */

diff --git a/providers/implementations/keymgmt/ecx_kmgmt.c b/providers/implementations/keymgmt/ecx_kmgmt.c
index 70874989345dfa27f26135540f6ad2c1af22c931..dd9c029a4ac5827d3a01a5498198fd59f298224b 100644 (file)
--- a/providers/implementations/keymgmt/ecx_kmgmt.c
+++ b/providers/implementations/keymgmt/ecx_kmgmt.c
@@ -797,7 +797,6 @@ static void *ed25519_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg)
     if (!key || ((gctx->selection & OSSL_KEYMGMT_SELECT_KEYPAIR) == 0))
         return key;
     if (ecd_fips140_pairwise_test(key, ECX_KEY_TYPE_ED25519, 1) != 1) {
-        ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
         ossl_ecx_key_free(key);
         return NULL;
     }
@@ -830,7 +829,6 @@ static void *ed448_gen(void *genctx, OSSL_CALLBACK *osslcb, void *cbarg)
     if (!key || ((gctx->selection & OSSL_KEYMGMT_SELECT_KEYPAIR) == 0))
         return key;
     if (ecd_fips140_pairwise_test(key, ECX_KEY_TYPE_ED448, 1) != 1) {
-        ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
         ossl_ecx_key_free(key);
         return NULL;
     }

diff --git a/providers/implementations/keymgmt/ml_dsa_kmgmt.c b/providers/implementations/keymgmt/ml_dsa_kmgmt.c
index 34710a4a2f711a16e527df848a7439892c12997a..83217066aad74babf44d2b650986771e5ddbbf4e 100644 (file)
--- a/providers/implementations/keymgmt/ml_dsa_kmgmt.c
+++ b/providers/implementations/keymgmt/ml_dsa_kmgmt.c
@@ -304,10 +304,8 @@ static int ml_dsa_import(void *keydata, int selection, const OSSL_PARAM params[]
 #ifdef FIPS_MODULE
     if (res > 0) {
         res = ml_dsa_pairwise_test(key);
-        if (!res) {
+        if (!res)
             ossl_ml_dsa_key_reset(key);
-            ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT_IMPORT);
-        }
     }
 #endif /* FIPS_MODULE */
     return res;
@@ -508,10 +506,8 @@ static void *ml_dsa_gen(void *genctx, int evp_type)
         goto err;
     }
 #ifdef FIPS_MODULE
-    if (!ml_dsa_pairwise_test(key)) {
-        ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
+    if (!ml_dsa_pairwise_test(key))
         goto err;
-    }
 #endif
     return key;
 err:

diff --git a/providers/implementations/keymgmt/ml_kem_kmgmt.c b/providers/implementations/keymgmt/ml_kem_kmgmt.c
index 8477cd9340bc7f3e450b4ce2be81e9d8779fbb9f..f82f3e6f68aeffce00865b8a75c24332a590e12b 100644 (file)
--- a/providers/implementations/keymgmt/ml_kem_kmgmt.c
+++ b/providers/implementations/keymgmt/ml_kem_kmgmt.c
@@ -499,9 +499,6 @@ static int ml_kem_import(void *vkey, int selection, const OSSL_PARAM params[])
     res = ml_kem_key_fromdata(key, params, include_private);
     if (res > 0 && include_private
         && !ml_kem_pairwise_test(key, key->prov_flags)) {
-#ifdef FIPS_MODULE
-        ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT_IMPORT);
-#endif
         ossl_ml_kem_key_reset(key);
         res = 0;
     }
@@ -792,7 +789,6 @@ static void *ml_kem_gen(void *vgctx, OSSL_CALLBACK *osslcb, void *cbarg)
     if (genok) {
 #ifdef FIPS_MODULE
         if (!ml_kem_pairwise_test(key, ML_KEM_KEY_FIXED_PCT)) {
-            ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
             ossl_ml_kem_key_free(key);
             return NULL;
         }

diff --git a/providers/implementations/keymgmt/slh_dsa_kmgmt.c b/providers/implementations/keymgmt/slh_dsa_kmgmt.c
index 5e7be8afd3800b74f8adcbfce81e1f017e80aef9..766953d265b85f0b909814d6c303a4192599f65e 100644 (file)
--- a/providers/implementations/keymgmt/slh_dsa_kmgmt.c
+++ b/providers/implementations/keymgmt/slh_dsa_kmgmt.c
@@ -372,10 +372,8 @@ static void *slh_dsa_gen(void *genctx, const char *alg)
             gctx->entropy, gctx->entropy_len))
         goto err;
 #ifdef FIPS_MODULE
-    if (!slh_dsa_fips140_pairwise_test(key, ctx)) {
-        ossl_set_error_state(OSSL_SELF_TEST_TYPE_PCT);
+    if (!slh_dsa_fips140_pairwise_test(key, ctx))
         goto err;
-    }
 #endif /* FIPS_MODULE */
     ossl_slh_dsa_hash_ctx_free(ctx);
     return key;