ossl_rsa_sp800_56b_check_public: Be more lenient with small keys

Fixes #13995

For small keys the MR test on the modulus can return
BN_PRIMETEST_COMPOSITE_WITH_FACTOR status although the modulus
is correct.

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14389)

diff --git a/crypto/rsa/rsa_sp800_56b_check.c b/crypto/rsa/rsa_sp800_56b_check.c
index c2066236f9fc50404ae992a917ac88253b191781..3bccbe4373c4f975630bd65c2295101f8aca0c1e 100644 (file)
--- a/crypto/rsa/rsa_sp800_56b_check.c
+++ b/crypto/rsa/rsa_sp800_56b_check.c
@@ -290,21 +290,19 @@ int ossl_rsa_get_lcm(BN_CTX *ctx, const BIGNUM *p, const BIGNUM *q,
 int ossl_rsa_sp800_56b_check_public(const RSA *rsa)
 {
     int ret = 0, status;
-#ifdef FIPS_MODULE
     int nbits;
-#endif
     BN_CTX *ctx = NULL;
     BIGNUM *gcd = NULL;
 
     if (rsa->n == NULL || rsa->e == NULL)
         return 0;
 
+    nbits = BN_num_bits(rsa->n);
 #ifdef FIPS_MODULE
     /*
      * (Step a): modulus must be 2048 or 3072 (caveat from SP800-56Br1)
      * NOTE: changed to allow keys >= 2048
      */
-    nbits = BN_num_bits(rsa->n);
     if (!ossl_rsa_sp800_56b_validate_strength(nbits, -1)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_KEY_LENGTH);
         return 0;
@@ -336,7 +334,13 @@ int ossl_rsa_sp800_56b_check_public(const RSA *rsa)
     }
 
     ret = ossl_bn_miller_rabin_is_prime(rsa->n, 0, ctx, NULL, 1, &status);
+#ifdef FIPS_MODULE
     if (ret != 1 || status != BN_PRIMETEST_COMPOSITE_NOT_POWER_OF_PRIME) {
+#else
+    if (ret != 1 || (status != BN_PRIMETEST_COMPOSITE_NOT_POWER_OF_PRIME
+                     && (nbits >= RSA_MIN_MODULUS_BITS
+                         || status != BN_PRIMETEST_COMPOSITE_WITH_FACTOR))) {
+#endif
         ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_MODULUS);
         ret = 0;
         goto err;