Allow small RSA exponents in the default provider

Fixes #16255

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16285)

diff --git a/crypto/rsa/rsa_sp800_56b_check.c b/crypto/rsa/rsa_sp800_56b_check.c
index 9b827d287252d8e05b646dba7e4e8bd4e03d14af..fc8f19b48770b1084b7a6e202d1fc5cc9bc3b53a 100644 (file)
--- a/crypto/rsa/rsa_sp800_56b_check.c
+++ b/crypto/rsa/rsa_sp800_56b_check.c
@@ -218,30 +218,21 @@ int ossl_rsa_check_private_exponent(const RSA *rsa, int nbits, BN_CTX *ctx)
     return ret;
 }
 
-#ifndef FIPS_MODULE
-static int bn_is_three(const BIGNUM *bn)
-{
-    BIGNUM *num = BN_dup(bn);
-    int ret = (num != NULL && BN_sub_word(num, 3) && BN_is_zero(num));
-
-    BN_free(num);
-    return ret;
-}
-#endif /* FIPS_MODULE */
-
-/* Check exponent is odd, and has a bitlen ranging from [17..256] */
+/*
+ * Check exponent is odd.
+ * For FIPS also check the bit length is in the range [17..256]
+ */
 int ossl_rsa_check_public_exponent(const BIGNUM *e)
 {
+#ifdef FIPS_MODULE
     int bitlen;
 
-    /* For legacy purposes RSA_3 is allowed in non fips mode */
-#ifndef FIPS_MODULE
-    if (bn_is_three(e))
-        return 1;
-#endif /* FIPS_MODULE */
-
     bitlen = BN_num_bits(e);
     return (BN_is_odd(e) && bitlen > 16 && bitlen < 257);
+#else
+    /* Allow small exponents larger than 1 for legacy purposes */
+    return BN_is_odd(e) && BN_cmp(e, BN_value_one()) > 0;
+#endif /* FIPS_MODULE */
 }
 
 /*

diff --git a/test/rsa_sp800_56b_test.c b/test/rsa_sp800_56b_test.c
index 033983d58e2db5e988142e17df397a1ae3a2091c..f5df0e495502ceb5eeb0bafb9341e964913c792b 100644 (file)
--- a/test/rsa_sp800_56b_test.c
+++ b/test/rsa_sp800_56b_test.c
@@ -104,26 +104,29 @@ static BIGNUM *bn_load_new(const unsigned char *data, int sz)
     return ret;
 }
 
+/* Check that small rsa exponents are allowed in non FIPS mode */
 static int test_check_public_exponent(void)
 {
     int ret = 0;
     BIGNUM *e = NULL;
 
     ret = TEST_ptr(e = BN_new())
-          /* e is too small */
-          && TEST_true(BN_set_word(e, 65535))
+          /* e is too small will fail */
+          && TEST_true(BN_set_word(e, 1))
           && TEST_false(ossl_rsa_check_public_exponent(e))
           /* e is even will fail */
           && TEST_true(BN_set_word(e, 65536))
           && TEST_false(ossl_rsa_check_public_exponent(e))
           /* e is ok */
+          && TEST_true(BN_set_word(e, 3))
+          && TEST_true(ossl_rsa_check_public_exponent(e))
+          && TEST_true(BN_set_word(e, 17))
+          && TEST_true(ossl_rsa_check_public_exponent(e))
           && TEST_true(BN_set_word(e, 65537))
           && TEST_true(ossl_rsa_check_public_exponent(e))
-          /* e = 2^256 is too big */
+          /* e = 2^256 + 1 is ok */
           && TEST_true(BN_lshift(e, BN_value_one(), 256))
-          && TEST_false(ossl_rsa_check_public_exponent(e))
-          /* e = 2^256-1 is odd and in range */
-          && TEST_true(BN_sub(e, e, BN_value_one()))
+          && TEST_true(BN_add(e, e, BN_value_one()))
           && TEST_true(ossl_rsa_check_public_exponent(e));
     BN_free(e);
     return ret;