nettle: check RSA modulus size in bits rather than bytes

Previously we checked RSA modulus size clamped to byte unit instead of
bits.  This makes the check stricter by explicitly calculating the
modulus size in bits.

Signed-off-by: Daiki Ueno <ueno@gnu.org>

diff --git a/lib/nettle/pk.c b/lib/nettle/pk.c
index eba246f0b3217f019ae1a8c8d9685a5b4ca85a6b..2d7328e370ce4f22d3109d30158aea4bbc0df2d5 100644 (file)
--- a/lib/nettle/pk.c
+++ b/lib/nettle/pk.c
@@ -1247,20 +1247,20 @@ _wrap_nettle_pk_sign(gnutls_pk_algorithm_t algo,
 
                        _rsa_params_to_privkey(pk_params, &priv);
 
-                       /* RSA key size should be 2048-bit or larger in FIPS
-                        * 140-3.  In addition to this, only SHA-2 is allowed
-                        * for SigGen; it is checked in pk_prepare_hash lib/pk.c
-                        */
-                       if (unlikely(priv.size < 256)) {
-                               not_approved = true;
-                       }
-
                        ret = _rsa_params_to_pubkey(pk_params, &pub);
                        if (ret < 0) {
                                gnutls_assert();
                                goto cleanup;
                        }
 
+                       /* RSA modulus size should be 2048-bit or larger in FIPS
+                        * 140-3.  In addition to this, only SHA-2 is allowed
+                        * for SigGen; it is checked in pk_prepare_hash lib/pk.c
+                        */
+                       if (unlikely(mpz_sizeinbase(pub.n, 2) < 2048)) {
+                               not_approved = true;
+                       }
+
                        mpz_init(s);
 
                        if (_gnutls_get_lib_state() == LIB_STATE_SELFTEST)
@@ -1298,22 +1298,22 @@ _wrap_nettle_pk_sign(gnutls_pk_algorithm_t algo,
 
                        _rsa_params_to_privkey(pk_params, &priv);
 
-                       /* RSA key size should be 2048-bit or larger in FIPS
+                       ret = _rsa_params_to_pubkey(pk_params, &pub);
+                       if (ret < 0) {
+                               gnutls_assert();
+                               goto cleanup;
+                       }
+
+                       /* RSA modulus size should be 2048-bit or larger in FIPS
                         * 140-3.  In addition to this, only SHA-2 is allowed
                         * for SigGen; however, Nettle only support SHA256,
                         * SHA384, and SHA512 for RSA-PSS (see
                         * _rsa_pss_sign_digest_tr in this file for details).
                         */
-                       if (unlikely(priv.size < 256)) {
+                       if (unlikely(mpz_sizeinbase(pub.n, 2) < 2048)) {
                                not_approved = true;
                        }
 
-                       ret = _rsa_params_to_pubkey(pk_params, &pub);
-                       if (ret < 0) {
-                               gnutls_assert();
-                               goto cleanup;
-                       }
-
                        mpz_init(s);
 
                        ret =
@@ -1650,12 +1650,12 @@ _wrap_nettle_pk_verify(gnutls_pk_algorithm_t algo,
                                goto cleanup;
                        }
 
-                       /* RSA key size should be 2048-bit or larger in FIPS
+                       /* RSA modulus size should be 2048-bit or larger in FIPS
                         * 140-3.  In addition to this, only SHA-1 and SHA-2 are
                         * allowed for SigVer; it is checked in
                         * _pkcs1_rsa_verify_sig in lib/pubkey.c
                         */
-                       if (unlikely(pub.size < 256)) {
+                       if (unlikely(mpz_sizeinbase(pub.n, 2) < 2048)) {
                                not_approved = true;
                        }
 
@@ -1701,13 +1701,13 @@ _wrap_nettle_pk_verify(gnutls_pk_algorithm_t algo,
                                goto cleanup;
                        }
 
-                       /* RSA key size should be 2048-bit or larger in FIPS
+                       /* RSA modulus size should be 2048-bit or larger in FIPS
                         * 140-3.  In addition to this, only SHA-1 and SHA-2 are
                         * allowed for SigVer, while Nettle only supports
                         * SHA256, SHA384, and SHA512 for RSA-PSS (see
                         * _rsa_pss_verify_digest in this file for the details).
                         */
-                       if (unlikely(pub.size < 256)) {
+                       if (unlikely(mpz_sizeinbase(pub.n, 2) < 2048)) {
                                not_approved = true;
                        }