nettle: mark non-compliant RSA-PSS salt length to be not-approved

According to FIPS 186-5 5.4, the salt length must be in the range
between 0 and the hash length inclusive.  While the use of those salt
lengths is still allowed for compatibility, it is reported as
non-approved operation through FIPS service indicator.

Signed-off-by: Daiki Ueno <ueno@gnu.org>

diff --git a/lib/nettle/pk.c b/lib/nettle/pk.c
index c098e2aa452b38b0fa7c1b0ff5739eab6e3b7bfa..7732e90542d42f553f7bd47d1a386590ba9b9e56 100644 (file)
--- a/lib/nettle/pk.c
+++ b/lib/nettle/pk.c
@@ -1316,6 +1316,15 @@ _wrap_nettle_pk_sign(gnutls_pk_algorithm_t algo,
 
                        mpz_init(s);
 
+                       me = hash_to_entry(sign_params->rsa_pss_dig);
+
+                       /* According to FIPS 186-5 5.4, the salt length must be
+                        * in the range between 0 and the hash length inclusive.
+                        */
+                       if (sign_params->salt_size > _gnutls_mac_get_algo_len(me)) {
+                               not_approved = true;
+                       }
+
                        ret =
                            _rsa_pss_sign_digest_tr(sign_params->rsa_pss_dig,
                                                    &pub, &priv,

diff --git a/tests/rsa-rsa-pss.c b/tests/rsa-rsa-pss.c
index 19a175b722cb98a43cf02f2d68150246cca76618..d7799c1961f3c6797af3a86c08f7938a2eddd855 100644 (file)
--- a/tests/rsa-rsa-pss.c
+++ b/tests/rsa-rsa-pss.c
@@ -46,6 +46,8 @@ const gnutls_datum_t raw_data = {
        11
 };
 
+static gnutls_fips140_context_t fips_context;
+
 static void inv_sign_check(unsigned sigalgo,
                           gnutls_privkey_t privkey, int exp_error)
 {
@@ -86,13 +88,16 @@ static void inv_encryption_check(gnutls_pk_algorithm_t algorithm,
 
 static void sign_verify_data(unsigned sigalgo, gnutls_privkey_t privkey,
                             unsigned int sign_flags, unsigned int verify_flags,
-                            int sign_exp_error, int verify_exp_error)
+                            int sign_exp_error, int verify_exp_error,
+                            gnutls_fips140_operation_state_t sign_exp_state)
 {
        int ret;
        gnutls_datum_t signature = { NULL, 0 };
 
+       fips_push_context(fips_context);
        ret = gnutls_privkey_sign_data2(privkey, sigalgo, sign_flags,
                                        &raw_data, &signature);
+       fips_pop_context(fips_context, sign_exp_state);
        if (ret != sign_exp_error)
                fail("gnutls_x509_privkey_sign_data returned unexpected error: %s\n",
                     gnutls_strerror(ret));
@@ -180,11 +185,16 @@ void doit(void)
        if (debug)
                gnutls_global_set_log_level(4711);
 
+       assert(gnutls_fips140_context_init(&fips_context) >= 0);
+
        prepare_keys(&pkey_rsa_pss, &pkey_rsa, GNUTLS_DIG_SHA256, 32);
 
-       sign_verify_data(GNUTLS_SIGN_RSA_PSS_SHA256, pkey_rsa_pss, 0, 0, 0, 0);
-       sign_verify_data(GNUTLS_SIGN_RSA_PSS_RSAE_SHA256, pkey_rsa, 0, 0, 0, 0);
-       sign_verify_data(GNUTLS_SIGN_RSA_PSS_SHA256, pkey_rsa, 0, 0, 0, 0);
+       sign_verify_data(GNUTLS_SIGN_RSA_PSS_SHA256, pkey_rsa_pss, 0, 0, 0, 0,
+                        GNUTLS_FIPS140_OP_APPROVED);
+       sign_verify_data(GNUTLS_SIGN_RSA_PSS_RSAE_SHA256, pkey_rsa, 0, 0, 0, 0,
+                        GNUTLS_FIPS140_OP_APPROVED);
+       sign_verify_data(GNUTLS_SIGN_RSA_PSS_SHA256, pkey_rsa, 0, 0, 0, 0,
+                        GNUTLS_FIPS140_OP_APPROVED);
 
        if (debug)
                success("success signing with RSA-PSS-SHA256\n");
@@ -213,41 +223,65 @@ void doit(void)
        gnutls_privkey_deinit(pkey_rsa_pss);
        gnutls_privkey_deinit(pkey_rsa);
 
+       /* Restrict key to use salt length larger than hash output
+        * length (not approved in FIPS).
+        */
+       prepare_keys(&pkey_rsa_pss, &pkey_rsa, GNUTLS_DIG_SHA256, 33);
+
+       sign_verify_data(GNUTLS_SIGN_RSA_PSS_SHA256, pkey_rsa_pss, 0, 0, 0, 0,
+                        GNUTLS_FIPS140_OP_NOT_APPROVED);
+
+       gnutls_privkey_deinit(pkey_rsa_pss);
+       gnutls_privkey_deinit(pkey_rsa);
+
        /* Use the mismatched salt length with the digest length */
        prepare_keys(&pkey_rsa_pss, &pkey_rsa, GNUTLS_DIG_SHA256, 48);
 
        sign_verify_data(GNUTLS_SIGN_RSA_PSS_RSAE_SHA256, pkey_rsa_pss,
-                        0, 0, 0, 0);
+                        0, 0, 0, 0, GNUTLS_FIPS140_OP_NOT_APPROVED);
        sign_verify_data(GNUTLS_SIGN_RSA_PSS_SHA256, pkey_rsa_pss,
                         GNUTLS_PRIVKEY_FLAG_RSA_PSS_FIXED_SALT_LENGTH,
                         0,
                         GNUTLS_E_CONSTRAINT_ERROR,
-                        0);
+                        0,
+                        /* The error is caught before calling the actual
+                         * signing operation.
+                         */
+                        GNUTLS_FIPS140_OP_INITIAL);
        sign_verify_data(GNUTLS_SIGN_RSA_PSS_SHA256, pkey_rsa_pss,
                         0,
                         GNUTLS_VERIFY_RSA_PSS_FIXED_SALT_LENGTH,
                         0,
-                        GNUTLS_E_PK_SIG_VERIFY_FAILED);
+                        GNUTLS_E_PK_SIG_VERIFY_FAILED,
+                        GNUTLS_FIPS140_OP_NOT_APPROVED);
 
        assert(gnutls_x509_spki_init(&spki)>=0);
        gnutls_x509_spki_set_rsa_pss_params(spki, GNUTLS_DIG_SHA256, 48);
        assert(gnutls_privkey_set_spki(pkey_rsa, spki, 0)>=0);
 
-       sign_verify_data(GNUTLS_SIGN_RSA_PSS_RSAE_SHA256, pkey_rsa, 0, 0, 0, 0);
+       sign_verify_data(GNUTLS_SIGN_RSA_PSS_RSAE_SHA256, pkey_rsa, 0, 0, 0, 0,
+                        GNUTLS_FIPS140_OP_NOT_APPROVED);
        sign_verify_data(GNUTLS_SIGN_RSA_PSS_RSAE_SHA256, pkey_rsa,
                         GNUTLS_PRIVKEY_FLAG_RSA_PSS_FIXED_SALT_LENGTH,
                         0,
                         GNUTLS_E_CONSTRAINT_ERROR,
-                        0);
+                        0,
+                        /* The error is caught before calling the actual
+                         * signing operation.
+                         */
+                        GNUTLS_FIPS140_OP_INITIAL);
        sign_verify_data(GNUTLS_SIGN_RSA_PSS_RSAE_SHA256, pkey_rsa,
                         0,
                         GNUTLS_VERIFY_RSA_PSS_FIXED_SALT_LENGTH,
                         0,
-                        GNUTLS_E_PK_SIG_VERIFY_FAILED);
+                        GNUTLS_E_PK_SIG_VERIFY_FAILED,
+                        GNUTLS_FIPS140_OP_NOT_APPROVED);
 
        gnutls_privkey_deinit(pkey_rsa_pss);
        gnutls_privkey_deinit(pkey_rsa);
        gnutls_x509_spki_deinit(spki);
 
+       gnutls_fips140_context_deinit(fips_context);
+
        gnutls_global_deinit();
 }