_gnutls_find_rsa_pss_salt_size: add a validity check for salt size

That is, in order to reject invalid parameters.

Resolves #402

Signed-off-by: Nikos Mavrogiannopoulos <nmav@redhat.com>

diff --git a/lib/pk.c b/lib/pk.c
index be1d8b6a85cbd5072d54b3d1a228da3d66ccd8f4..3014396bc0e628a7c2a9a66e6257a3615f20e214 100644 (file)
--- a/lib/pk.c
+++ b/lib/pk.c
@@ -382,19 +382,29 @@ void gnutls_pk_params_clear(gnutls_pk_params_st * p)
        }
 }
 
-unsigned
+int
 _gnutls_find_rsa_pss_salt_size(unsigned bits, const mac_entry_st *me,
                               unsigned salt_size)
 {
-       unsigned max_salt_size, digest_size;
+       unsigned digest_size;
+       int max_salt_size;
+       unsigned key_size;
 
        digest_size = _gnutls_hash_get_algo_len(me);
-       max_salt_size = (bits + 7) / 8 - digest_size - 2;
+       key_size = (bits + 7) / 8;
+
+       if (key_size == 0) {
+               return gnutls_assert_val(GNUTLS_E_PK_INVALID_PUBKEY);
+       } else {
+               max_salt_size = key_size - digest_size - 2;
+               if (max_salt_size < 0)
+                       return gnutls_assert_val(GNUTLS_E_CONSTRAINT_ERROR);
+       }
 
        if (salt_size < digest_size)
                salt_size = digest_size;
 
-       if (salt_size > max_salt_size)
+       if (salt_size > (unsigned)max_salt_size)
                salt_size = max_salt_size;
 
        return salt_size;

diff --git a/lib/pk.h b/lib/pk.h
index 7ff76d12df0c3fa692e09ba8e062c85ac9c9a6bb..a6eb043333cb57df25c4def4fb01fb296168a200 100644 (file)
--- a/lib/pk.h
+++ b/lib/pk.h
@@ -104,7 +104,7 @@ int pk_hash_data(gnutls_pk_algorithm_t pk, const mac_entry_st * hash,
                 gnutls_pk_params_st * params, const gnutls_datum_t * data,
                 gnutls_datum_t * digest);
 
-unsigned _gnutls_find_rsa_pss_salt_size(unsigned bits, const mac_entry_st *me,
-                                       unsigned salt_size);
+int _gnutls_find_rsa_pss_salt_size(unsigned bits, const mac_entry_st *me,
+                                  unsigned salt_size);
 
 #endif                         /* GNUTLS_PK_H */

diff --git a/lib/privkey.c b/lib/privkey.c
index 6c1a52ee30abd24ee7bbff2f1a7f1bd64b0006a8..63cc7fcbd3bb22028d280946b1197bedf37209b5 100644 (file)
--- a/lib/privkey.c
+++ b/lib/privkey.c
@@ -333,6 +333,7 @@ _gnutls_privkey_update_spki_params(gnutls_privkey_t key,
 
        if (pk == GNUTLS_PK_RSA_PSS) {
                const mac_entry_st *me;
+               int ret;
 
                me = hash_to_entry(dig);
                if (unlikely(me == NULL))
@@ -350,8 +351,12 @@ _gnutls_privkey_update_spki_params(gnutls_privkey_t key,
 
                if (flags & GNUTLS_PRIVKEY_FLAG_REPRODUCIBLE)
                        params->salt_size = 0;
-               else
-                       params->salt_size = _gnutls_find_rsa_pss_salt_size(bits, me, salt_size);
+               else {
+                       ret = _gnutls_find_rsa_pss_salt_size(bits, me, salt_size);
+                       if (ret < 0)
+                               return gnutls_assert_val(ret);
+                       params->salt_size = ret;
+               }
                params->rsa_pss_dig = dig;
        }
 

diff --git a/lib/pubkey.c b/lib/pubkey.c
index a53122f68ae5c4cd7f2f9c37bafdc4728764c0a9..466e2dee63e438ad7dbc2b050bad6d39ca2c3560 100644 (file)
--- a/lib/pubkey.c
+++ b/lib/pubkey.c
@@ -1527,7 +1527,7 @@ int fixup_spki_params(const gnutls_pk_params_st *key_params, const gnutls_sign_e
        }
 
        if (params->pk == GNUTLS_PK_RSA_PSS) {
-
+               int ret;
                if (!GNUTLS_PK_IS_RSA(key_params->algo))
                        return gnutls_assert_val(GNUTLS_E_CONSTRAINT_ERROR);
 
@@ -1537,7 +1537,11 @@ int fixup_spki_params(const gnutls_pk_params_st *key_params, const gnutls_sign_e
                if (key_params->algo == GNUTLS_PK_RSA || params->rsa_pss_dig == 0) {
                        bits = pubkey_to_bits(key_params);
                        params->rsa_pss_dig = se->hash;
-                       params->salt_size = _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+                       ret = _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+                       if (ret < 0)
+                               return gnutls_assert_val(ret);
+
+                       params->salt_size = ret;
                }
 
                if (params->rsa_pss_dig != se->hash)

diff --git a/lib/x509/crq.c b/lib/x509/crq.c
index ac58529f6f9a64bd8e719dec4ebb46e6769738c3..417d63040519ac9cb8d08231cacc8a0fddd2129a 100644 (file)
--- a/lib/x509/crq.c
+++ b/lib/x509/crq.c
@@ -3052,9 +3052,14 @@ gnutls_x509_crq_set_spki(gnutls_x509_crq_t crq,
 
                /* If salt size is zero, find the optimal salt size. */
                if (spki->salt_size == 0) {
-                       tpki.salt_size =
+                       ret =
                            _gnutls_find_rsa_pss_salt_size(bits, me,
                                                           spki->salt_size);
+                       if (ret < 0) {
+                               gnutls_assert();
+                               goto cleanup;
+                       }
+                       tpki.salt_size = ret;
                } else
                        tpki.salt_size = spki->salt_size;
        } else if (crq_pk == GNUTLS_PK_RSA_PSS) {

diff --git a/lib/x509/privkey.c b/lib/x509/privkey.c
index badeb945dfce88293db1fbce15b99f1eb3a6de3c..d4be99ef0566ce0b23fe8799b88b3f031045c6dd 100644 (file)
--- a/lib/x509/privkey.c
+++ b/lib/x509/privkey.c
@@ -1682,8 +1682,13 @@ gnutls_x509_privkey_generate2(gnutls_x509_privkey_t key,
                        goto cleanup;
                }
 
-               key->params.spki.salt_size =
-                   _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+               ret = _gnutls_find_rsa_pss_salt_size(bits, me, 0);
+               if (ret < 0) {
+                       gnutls_assert();
+                       goto cleanup;
+               }
+
+               key->params.spki.salt_size = ret;
        }
 
        ret = _gnutls_pk_generate_keys(algo, bits, &key->params, 0);

diff --git a/lib/x509/x509_write.c b/lib/x509/x509_write.c
index 6d5ed6a52e67a4570b4c710a4083378042843eb5..db90dab9c4222783dcfc030ad81633b79c888065 100644 (file)
--- a/lib/x509/x509_write.c
+++ b/lib/x509/x509_write.c
@@ -2080,9 +2080,13 @@ gnutls_x509_crt_set_spki(gnutls_x509_crt_t crt,
 
                /* If salt size is zero, find the optimal salt size. */
                if (spki->salt_size == 0) {
-                       tpki.salt_size =
-                           _gnutls_find_rsa_pss_salt_size(bits, me,
+                       ret = _gnutls_find_rsa_pss_salt_size(bits, me,
                                                           spki->salt_size);
+                       if (ret < 0) {
+                               gnutls_assert();
+                               goto cleanup;
+                       }
+                       tpki.salt_size = ret;
                } else
                        tpki.salt_size = spki->salt_size;
        } else if (crt_pk == GNUTLS_PK_RSA_PSS) {