RSA: Be less strict on PSS parameters when exporting to provider

We have a key in test/recipes/30-test_evp_data/evppkey.txt with bad
PSS parameters (RSA-PSS-BAD), which is supposed to trigger signature
computation faults.  However, if this key needs to be exported to the
RSA provider implementation, the result would be an earlier error,
giving the computation that's supposed to be checked n chance to even
be reached.

Either way, the legacy to provider export is no place to validate the
values of the key.

We also ensure that the provider implementation can handle and detect
signed (negative) saltlen values.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12583)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index f5911ad233677391430c5abc16ca5edd12666924..749cd8764be40f8f2be8e77fcfb021f3a98a7f0d 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -1218,10 +1218,11 @@ static int rsa_int_export_to(const EVP_PKEY *from, int rsa_type,
 
     if (rsa->pss != NULL) {
         const EVP_MD *md = NULL, *mgf1md = NULL;
-        int md_nid, mgf1md_nid, saltlen;
+        int md_nid, mgf1md_nid, saltlen, trailerfield;
         RSA_PSS_PARAMS_30 pss_params;
 
-        if (!rsa_pss_get_param(rsa->pss, &md, &mgf1md, &saltlen))
+        if (!rsa_pss_get_param_unverified(rsa->pss, &md, &mgf1md,
+                                          &saltlen, &trailerfield))
             goto err;
         md_nid = EVP_MD_type(md);
         mgf1md_nid = EVP_MD_type(mgf1md);

diff --git a/include/crypto/rsa.h b/include/crypto/rsa.h
index 9469ec9233a735886dfaadd792421daeadf1a11a..97cbfa1d7ede787c32239d16730b6049b0304acb 100644 (file)
--- a/include/crypto/rsa.h
+++ b/include/crypto/rsa.h
@@ -19,8 +19,8 @@ typedef struct rsa_pss_params_30_st {
         int algorithm_nid;       /* Currently always NID_mgf1 */
         int hash_algorithm_nid;
     } mask_gen;
-    unsigned int salt_len;
-    unsigned int trailer_field;
+    int salt_len;
+    int trailer_field;
 } RSA_PSS_PARAMS_30;
 
 RSA_PSS_PARAMS_30 *rsa_get0_pss_params_30(RSA *r);

diff --git a/providers/implementations/signature/rsa.c b/providers/implementations/signature/rsa.c
index 6de10d1f5303f5d1a99657feb7decb482cab1c8c..491c72d990bf9fcfe817eddd5231ad2038c2483f 100644 (file)
--- a/providers/implementations/signature/rsa.c
+++ b/providers/implementations/signature/rsa.c
@@ -176,16 +176,16 @@ static int rsa_check_padding(int mdnid, int padding)
     return 1;
 }
 
-static int rsa_check_parameters(EVP_MD *md, PROV_RSA_CTX *prsactx)
+static int rsa_check_parameters(PROV_RSA_CTX *prsactx)
 {
     if (prsactx->pad_mode == RSA_PKCS1_PSS_PADDING) {
         int max_saltlen;
 
         /* See if minimum salt length exceeds maximum possible */
-        max_saltlen = RSA_size(prsactx->rsa) - EVP_MD_size(md);
+        max_saltlen = RSA_size(prsactx->rsa) - EVP_MD_size(prsactx->md);
         if ((RSA_bits(prsactx->rsa) & 0x7) == 1)
             max_saltlen--;
-        if (prsactx->min_saltlen > max_saltlen) {
+        if (prsactx->min_saltlen < 0 || prsactx->min_saltlen > max_saltlen) {
             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH);
             return 0;
         }
@@ -230,7 +230,6 @@ static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
         if (md == NULL
             || md_nid == NID_undef
             || !rsa_check_padding(md_nid, ctx->pad_mode)
-            || !rsa_check_parameters(md, ctx)
             || mdname_len >= sizeof(ctx->mdname)) {
             if (md == NULL)
                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
@@ -365,7 +364,8 @@ static int rsa_signature_init(void *vprsactx, void *vrsa, int operation)
                 prsactx->saltlen = min_saltlen;
 
                 return rsa_setup_md(prsactx, mdname, prsactx->propq)
-                    && rsa_setup_mgf1_md(prsactx, mgf1mdname, prsactx->propq);
+                    && rsa_setup_mgf1_md(prsactx, mgf1mdname, prsactx->propq)
+                    && rsa_check_parameters(prsactx);
             }
         }
 
@@ -1151,7 +1151,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
         }
 
         if (rsa_pss_restricted(prsactx)) {
-            switch (prsactx->saltlen) {
+            switch (saltlen) {
             case RSA_PSS_SALTLEN_AUTO:
                 if (prsactx->operation == EVP_PKEY_OP_VERIFY) {
                     ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PSS_SALTLEN);
@@ -1168,7 +1168,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
                                    EVP_MD_size(prsactx->md));
                     return 0;
                 }
-                /* FALLTHRU */
+                break;
             default:
                 if (saltlen >= 0 && saltlen < prsactx->min_saltlen) {
                     ERR_raise_data(ERR_LIB_PROV,