rsa/rsa_ameth.c: Add the check for the EVP_MD_get_size()

Add the check for the EVP_MD_get_size() to avoid invalid negative numbers.

Fixes: 17c63d1cca ("RSA PSS ASN1 signing method")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23973)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 148d0bbbd153b139da936a0906e0dd652dc8adfd..3a8ab79cabe156206e78a077afe62eeeb49bca80 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -454,15 +454,19 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
     EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
     int saltlen;
     int saltlenMax = -1;
+    int md_size;
 
     if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
         return NULL;
+    md_size = EVP_MD_get_size(sigmd);
+    if (md_size <= 0)
+        return NULL;
     if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
         return NULL;
     if (EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen) <= 0)
         return NULL;
     if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
-        saltlen = EVP_MD_get_size(sigmd);
+        saltlen = md_size;
     } else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
         /* FIPS 186-4 section 5 "The RSA Digital Signature Algorithm",
          * subsection 5.5 "PKCS #1" says: "For RSASSA-PSS […] the length (in
@@ -472,10 +476,10 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
          * Provide a way to use at most the digest length, so that the default
          * does not violate FIPS 186-4. */
         saltlen = RSA_PSS_SALTLEN_MAX;
-        saltlenMax = EVP_MD_get_size(sigmd);
+        saltlenMax = md_size;
     }
     if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) {
-        saltlen = EVP_PKEY_get_size(pk) - EVP_MD_get_size(sigmd) - 2;
+        saltlen = EVP_PKEY_get_size(pk) - md_size - 2;
         if ((EVP_PKEY_get_bits(pk) & 0x7) == 1)
             saltlen--;
         if (saltlen < 0)
@@ -719,7 +723,7 @@ static int rsa_sig_info_set(X509_SIG_INFO *siginf, const X509_ALGOR *sigalg,
                             const ASN1_STRING *sig)
 {
     int rv = 0;
-    int mdnid, saltlen;
+    int mdnid, saltlen, md_size;
     uint32_t flags;
     const EVP_MD *mgf1md = NULL, *md = NULL;
     RSA_PSS_PARAMS *pss;
@@ -732,6 +736,9 @@ static int rsa_sig_info_set(X509_SIG_INFO *siginf, const X509_ALGOR *sigalg,
     pss = ossl_rsa_pss_decode(sigalg);
     if (!ossl_rsa_pss_get_param(pss, &md, &mgf1md, &saltlen))
         goto err;
+    md_size = EVP_MD_get_size(md);
+    if (md_size <= 0)
+        goto err;
     mdnid = EVP_MD_get_type(md);
     /*
      * For TLS need SHA256, SHA384 or SHA512, digest and MGF1 digest must
@@ -739,12 +746,12 @@ static int rsa_sig_info_set(X509_SIG_INFO *siginf, const X509_ALGOR *sigalg,
      */
     if ((mdnid == NID_sha256 || mdnid == NID_sha384 || mdnid == NID_sha512)
             && mdnid == EVP_MD_get_type(mgf1md)
-            && saltlen == EVP_MD_get_size(md))
+            && saltlen == md_size)
         flags = X509_SIG_INFO_TLS;
     else
         flags = 0;
     /* Note: security bits half number of digest bits */
-    secbits = EVP_MD_get_size(md) * 4;
+    secbits = md_size * 4;
     /*
      * SHA1 and MD5 are known to be broken. Reduce security bits so that
      * they're no longer accepted at security level 1. The real values don't