x509/x509_set.c: Add the check for the EVP_MD_CTX_get_size()

Add the check for the return value of EVP_MD_CTX_get_size() to avoid invalid negative numbers.

Fixes: 786dd2c22c ("Add support for custom signature parameters")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23956)

diff --git a/crypto/x509/x509_set.c b/crypto/x509/x509_set.c
index 0881be7292b06b6605dbf8df252dfb68fdf63b0b..3b4d53c93434c9b5ac0ae337171e7ad24091a132 100644 (file)
--- a/crypto/x509/x509_set.c
+++ b/crypto/x509/x509_set.c
@@ -212,7 +212,7 @@ int X509_get_signature_info(X509 *x, int *mdnid, int *pknid, int *secbits,
 static int x509_sig_info_init(X509_SIG_INFO *siginf, const X509_ALGOR *alg,
                               const ASN1_STRING *sig, const EVP_PKEY *pubkey)
 {
-    int pknid, mdnid;
+    int pknid, mdnid, md_size;
     const EVP_MD *md;
     const EVP_PKEY_ASN1_METHOD *ameth;
 
@@ -279,7 +279,10 @@ static int x509_sig_info_init(X509_SIG_INFO *siginf, const X509_ALGOR *alg,
             ERR_raise(ERR_LIB_X509, X509_R_ERROR_GETTING_MD_BY_NID);
             return 0;
         }
-        siginf->secbits = EVP_MD_get_size(md) * 4;
+        md_size = EVP_MD_get_size(md);
+        if (md_size <= 0)
+            return 0;
+        siginf->secbits = md_size * 4;
         break;
     }
     switch (mdnid) {