ssl/t1_lib.c: Add checks for the EVP_MD_get_size()

Add checks for the EVP_MD_get_size() to avoid unexpected negative numbers.

Fixes: b362ccab5c ("Security framework.")
Fixes: 0fe3db251a ("Use size of server key when selecting signature algorithm.")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
(Merged from https://github.com/openssl/openssl/pull/23943)

diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 26a838f9f2b4004c4b8ea7b3c5bd5aad05f137be..1b6802e7fcf34b023bc731e7897d97a4a23bd440 100644 (file)
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -1649,6 +1649,8 @@ static int rsa_pss_check_min_key_size(SSL_CTX *ctx, const EVP_PKEY *pkey,
         return 0;
     if (!tls1_lookup_md(ctx, lu, &md) || md == NULL)
         return 0;
+    if (EVP_MD_get_size(md) <= 0)
+        return 0;
     if (EVP_PKEY_get_size(pkey) < RSA_PSS_MINIMUM_KEY_SIZE(md))
         return 0;
     return 1;
@@ -1831,6 +1833,8 @@ static int sigalg_security_bits(SSL_CTX *ctx, const SIGALG_LOOKUP *lu)
 
         /* Security bits: half digest bits */
         secbits = EVP_MD_get_size(md) * 4;
+        if (secbits <= 0)
+            return 0;
         /*
          * SHA1 and MD5 are known to be broken. Reduce security bits so that
          * they're no longer accepted at security level 1. The real values don't