signature/rsa_sig.c: Add checks for the EVP_MD_get_size()

Add checks for the EVP_MD_get_size() to avoid integer overflow and then explicitly cast from int to size_t.

Fixes: 6f4b766315 ("PROV: add RSA signature implementation")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23949)

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 76db37dd02c24d3e0dde58acd642fb21905b4835..cc7353bbcaf27ddb16ecb09e58704cce5711c418 100644 (file)
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -114,8 +114,14 @@ typedef struct {
 
 static size_t rsa_get_md_size(const PROV_RSA_CTX *prsactx)
 {
-    if (prsactx->md != NULL)
-        return EVP_MD_get_size(prsactx->md);
+    int md_size;
+
+    if (prsactx->md != NULL) {
+        md_size = EVP_MD_get_size(prsactx->md);
+        if (md_size <= 0)
+            return 0;
+        return md_size;
+    }
     return 0;
 }