From: Tomas Mraz <tomas@openssl.org>
Date: Mon, 5 Aug 2024 13:08:39 +0000 (+0200)
Subject: rsa_pss_compute_saltlen(): Avoid integer overflows and check MD and RSA sizes
X-Git-Tag: openssl-3.1.7~28
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=819afba7a5204f0d7981c9bc5014817c51ec1c51;p=thirdparty%2Fopenssl.git

rsa_pss_compute_saltlen(): Avoid integer overflows and check MD and RSA sizes

Fixes Coverity 1604651

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
(Merged from https://github.com/openssl/openssl/pull/25085)

(cherry picked from commit 217e215e99dd526ad2e6f83601449742d1d03d6a)
---

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 5876158d27e..d90b64128b5 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -208,13 +208,29 @@ static int rsa_pss_compute_saltlen(PROV_RSA_CTX *ctx)
      * Provide a way to use at most the digest length, so that the default does
      * not violate FIPS 186-4. */
     if (saltlen == RSA_PSS_SALTLEN_DIGEST) {
-        saltlen = EVP_MD_get_size(ctx->md);
+        if ((saltlen = EVP_MD_get_size(ctx->md)) <= 0) {
+            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST);
+            return -1;
+        }
     } else if (saltlen == RSA_PSS_SALTLEN_AUTO_DIGEST_MAX) {
         saltlen = RSA_PSS_SALTLEN_MAX;
-        saltlenMax = EVP_MD_get_size(ctx->md);
+        if ((saltlenMax = EVP_MD_get_size(ctx->md)) <= 0) {
+            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST);
+            return -1;
+        }
     }
     if (saltlen == RSA_PSS_SALTLEN_MAX || saltlen == RSA_PSS_SALTLEN_AUTO) {
-        saltlen = RSA_size(ctx->rsa) - EVP_MD_get_size(ctx->md) - 2;
+        int mdsize, rsasize;
+
+        if ((mdsize = EVP_MD_get_size(ctx->md)) <= 0) {
+            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_DIGEST);
+            return -1;
+        }
+        if ((rsasize = RSA_size(ctx->rsa)) <= 2 || rsasize - 2 < mdsize) {
+            ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY);
+            return -1;
+        }
+        saltlen = rsasize - mdsize - 2;
         if ((RSA_bits(ctx->rsa) & 0x7) == 1)
             saltlen--;
         if (saltlenMax >= 0 && saltlen > saltlenMax)