From: Niels Dossche <niels.dossche@ugent.be>
Date: Tue, 19 Aug 2025 20:56:38 +0000 (+0200)
Subject: Make error checks on RSA_public_decrypt() consistent
X-Git-Tag: openssl-3.4.3~65
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=26ea7076f53573cc0fc9544c1785d3b4145111ee;p=thirdparty%2Fopenssl.git

Make error checks on RSA_public_decrypt() consistent

Some are only checking for a value < 0, some for <= 0, some for == 0, etc.
The documentation tells us that -1 is returned on error, so at least the
== 0 ones are wrong. In general, the return values are checked
inconsistently. This patch makes the return value checks consistent to
the form that seems to occur most.

Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28306)

(cherry picked from commit 3e2f54a718f541b02b599bbf5109587189368e4d)
---

diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index 8f89f748e7a..75d807835e0 100644
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -228,7 +228,7 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
                 return -1;
             ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
                                      RSA_X931_PADDING);
-            if (ret < 1)
+            if (ret <= 0)
                 return 0;
             ret--;
             if (rctx->tbuf[ret] != RSA_X931_hash_id(EVP_MD_get_type(rctx->md))) {
@@ -255,7 +255,7 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
     } else {
         ret = RSA_public_decrypt(siglen, sig, rout, rsa, rctx->pad_mode);
     }
-    if (ret < 0)
+    if (ret <= 0)
         return ret;
     *routlen = ret;
     return 1;
@@ -313,7 +313,7 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
             return -1;
         rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
                                    rsa, rctx->pad_mode);
-        if (rslen == 0)
+        if (rslen <= 0)
             return 0;
     }
 
diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index d11b8dc3458..088a6e88c2a 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -952,7 +952,7 @@ static int rsa_verify_recover(void *vprsactx,
                 return 0;
             ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
                                      RSA_X931_PADDING);
-            if (ret < 1) {
+            if (ret <= 0) {
                 ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB);
                 return 0;
             }
@@ -1002,7 +1002,7 @@ static int rsa_verify_recover(void *vprsactx,
     } else {
         ret = RSA_public_decrypt(siglen, sig, rout, prsactx->rsa,
                                  prsactx->pad_mode);
-        if (ret < 0) {
+        if (ret <= 0) {
             ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB);
             return 0;
         }