Make error checks on RSA_public_decrypt() consistent

Some are only checking for a value < 0, some for <= 0, some for == 0, etc.
The documentation tells us that -1 is returned on error, so at least the
== 0 ones are wrong. In general, the return values are checked
inconsistently. This patch makes the return value checks consistent to
the form that seems to occur most.

Reviewed-by: Paul Dale <ppzgs1@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28306)

(cherry picked from commit 3e2f54a718f541b02b599bbf5109587189368e4d)

diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index fc3391ead20b56907b732a9876166848b60cf97e..efc311658e66d20b19c80bd231de146700b42fbe 100644 (file)
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -221,7 +221,7 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
                 return -1;
             ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
                                      RSA_X931_PADDING);
-            if (ret < 1)
+            if (ret <= 0)
                 return 0;
             ret--;
             if (rctx->tbuf[ret] != RSA_X931_hash_id(EVP_MD_get_type(rctx->md))) {
@@ -248,7 +248,7 @@ static int pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx,
     } else {
         ret = RSA_public_decrypt(siglen, sig, rout, rsa, rctx->pad_mode);
     }
-    if (ret < 0)
+    if (ret <= 0)
         return ret;
     *routlen = ret;
     return 1;
@@ -300,7 +300,7 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx,
             return -1;
         rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf,
                                    rsa, rctx->pad_mode);
-        if (rslen == 0)
+        if (rslen <= 0)
             return 0;
     }
 

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index f98fb61ce3c352b5bcd28821d09f6bdc985a468c..b954f725a8dc066f4ab4f39aba11e3528a1df5c1 100644 (file)
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -718,7 +718,7 @@ static int rsa_verify_recover(void *vprsactx,
                 return 0;
             ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
                                      RSA_X931_PADDING);
-            if (ret < 1) {
+            if (ret <= 0) {
                 ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB);
                 return 0;
             }
@@ -768,7 +768,7 @@ static int rsa_verify_recover(void *vprsactx,
     } else {
         ret = RSA_public_decrypt(siglen, sig, rout, prsactx->rsa,
                                  prsactx->pad_mode);
-        if (ret < 0) {
+        if (ret <= 0) {
             ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB);
             return 0;
         }