The return value of RSA_*_{en,de}crypt() is signed

The functions RSA_(public|private)_(en|de)crypt() return a signed
result, in particular `-1` may be returned on error, so the caller
MUST treat the value as signed.

Reviewed-by: Saša Nedvědický <sashan@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/29323)

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index ecc0b1b829b4efdca1aa6876db615141f9344091..9e053edb7c2021ef0754f0351db65b5a29c1b53a 100644 (file)
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -693,12 +693,12 @@ static int rsa_keygen(OSSL_LIB_CTX *libctx, RSA *rsa, int bits, int primes,
 static int rsa_keygen_pairwise_test(RSA *rsa, OSSL_CALLBACK *cb, void *cbarg)
 {
     int ret = 0;
-    unsigned int plaintxt_len;
     unsigned char *plaintxt = NULL;
-    unsigned int ciphertxt_len;
     unsigned char *ciphertxt = NULL;
     unsigned char *decoded = NULL;
-    unsigned int decoded_len;
+    int plaintxt_len;
+    int ciphertxt_len;
+    int decoded_len;
     int padding = RSA_NO_PADDING;
     OSSL_SELF_TEST *st = NULL;