Fix size_t/int mismatch in cms_ec.c and rsa_sig.c

Fixes #20435

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/20457)

diff --git a/crypto/cms/cms_dh.c b/crypto/cms/cms_dh.c
index ea8b24528f8a3f50851defb0d2d3b88bafa10937..c1b763e98ec038066227ea0b6392c76b832f3e89 100644 (file)
--- a/crypto/cms/cms_dh.c
+++ b/crypto/cms/cms_dh.c
@@ -309,7 +309,7 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
      */
     penc = NULL;
     penclen = i2d_X509_ALGOR(wrap_alg, &penc);
-    if (penc == NULL || penclen == 0)
+    if (penclen <= 0)
         goto err;
     wrap_str = ASN1_STRING_new();
     if (wrap_str == NULL)

diff --git a/crypto/cms/cms_ec.c b/crypto/cms/cms_ec.c
index 896eda61da7062e836be212d915dede97cf91124..2e4f19552f507892e6f8e44d43fb146ee709c744 100644 (file)
--- a/crypto/cms/cms_ec.c
+++ b/crypto/cms/cms_ec.c
@@ -8,6 +8,7 @@
  */
 
 #include <assert.h>
+#include <limits.h>
 #include <openssl/cms.h>
 #include <openssl/err.h>
 #include <openssl/decoder.h>
@@ -258,7 +259,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
     ASN1_STRING *wrap_str;
     ASN1_OCTET_STRING *ukm;
     unsigned char *penc = NULL;
-    size_t penclen;
+    int penclen;
     int rv = 0;
     int ecdh_nid, kdf_type, kdf_nid, wrap_nid;
     const EVP_MD *kdf_md;
@@ -275,9 +276,12 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
     /* Is everything uninitialised? */
     if (aoid == OBJ_nid2obj(NID_undef)) {
         /* Set the key */
+        size_t enckeylen;
 
-        penclen = EVP_PKEY_get1_encoded_public_key(pkey, &penc);
-        ASN1_STRING_set0(pubkey, penc, penclen);
+        enckeylen = EVP_PKEY_get1_encoded_public_key(pkey, &penc);
+        if (enckeylen > INT_MAX || enckeylen == 0)
+            goto err;
+        ASN1_STRING_set0(pubkey, penc, (int)enckeylen);
         ossl_asn1_string_set_bits_left(pubkey, 0);
 
         penc = NULL;
@@ -358,7 +362,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
      * of another AlgorithmIdentifier.
      */
     penclen = i2d_X509_ALGOR(wrap_alg, &penc);
-    if (penc == NULL || penclen == 0)
+    if (penclen <= 0)
         goto err;
     wrap_str = ASN1_STRING_new();
     if (wrap_str == NULL)

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index e0faf1c1ad3c511c787d8f4854cb5fd4a92e6671..4ebb6517d6471c3bd253742100c7589b6c64e1ed 100644 (file)
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -834,14 +834,17 @@ static int rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen,
             return 0;
         }
     } else {
+        int ret;
+
         if (!setup_tbuf(prsactx))
             return 0;
-        rslen = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
-                                   prsactx->pad_mode);
-        if (rslen <= 0) {
+        ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
+                                 prsactx->pad_mode);
+        if (ret <= 0) {
             ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB);
             return 0;
         }
+        rslen = (size_t)ret;
     }
 
     if ((rslen != tbslen) || memcmp(tbs, prsactx->tbuf, rslen))