cryptoapi: Avoid conversion warnings

Due to the differences in the types of APIs
between xkey provider and Windows cryptoapi
we can't avoid the casts. And they should be
safe generally since the involved sizes should
be small compared to the maximum values. So
just add asserts and explicit cast to avoid
the warnings.

Change-Id: I789022af7c4977c4dff4f7671f491fe5836828fa
Signed-off-by: Frank Lichtenheld <frank@lichtenheld.com>
Acked-by: Selva Nair <selva.nair@gmail.com>
Gerrit URL: https://gerrit.openvpn.net/c/openvpn/+/1464
Message-Id: <20260116135729.40545-1-frank@lichtenheld.com>
URL: https://www.mail-archive.com/openvpn-devel@lists.sourceforge.net/msg35304.html
Signed-off-by: Gert Doering <gert@greenie.muc.de>

diff --git a/src/openvpn/cryptoapi.c b/src/openvpn/cryptoapi.c
index b18b9d4206cf0e2026f08e32b41548b63308524e..49f5bbb698910fede0aa66d473253784417e68b0 100644 (file)
--- a/src/openvpn/cryptoapi.c
+++ b/src/openvpn/cryptoapi.c
@@ -61,7 +61,7 @@ SSL_CTX_use_CryptoAPI_certificate(SSL_CTX *ssl_ctx, const char *cert_prop)
     return 0;
 }
 
-#else /* HAVE_XKEY_PROVIDER */
+#else  /* HAVE_XKEY_PROVIDER */
 
 static XKEY_EXTERNAL_SIGN_fn xkey_cng_sign;
 
@@ -341,21 +341,18 @@ out:
     return rv;
 }
 
-#if defined(__GNUC__) || defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wconversion"
-#endif
-
 /** Sign hash in tbs using EC key in cd and NCryptSignHash */
 static int
 xkey_cng_ec_sign(CAPI_DATA *cd, unsigned char *sig, size_t *siglen, const unsigned char *tbs,
                  size_t tbslen)
 {
-    DWORD len = *siglen;
+    ASSERT(*siglen <= UINT_MAX);
+    ASSERT(tbslen <= UINT_MAX);
+    DWORD len = (DWORD)*siglen;
 
     msg(D_LOW, "Signing using NCryptSignHash with EC key");
 
-    DWORD status = NCryptSignHash(cd->crypt_prov, NULL, (BYTE *)tbs, tbslen, sig, len, &len, 0);
+    DWORD status = NCryptSignHash(cd->crypt_prov, NULL, (BYTE *)tbs, (DWORD)tbslen, sig, len, &len, 0);
 
     if (status != ERROR_SUCCESS)
     {
@@ -383,7 +380,9 @@ xkey_cng_rsa_sign(CAPI_DATA *cd, unsigned char *sig, size_t *siglen, const unsig
 
     ASSERT(cd);
     ASSERT(sig);
+    ASSERT(*siglen <= UINT_MAX);
     ASSERT(tbs);
+    ASSERT(tbslen <= INT_MAX);
 
     DWORD status = ERROR_SUCCESS;
     DWORD len = 0;
@@ -406,10 +405,10 @@ xkey_cng_rsa_sign(CAPI_DATA *cd, unsigned char *sig, size_t *siglen, const unsig
     }
     else if (!strcmp(sigalg.padmode, "pss"))
     {
-        int saltlen = tbslen; /* digest size by default */
+        int saltlen = (int)tbslen; /* digest size by default */
         if (!strcmp(sigalg.saltlen, "max"))
         {
-            saltlen = xkey_max_saltlen(EVP_PKEY_bits(cd->pubkey), tbslen);
+            saltlen = xkey_max_saltlen(EVP_PKEY_bits(cd->pubkey), saltlen);
             if (saltlen < 0)
             {
                 msg(M_NONFATAL, "Error in cryptoapicert: invalid salt length (%d)", saltlen);
@@ -420,8 +419,8 @@ xkey_cng_rsa_sign(CAPI_DATA *cd, unsigned char *sig, size_t *siglen, const unsig
         msg(D_LOW, "Signing using NCryptSignHash with PSS padding: hashalg <%s>, saltlen <%d>",
             sigalg.mdname, saltlen);
 
-        BCRYPT_PSS_PADDING_INFO padinfo = { hashalg,
-                                            (DWORD)saltlen }; /* cast is safe as saltlen >= 0 */
+        /* cast is safe as saltlen >= 0 */
+        BCRYPT_PSS_PADDING_INFO padinfo = { hashalg, (DWORD)saltlen };
         status = NCryptSignHash(cd->crypt_prov, &padinfo, (BYTE *)tbs, (DWORD)tbslen, sig,
                                 (DWORD)*siglen, &len, BCRYPT_PAD_PSS);
     }
@@ -442,10 +441,6 @@ xkey_cng_rsa_sign(CAPI_DATA *cd, unsigned char *sig, size_t *siglen, const unsig
     return (*siglen > 0);
 }
 
-#if defined(__GNUC__) || defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
 /** Dispatch sign op to xkey_cng_<rsa/ec>_sign */
 static int
 xkey_cng_sign(void *handle, unsigned char *sig, size_t *siglen, const unsigned char *tbs,