From: Frank Lichtenheld <frank@lichtenheld.com>
Date: Wed, 27 Mar 2024 16:26:21 +0000 (+0100)
Subject: crypto_backend: fix type of enc parameter
X-Git-Tag: v2.6.14~3
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a421e94344aa2a2f15575eb5c6d57af48d669599;p=thirdparty%2Fopenvpn.git

crypto_backend: fix type of enc parameter

We had parts of a abstraction, but it wasn't consistent.
GCC 13 now complains about the type mismatch with mbedtls now:

crypto_mbedtls.c:568:1: error:
conflicting types for ‘cipher_ctx_init’ due to enum/integer mismatch;
have ‘void(mbedtls_cipher_context_t *, const uint8_t *, const char *, const mbedtls_operation_t)’
[...] [-Werror=enum-int-mismatch]
crypto_backend.h:341:6: note:
previous declaration of ‘cipher_ctx_init’ with type
‘void(cipher_ctx_t *, const uint8_t *, const char *, int)’ [...]

Previous compiler versions did not complain.

v2:
 - clean solution instead of quick solution. Fix the actual API
   definition

Change-Id: If0dcdde30879fd6185efb2ad31399c1629c04d22
Signed-off-by: Frank Lichtenheld <frank@lichtenheld.com>
Acked-by: Arne Schwabe <arne-openvpn@rfc2549.org>
Message-Id: <20240327162621.1792414-1-frank@lichtenheld.com>
URL: https://www.mail-archive.com/openvpn-devel@lists.sourceforge.net/msg28498.html
Signed-off-by: Gert Doering <gert@greenie.muc.de>
(cherry picked from commit 4d907bf46a470ccbd2940b9ecb64d6502d9d86bf)
---

diff --git a/src/openvpn/crypto_backend.h b/src/openvpn/crypto_backend.h
index 46b375259..391ce0e6a 100644
--- a/src/openvpn/crypto_backend.h
+++ b/src/openvpn/crypto_backend.h
@@ -347,10 +347,10 @@ void cipher_ctx_free(cipher_ctx_t *ctx);
  * @param key           Buffer containing the key to use
  * @param ciphername    Ciphername of the cipher to use
  * @param enc           Whether to encrypt or decrypt (either
- *                      \c MBEDTLS_OP_ENCRYPT or \c MBEDTLS_OP_DECRYPT).
+ *                      \c OPENVPN_OP_ENCRYPT or \c OPENVPN_OP_DECRYPT).
  */
 void cipher_ctx_init(cipher_ctx_t *ctx, const uint8_t *key,
-                     const char *cipername, int enc);
+                     const char *cipername, crypto_operation_t enc);
 
 /**
  * Returns the size of the IV used by the cipher, in bytes, or 0 if no IV is
diff --git a/src/openvpn/crypto_mbedtls.c b/src/openvpn/crypto_mbedtls.c
index d098c7c1f..4a3e693e0 100644
--- a/src/openvpn/crypto_mbedtls.c
+++ b/src/openvpn/crypto_mbedtls.c
@@ -566,7 +566,7 @@ cipher_ctx_free(mbedtls_cipher_context_t *ctx)
 
 void
 cipher_ctx_init(mbedtls_cipher_context_t *ctx, const uint8_t *key,
-                const char *ciphername, const mbedtls_operation_t operation)
+                const char *ciphername, crypto_operation_t enc)
 {
     ASSERT(NULL != ciphername && NULL != ctx);
     CLEAR(*ctx);
@@ -580,7 +580,7 @@ cipher_ctx_init(mbedtls_cipher_context_t *ctx, const uint8_t *key,
         msg(M_FATAL, "mbed TLS cipher context init #1");
     }
 
-    if (!mbed_ok(mbedtls_cipher_setkey(ctx, key, (int)key_bitlen, operation)))
+    if (!mbed_ok(mbedtls_cipher_setkey(ctx, key, (int)key_bitlen, enc)))
     {
         msg(M_FATAL, "mbed TLS cipher set key");
     }
diff --git a/src/openvpn/crypto_mbedtls.h b/src/openvpn/crypto_mbedtls.h
index 46f76e251..48d1e207f 100644
--- a/src/openvpn/crypto_mbedtls.h
+++ b/src/openvpn/crypto_mbedtls.h
@@ -63,6 +63,8 @@ typedef void provider_t;
 /** Cipher is in GCM mode */
 #define OPENVPN_MODE_GCM        MBEDTLS_MODE_GCM
 
+typedef mbedtls_operation_t crypto_operation_t;
+
 /** Cipher should encrypt */
 #define OPENVPN_OP_ENCRYPT      MBEDTLS_ENCRYPT
 
diff --git a/src/openvpn/crypto_openssl.c b/src/openvpn/crypto_openssl.c
index fbc95ff7f..331af996e 100644
--- a/src/openvpn/crypto_openssl.c
+++ b/src/openvpn/crypto_openssl.c
@@ -863,7 +863,7 @@ cipher_ctx_free(EVP_CIPHER_CTX *ctx)
 
 void
 cipher_ctx_init(EVP_CIPHER_CTX *ctx, const uint8_t *key,
-                const char *ciphername, int enc)
+                const char *ciphername, crypto_operation_t enc)
 {
     ASSERT(NULL != ciphername && NULL != ctx);
     evp_cipher_type *kt = cipher_get(ciphername);
diff --git a/src/openvpn/crypto_openssl.h b/src/openvpn/crypto_openssl.h
index c0e95b485..4cd988a05 100644
--- a/src/openvpn/crypto_openssl.h
+++ b/src/openvpn/crypto_openssl.h
@@ -85,6 +85,8 @@ typedef EVP_MD evp_md_type;
 /** Cipher is in GCM mode */
 #define OPENVPN_MODE_GCM        EVP_CIPH_GCM_MODE
 
+typedef int crypto_operation_t;
+
 /** Cipher should encrypt */
 #define OPENVPN_OP_ENCRYPT      1