From: Tomas Mraz <tomas@openssl.org>
Date: Wed, 23 Aug 2023 16:12:32 +0000 (+0200)
Subject: Avoid issues with endianness when type is used in SSL_trace()
X-Git-Tag: openssl-3.2.0-alpha1~149
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1cc8c53b0fc06d148a9f62e5d2d5bcd859f948cf;p=thirdparty%2Fopenssl.git

Avoid issues with endianness when type is used in SSL_trace()

The TLS record type is a single byte value so we can
use uint8_t for it. This allows passing its address
directly to SSL_trace() instead of converting it to
a single byte type first.

Reviewed-by: Hugo Landau <hlandau@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/21823)
---

diff --git a/doc/designs/quic-design/record-layer.md b/doc/designs/quic-design/record-layer.md
index da424064a38..8db4b319c54 100644
--- a/doc/designs/quic-design/record-layer.md
+++ b/doc/designs/quic-design/record-layer.md
@@ -507,7 +507,7 @@ struct ossl_record_method_st {
      * multiple records in one go and buffer them.
      */
     int (*read_record)(OSSL_RECORD_LAYER *rl, void **rechandle, int *rversion,
-                      int *type, unsigned char **data, size_t *datalen,
+                      uint8_t *type, unsigned char **data, size_t *datalen,
                       uint16_t *epoch, unsigned char *seq_num);
     /*
      * Release a buffer associated with a record previously read with
diff --git a/include/internal/recordmethod.h b/include/internal/recordmethod.h
index 247c24f0324..c49b5da4ee5 100644
--- a/include/internal/recordmethod.h
+++ b/include/internal/recordmethod.h
@@ -232,7 +232,7 @@ struct ossl_record_method_st {
      * multiple records in one go and buffer them.
      */
     int (*read_record)(OSSL_RECORD_LAYER *rl, void **rechandle, int *rversion,
-                      int *type, const unsigned char **data, size_t *datalen,
+                      uint8_t *type, const unsigned char **data, size_t *datalen,
                       uint16_t *epoch, unsigned char *seq_num);
     /*
      * Release length bytes from a buffer associated with a record previously
diff --git a/ssl/quic/quic_tls.c b/ssl/quic/quic_tls.c
index 8f104f63e3f..14b23baeb27 100644
--- a/ssl/quic/quic_tls.c
+++ b/ssl/quic/quic_tls.c
@@ -356,7 +356,7 @@ static int quic_retry_write_records(OSSL_RECORD_LAYER *rl)
 }
 
 static int quic_read_record(OSSL_RECORD_LAYER *rl, void **rechandle,
-                            int *rversion, int *type, const unsigned char **data,
+                            int *rversion, uint8_t *type, const unsigned char **data,
                             size_t *datalen, uint16_t *epoch,
                             unsigned char *seq_num)
 {
diff --git a/ssl/record/methods/recmethod_local.h b/ssl/record/methods/recmethod_local.h
index b5dfaf30d28..cf0af62f3a1 100644
--- a/ssl/record/methods/recmethod_local.h
+++ b/ssl/record/methods/recmethod_local.h
@@ -459,7 +459,7 @@ int tls_retry_write_records(OSSL_RECORD_LAYER *rl);
 int tls_get_alert_code(OSSL_RECORD_LAYER *rl);
 int tls_set1_bio(OSSL_RECORD_LAYER *rl, BIO *bio);
 int tls_read_record(OSSL_RECORD_LAYER *rl, void **rechandle, int *rversion,
-                    int *type, const unsigned char **data, size_t *datalen,
+                    uint8_t *type, const unsigned char **data, size_t *datalen,
                     uint16_t *epoch, unsigned char *seq_num);
 int tls_release_record(OSSL_RECORD_LAYER *rl, void *rechandle, size_t length);
 int tls_default_set_protocol_version(OSSL_RECORD_LAYER *rl, int version);
diff --git a/ssl/record/methods/tls_common.c b/ssl/record/methods/tls_common.c
index 505e1331618..576b651aa7c 100644
--- a/ssl/record/methods/tls_common.c
+++ b/ssl/record/methods/tls_common.c
@@ -1088,7 +1088,7 @@ int tls13_common_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec)
 }
 
 int tls_read_record(OSSL_RECORD_LAYER *rl, void **rechandle, int *rversion,
-                    int *type, const unsigned char **data, size_t *datalen,
+                    uint8_t *type, const unsigned char **data, size_t *datalen,
                     uint16_t *epoch, unsigned char *seq_num)
 {
     TLS_RL_RECORD *rec;
diff --git a/ssl/record/record.h b/ssl/record/record.h
index 7dcbbb36e90..a8fa6833e90 100644
--- a/ssl/record/record.h
+++ b/ssl/record/record.h
@@ -22,7 +22,7 @@
 typedef struct tls_record_st {
     void *rechandle;
     int version;
-    int type;
+    uint8_t type;
     /* The data buffer containing bytes from the record */
     const unsigned char *data;
     /*