tls_common.c: prevent max_early_data overflow in rlayer_early_data_count_ok()

Make the local max_early_data variable uint64_t so an overflow
cannot occur if the max_early_data field in the record layer struct
has the maximum value: UNT32_MAX (0xFFFFFFFF).

Resolves: https://github.com/openssl/openssl/issues/31533

Reviewed-by: Eugene Syromiatnikov <esyr@openssl.org>
Reviewed-by: Frederik Wedel-Heinen <fwh.openssl@gmail.com>
Reviewed-by: Tim Hudson <tjh@openssl.org>
MergeDate: Sun Jun 21 23:50:02 2026
(Merged from https://github.com/openssl/openssl/pull/31538)

diff --git a/ssl/record/methods/tls_common.c b/ssl/record/methods/tls_common.c
index 685139531e82692f00392ba5d7f9be3fee09132a..e149d09898523d2c4b22adf935ef4fcfe08cf270 100644 (file)
--- a/ssl/record/methods/tls_common.c
+++ b/ssl/record/methods/tls_common.c
@@ -497,7 +497,7 @@ static int tls_record_app_data_waiting(OSSL_RECORD_LAYER *rl)
 static int rlayer_early_data_count_ok(OSSL_RECORD_LAYER *rl, size_t length,
     size_t overhead, int send)
 {
-    uint32_t max_early_data = rl->max_early_data;
+    uint64_t max_early_data = rl->max_early_data;
 
     if (max_early_data == 0) {
         RLAYERfatal(rl, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE,
@@ -506,7 +506,7 @@ static int rlayer_early_data_count_ok(OSSL_RECORD_LAYER *rl, size_t length,
     }
 
     /* If we are dealing with ciphertext we need to allow for the overhead */
-    max_early_data += (uint32_t)overhead;
+    max_early_data += overhead;
 
     if (rl->early_data_count + length > max_early_data) {
         RLAYERfatal(rl, send ? SSL_AD_INTERNAL_ERROR : SSL_AD_UNEXPECTED_MESSAGE,