From: Nikos Mavrogiannopoulos <nmav@gnutls.org>
Date: Thu, 24 Dec 2015 09:44:00 +0000 (+0200)
Subject: respect the max-record extension under DTLS
X-Git-Tag: gnutls_3_5_0~449
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eeb514a5b6b3be578ee779d6bfdaf4bf9521062f;p=thirdparty%2Fgnutls.git

respect the max-record extension under DTLS

This resolves issue with max-record being negotiated but
ignored. Resolves #61
---

diff --git a/lib/dtls.c b/lib/dtls.c
index fd8163319b..17a91285fd 100644
--- a/lib/dtls.c
+++ b/lib/dtls.c
@@ -62,8 +62,13 @@ transmit_message(gnutls_session_t session,
 	uint8_t *data, *mtu_data;
 	int ret = 0;
 	unsigned int offset, frag_len, data_size;
-	const unsigned int mtu =
-	    gnutls_dtls_get_data_mtu(session) - DTLS_HANDSHAKE_HEADER_SIZE;
+	unsigned int mtu =
+	    gnutls_dtls_get_data_mtu(session);
+
+	if (session->security_parameters.max_record_recv_size < mtu)
+		mtu = session->security_parameters.max_record_recv_size;
+
+	mtu -= DTLS_HANDSHAKE_HEADER_SIZE;
 
 	if (bufel->type == GNUTLS_CHANGE_CIPHER_SPEC) {
 		_gnutls_dtls_log
diff --git a/lib/gnutls_int.h b/lib/gnutls_int.h
index 4469603dfd..faad0d46a1 100644
--- a/lib/gnutls_int.h
+++ b/lib/gnutls_int.h
@@ -168,7 +168,7 @@ typedef enum record_flush_t {
  */
 #define MAX_RECORD_SEND_OVERHEAD(session) (MAX_CIPHER_BLOCK_SIZE/*iv*/+MAX_PAD_SIZE+((gnutls_compression_get(session)!=GNUTLS_COMP_NULL)?(EXTRA_COMP_SIZE):(0))+MAX_HASH_SIZE/*MAC*/)
 #define MAX_RECORD_SEND_SIZE(session) (IS_DTLS(session)? \
-	((size_t)gnutls_dtls_get_mtu(session)): \
+	(MIN((size_t)gnutls_dtls_get_mtu(session), (size_t)session->security_parameters.max_record_send_size+MAX_RECORD_SEND_OVERHEAD(session))): \
 	((size_t)session->security_parameters.max_record_send_size+MAX_RECORD_SEND_OVERHEAD(session)))
 #define MAX_PAD_SIZE 255
 #define EXTRA_COMP_SIZE 2048
@@ -1087,9 +1087,9 @@ inline static size_t max_user_send_size(gnutls_session_t session,
 {
 	size_t max;
 
-	if (IS_DTLS(session))
-		max = gnutls_dtls_get_data_mtu(session);
-	else {
+	if (IS_DTLS(session)) {
+		max = MIN(gnutls_dtls_get_data_mtu(session), session->security_parameters.max_record_send_size);
+	} else {
 		max = session->security_parameters.max_record_send_size;
 	}