From: Frederik Wedel-Heinen <frederik.wedel-heinen@dencrypt.dk>
Date: Wed, 18 Dec 2024 19:38:04 +0000 (+0100)
Subject: Avoid mallocing unprocessed_rcds and processed_rcds in dtls record layer
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=55c95848d1294f255bf03dfae9b16a6f873f2549;p=thirdparty%2Fopenssl.git

Avoid mallocing unprocessed_rcds and processed_rcds in dtls record layer

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/26211)
---

diff --git a/ssl/pqueue.c b/ssl/pqueue.c
index db161e25da1..a2a57cbf456 100644
--- a/ssl/pqueue.c
+++ b/ssl/pqueue.c
@@ -10,11 +10,6 @@
 #include "ssl_local.h"
 #include <openssl/bn.h>
 
-struct pqueue_st {
-    pitem *items;
-    int count;
-};
-
 pitem *pitem_new(unsigned char *prio64be, void *data)
 {
     pitem *item = OPENSSL_malloc(sizeof(*item));
diff --git a/ssl/record/methods/dtls_meth.c b/ssl/record/methods/dtls_meth.c
index a939238ddef..de497c57858 100644
--- a/ssl/record/methods/dtls_meth.c
+++ b/ssl/record/methods/dtls_meth.c
@@ -435,7 +435,7 @@ int dtls_get_more_records(OSSL_RECORD_LAYER *rl)
 
  again:
     /* if we're renegotiating, then there may be buffered records */
-    if (dtls_retrieve_rlayer_buffered_record(rl, rl->processed_rcds)) {
+    if (dtls_retrieve_rlayer_buffered_record(rl, &rl->processed_rcds)) {
         rl->num_recs = 1;
         return OSSL_RECORD_RETURN_SUCCESS;
     }
@@ -602,7 +602,7 @@ int dtls_get_more_records(OSSL_RECORD_LAYER *rl)
      */
     if (is_next_epoch) {
         if (rl->in_init) {
-            if (dtls_rlayer_buffer_record(rl, rl->unprocessed_rcds,
+            if (dtls_rlayer_buffer_record(rl, &rl->unprocessed_rcds,
                                           rr->seq_num) < 0) {
                 /* RLAYERfatal() already called */
                 return OSSL_RECORD_RETURN_FATAL;
@@ -652,27 +652,21 @@ static int dtls_free(OSSL_RECORD_LAYER *rl)
         rbuf->left = 0;
     }
 
-    if (rl->unprocessed_rcds != NULL) {
-        while ((item = pqueue_pop(rl->unprocessed_rcds)) != NULL) {
-            rdata = (DTLS_RLAYER_RECORD_DATA *)item->data;
-            /* Push to the next record layer */
-            ret &= BIO_write_ex(rl->next, rdata->packet, rdata->packet_length,
-                                &written);
-            OPENSSL_free(rdata->rbuf.buf);
-            OPENSSL_free(item->data);
-            pitem_free(item);
-        }
-        pqueue_free(rl->unprocessed_rcds);
+    while ((item = pqueue_pop(&rl->unprocessed_rcds)) != NULL) {
+        rdata = (DTLS_RLAYER_RECORD_DATA *)item->data;
+        /* Push to the next record layer */
+        ret &= BIO_write_ex(rl->next, rdata->packet, rdata->packet_length,
+                            &written);
+        OPENSSL_free(rdata->rbuf.buf);
+        OPENSSL_free(item->data);
+        pitem_free(item);
     }
 
-    if (rl->processed_rcds!= NULL) {
-        while ((item = pqueue_pop(rl->processed_rcds)) != NULL) {
-            rdata = (DTLS_RLAYER_RECORD_DATA *)item->data;
-            OPENSSL_free(rdata->rbuf.buf);
-            OPENSSL_free(item->data);
-            pitem_free(item);
-        }
-        pqueue_free(rl->processed_rcds);
+    while ((item = pqueue_pop(&rl->processed_rcds)) != NULL) {
+        rdata = (DTLS_RLAYER_RECORD_DATA *)item->data;
+        OPENSSL_free(rdata->rbuf.buf);
+        OPENSSL_free(item->data);
+        pitem_free(item);
     }
 
     return tls_free(rl) && ret;
@@ -705,17 +699,6 @@ dtls_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers,
     if (ret != OSSL_RECORD_RETURN_SUCCESS)
         return ret;
 
-    (*retrl)->unprocessed_rcds = pqueue_new();
-    (*retrl)->processed_rcds = pqueue_new();
-
-    if ((*retrl)->unprocessed_rcds == NULL
-            || (*retrl)->processed_rcds == NULL) {
-        dtls_free(*retrl);
-        *retrl = NULL;
-        ERR_raise(ERR_LIB_SSL, ERR_R_SSL_LIB);
-        return OSSL_RECORD_RETURN_FATAL;
-    }
-
     (*retrl)->isdtls = 1;
     (*retrl)->epoch = epoch;
     (*retrl)->in_init = 1;
diff --git a/ssl/record/methods/recmethod_local.h b/ssl/record/methods/recmethod_local.h
index d38ee2c7cd7..8cf1141ee01 100644
--- a/ssl/record/methods/recmethod_local.h
+++ b/ssl/record/methods/recmethod_local.h
@@ -351,8 +351,8 @@ struct ossl_record_layer_st {
     size_t taglen;
 
     /* DTLS received handshake records (processed and unprocessed) */
-    struct pqueue_st *unprocessed_rcds;
-    struct pqueue_st *processed_rcds;
+    pqueue unprocessed_rcds;
+    pqueue processed_rcds;
 
     /* records being received in the current epoch */
     DTLS_BITMAP bitmap;
diff --git a/ssl/ssl_local.h b/ssl/ssl_local.h
index 6e2662e7716..1ae86d2e9de 100644
--- a/ssl/ssl_local.h
+++ b/ssl/ssl_local.h
@@ -1963,6 +1963,11 @@ typedef struct hm_fragment_st {
 typedef struct pqueue_st pqueue;
 typedef struct pitem_st pitem;
 
+struct pqueue_st {
+    pitem *items;
+    int count;
+};
+
 struct pitem_st {
     unsigned char priority[8];  /* 64-bit value in big-endian encoding */
     void *data;