fix UB in priority_queue

clang-22 reports UB during test make test V=0 TESTS=test_priority_queue

ssl/priority_queue.c:159:13: runtime error: call to function size_t_compare through pointer to incorrect function type 'int (*)(const void *, const void *)'
/home/npajkovsky/openssl/openssl/test/priority_queue_test.c:28: note: size_t_compare defined here
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior ssl/priority_queue.c:159:13

Fixes: https://github.com/openssl/project/issues/1953
Signed-off-by: Nikola Pajkovsky <nikolap@openssl.org>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Norbert Pocs <norbertp@openssl.org>
MergeDate: Tue May 19 10:43:59 2026
(Merged from https://github.com/openssl/openssl/pull/31209)

diff --git a/include/internal/priority_queue.h b/include/internal/priority_queue.h
index 067c8815961e96db44d7adb4b7a9a04409468c4f..9f4a6361656e7a747054c735496e8a3332fcc11f 100644 (file)
--- a/include/internal/priority_queue.h
+++ b/include/internal/priority_queue.h
@@ -16,56 +16,56 @@
 
 #define PRIORITY_QUEUE_OF(type) OSSL_PRIORITY_QUEUE_##type
 
-#define DEFINE_PRIORITY_QUEUE_OF_INTERNAL(type, ctype)                                                                              \
-    typedef struct ossl_priority_queue_st_##type PRIORITY_QUEUE_OF(type);                                                           \
-    static ossl_unused ossl_inline PRIORITY_QUEUE_OF(type) * ossl_pqueue_##type##_new(int (*compare)(const ctype *, const ctype *)) \
-    {                                                                                                                               \
-        return (PRIORITY_QUEUE_OF(type) *)ossl_pqueue_new(                                                                          \
-            (int (*)(const void *, const void *))compare);                                                                          \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline void                                                                                             \
-    ossl_pqueue_##type##_free(PRIORITY_QUEUE_OF(type) * pq)                                                                         \
-    {                                                                                                                               \
-        ossl_pqueue_free((OSSL_PQUEUE *)pq);                                                                                        \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline void                                                                                             \
-    ossl_pqueue_##type##_pop_free(PRIORITY_QUEUE_OF(type) * pq,                                                                     \
-        void (*freefunc)(ctype *))                                                                                                  \
-    {                                                                                                                               \
-        ossl_pqueue_pop_free((OSSL_PQUEUE *)pq, (void (*)(void *))freefunc);                                                        \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline int                                                                                              \
-    ossl_pqueue_##type##_reserve(PRIORITY_QUEUE_OF(type) * pq, size_t n)                                                            \
-    {                                                                                                                               \
-        return ossl_pqueue_reserve((OSSL_PQUEUE *)pq, n);                                                                           \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline size_t                                                                                           \
-    ossl_pqueue_##type##_num(const PRIORITY_QUEUE_OF(type) * pq)                                                                    \
-    {                                                                                                                               \
-        return ossl_pqueue_num((OSSL_PQUEUE *)pq);                                                                                  \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline int                                                                                              \
-    ossl_pqueue_##type##_push(PRIORITY_QUEUE_OF(type) * pq,                                                                         \
-        ctype * data, size_t *elem)                                                                                                 \
-    {                                                                                                                               \
-        return ossl_pqueue_push((OSSL_PQUEUE *)pq, (void *)data, elem);                                                             \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline ctype *                                                                                          \
-    ossl_pqueue_##type##_peek(const PRIORITY_QUEUE_OF(type) * pq)                                                                   \
-    {                                                                                                                               \
-        return (type *)ossl_pqueue_peek((OSSL_PQUEUE *)pq);                                                                         \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline ctype *                                                                                          \
-    ossl_pqueue_##type##_pop(PRIORITY_QUEUE_OF(type) * pq)                                                                          \
-    {                                                                                                                               \
-        return (type *)ossl_pqueue_pop((OSSL_PQUEUE *)pq);                                                                          \
-    }                                                                                                                               \
-    static ossl_unused ossl_inline ctype *                                                                                          \
-    ossl_pqueue_##type##_remove(PRIORITY_QUEUE_OF(type) * pq,                                                                       \
-        size_t elem)                                                                                                                \
-    {                                                                                                                               \
-        return (type *)ossl_pqueue_remove((OSSL_PQUEUE *)pq, elem);                                                                 \
-    }                                                                                                                               \
+#define DEFINE_PRIORITY_QUEUE_OF_INTERNAL(type, ctype)                                                                            \
+    typedef struct ossl_priority_queue_st_##type PRIORITY_QUEUE_OF(type);                                                         \
+    static ossl_unused ossl_inline PRIORITY_QUEUE_OF(type) * ossl_pqueue_##type##_new(int (*compare)(const void *, const void *)) \
+    {                                                                                                                             \
+        return (PRIORITY_QUEUE_OF(type) *)ossl_pqueue_new(                                                                        \
+            compare);                                                                                                             \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline void                                                                                           \
+    ossl_pqueue_##type##_free(PRIORITY_QUEUE_OF(type) * pq)                                                                       \
+    {                                                                                                                             \
+        ossl_pqueue_free((OSSL_PQUEUE *)pq);                                                                                      \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline void                                                                                           \
+    ossl_pqueue_##type##_pop_free(PRIORITY_QUEUE_OF(type) * pq,                                                                   \
+        void (*freefunc)(void *))                                                                                                 \
+    {                                                                                                                             \
+        ossl_pqueue_pop_free((OSSL_PQUEUE *)pq, freefunc);                                                                        \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline int                                                                                            \
+    ossl_pqueue_##type##_reserve(PRIORITY_QUEUE_OF(type) * pq, size_t n)                                                          \
+    {                                                                                                                             \
+        return ossl_pqueue_reserve((OSSL_PQUEUE *)pq, n);                                                                         \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline size_t                                                                                         \
+    ossl_pqueue_##type##_num(const PRIORITY_QUEUE_OF(type) * pq)                                                                  \
+    {                                                                                                                             \
+        return ossl_pqueue_num((OSSL_PQUEUE *)pq);                                                                                \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline int                                                                                            \
+    ossl_pqueue_##type##_push(PRIORITY_QUEUE_OF(type) * pq,                                                                       \
+        ctype * data, size_t *elem)                                                                                               \
+    {                                                                                                                             \
+        return ossl_pqueue_push((OSSL_PQUEUE *)pq, (void *)data, elem);                                                           \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline ctype *                                                                                        \
+    ossl_pqueue_##type##_peek(const PRIORITY_QUEUE_OF(type) * pq)                                                                 \
+    {                                                                                                                             \
+        return (type *)ossl_pqueue_peek((OSSL_PQUEUE *)pq);                                                                       \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline ctype *                                                                                        \
+    ossl_pqueue_##type##_pop(PRIORITY_QUEUE_OF(type) * pq)                                                                        \
+    {                                                                                                                             \
+        return (type *)ossl_pqueue_pop((OSSL_PQUEUE *)pq);                                                                        \
+    }                                                                                                                             \
+    static ossl_unused ossl_inline ctype *                                                                                        \
+    ossl_pqueue_##type##_remove(PRIORITY_QUEUE_OF(type) * pq,                                                                     \
+        size_t elem)                                                                                                              \
+    {                                                                                                                             \
+        return (type *)ossl_pqueue_remove((OSSL_PQUEUE *)pq, elem);                                                               \
+    }                                                                                                                             \
     struct ossl_priority_queue_st_##type
 
 #define DEFINE_PRIORITY_QUEUE_OF(type) \

diff --git a/ssl/quic/quic_rcidm.c b/ssl/quic/quic_rcidm.c
index 0d5cb0337b29bc68bd4756fe142897835046d1bd..9a92ec6e351d1290fe46d46ddd7e155d0b1e3dd6 100644 (file)
--- a/ssl/quic/quic_rcidm.c
+++ b/ssl/quic/quic_rcidm.c
@@ -270,8 +270,11 @@ static void rcidm_check_rcid(QUIC_RCIDM *rcidm, RCID *rcid)
     assert(rcid->state != RCID_STATE_RETIRING || rcidm->num_retiring > 0);
 }
 
-static int rcid_cmp(const RCID *a, const RCID *b)
+static int rcid_cmp(const void *av, const void *bv)
 {
+    const RCID *a = av;
+    const RCID *b = bv;
+
     if (a->seq_num < b->seq_num)
         return -1;
     if (a->seq_num > b->seq_num)

diff --git a/test/priority_queue_test.c b/test/priority_queue_test.c
index 8c09b50418c6da2b2f07562aa9b2ac0e28aa7756..9cf85575941709b8c15be91f9f10cda7c288554d 100644 (file)
--- a/test/priority_queue_test.c
+++ b/test/priority_queue_test.c
@@ -24,8 +24,11 @@ DEFINE_PRIORITY_QUEUE_OF(size_t);
 
 static size_t num_rec_freed;
 
-static int size_t_compare(const size_t *a, const size_t *b)
+static int size_t_compare(const void *av, const void *bv)
 {
+    const size_t *a = av;
+    const size_t *b = bv;
+
     if (*a < *b)
         return -1;
     if (*a > *b)
@@ -33,17 +36,12 @@ static int size_t_compare(const size_t *a, const size_t *b)
     return 0;
 }
 
-static int qsort_size_t_compare(const void *a, const void *b)
-{
-    return size_t_compare((const size_t *)a, (const size_t *)b);
-}
-
 static int qsort_size_t_compare_rev(const void *a, const void *b)
 {
-    return size_t_compare((const size_t *)b, (const size_t *)a);
+    return size_t_compare(b, a);
 }
 
-static void free_checker(ossl_unused size_t *p)
+static void free_checker(ossl_unused void *p)
 {
     num_rec_freed++;
 }
@@ -72,7 +70,7 @@ static int test_size_t_priority_queue_int(int reserve, int order, int count,
     for (i = 0; i < count; i++)
         values[i] = random ? test_random() : (size_t)(count - i);
     memcpy(sorted, values, sizeof(*sorted) * count);
-    qsort(sorted, count, sizeof(*sorted), &qsort_size_t_compare);
+    qsort(sorted, count, sizeof(*sorted), &size_t_compare);
 
     if (order == 1)
         memcpy(values, sorted, sizeof(*values) * count);
@@ -105,7 +103,7 @@ static int test_size_t_priority_queue_int(int reserve, int order, int count,
             }
         }
         memcpy(sorted, values, sizeof(*sorted) * count);
-        qsort(sorted, count, sizeof(*sorted), &qsort_size_t_compare);
+        qsort(sorted, count, sizeof(*sorted), &size_t_compare);
     }
     for (i = 0; ossl_pqueue_size_t_peek(pq) != NULL; i++)
         if (!TEST_size_t_eq(*ossl_pqueue_size_t_peek(pq), sorted[i])
@@ -164,8 +162,11 @@ typedef struct info_st {
 
 DEFINE_PRIORITY_QUEUE_OF(INFO);
 
-static int cmp(const INFO *a, const INFO *b)
+static int cmp(const void *av, const void *bv)
 {
+    const INFO *a = av;
+    const INFO *b = bv;
+
     if (a->seq_num < b->seq_num)
         return -1;
     if (a->seq_num > b->seq_num)

diff --git a/test/quic_cc_test.c b/test/quic_cc_test.c
index abe10780fc2cc3544efb933bba98d03cd7341be3..b4ce5caa1b15b22616b681dcde25adb71285565c 100644 (file)
--- a/test/quic_cc_test.c
+++ b/test/quic_cc_test.c
@@ -86,8 +86,11 @@ typedef struct net_pkt_st {
 
 DEFINE_PRIORITY_QUEUE_OF(NET_PKT);
 
-static int net_pkt_cmp(const NET_PKT *a, const NET_PKT *b)
+static int net_pkt_cmp(const void *av, const void *bv)
 {
+    const NET_PKT *a = av;
+    const NET_PKT *b = bv;
+
     return ossl_time_compare(a->next_time, b->next_time);
 }
 
@@ -125,7 +128,7 @@ static int net_sim_init(struct net_sim *s,
     return 1;
 }
 
-static void do_free(NET_PKT *pkt)
+static void do_free(void *pkt)
 {
     OPENSSL_free(pkt);
 }