MINOR: quic: split global CID tree between FE and BE sides

QUIC CIDs are stored in a global tree. Prior to this patch, CIDs used on
both frontend and backend sides were mixed together.

This patch implement CID storage separation between FE and BE sides. The
original tre quic_cid_trees is splitted as
quic_fe_cid_trees/quic_be_cid_trees.

This patch should reduce contention between frontend and backend usages.
Also, it should reduce the risk of random CID collision.

diff --git a/include/haproxy/quic_cid-t.h b/include/haproxy/quic_cid-t.h
index f19ce2626c075d53982ff06be4f20ac0b442280e..703fe6e7628e8d62367ba851da275b2f2f9c31ea 100644 (file)
--- a/include/haproxy/quic_cid-t.h
+++ b/include/haproxy/quic_cid-t.h
@@ -24,6 +24,12 @@ struct quic_cid {
        unsigned char len; /* size of QUIC CID */
 };
 
+/* Determines whether a CID is used for frontend or backend connections. */
+enum quic_cid_side {
+       QUIC_CID_SIDE_FE,
+       QUIC_CID_SIDE_BE
+};
+
 /* QUIC connection id attached to a QUIC connection.
  *
  * This structure is used to match received packets DCIDs with the
@@ -34,11 +40,12 @@ struct quic_connection_id {
        uint64_t retire_prior_to;
        unsigned char stateless_reset_token[QUIC_STATELESS_RESET_TOKEN_LEN];
 
-       struct ebmb_node node; /* node for receiver tree, cid.data as key */
-       struct quic_cid cid;   /* CID data */
+       struct ebmb_node node;   /* node for receiver tree, cid.data as key */
+       struct quic_cid cid;     /* CID data */
 
-       struct quic_conn *qc;  /* QUIC connection using this CID */
-       uint tid;              /* Attached Thread ID for the connection. */
+       struct quic_conn *qc;    /* QUIC connection using this CID */
+       uint tid;                /* Attached Thread ID for the connection. */
+       enum quic_cid_side side; /* side where this CID is used */
 };
 
 #endif /* _HAPROXY_QUIC_CID_T_H */

diff --git a/include/haproxy/quic_cid.h b/include/haproxy/quic_cid.h
index 13a0669d1d4b7aba618a772f804eb157dd8b33bb..05f3e5b80d014d0dc70c0b9f403456648cc5d5e8 100644 (file)
--- a/include/haproxy/quic_cid.h
+++ b/include/haproxy/quic_cid.h
@@ -15,9 +15,10 @@
 #include <haproxy/quic_rx-t.h>
 #include <haproxy/proto_quic.h>
 
-extern struct quic_cid_tree *quic_cid_trees;
+extern struct quic_cid_tree *quic_fe_cid_trees;
+extern struct quic_cid_tree *quic_be_cid_trees;
 
-struct quic_connection_id *quic_cid_alloc(void);
+struct quic_connection_id *quic_cid_alloc(enum quic_cid_side side);
 
 int quic_cid_generate_random(struct quic_connection_id *conn_id);
 int quic_cid_generate_from_hash(struct quic_connection_id *conn_id, uint64_t hash64);
@@ -81,11 +82,18 @@ static inline uchar quic_cid_tree_idx(const struct quic_cid *cid)
        return _quic_cid_tree_idx(cid->data);
 }
 
+/* Returns the tree instance responsible for <conn_id> storage. */
+static inline struct quic_cid_tree *quic_cid_get_tree(const struct quic_connection_id *conn_id)
+{
+       const int tree_idx = quic_cid_tree_idx(&conn_id->cid);
+       return conn_id->side == QUIC_CID_SIDE_FE ?
+         &quic_fe_cid_trees[tree_idx] : &quic_be_cid_trees[tree_idx];
+}
+
 /* Remove <conn_id> from global CID tree as a thread-safe operation. */
 static inline void quic_cid_delete(struct quic_connection_id *conn_id)
 {
-       const uchar idx = quic_cid_tree_idx(&conn_id->cid);
-       struct quic_cid_tree __maybe_unused *tree = &quic_cid_trees[idx];
+       struct quic_cid_tree __maybe_unused *tree = quic_cid_get_tree(conn_id);
 
        HA_RWLOCK_WRLOCK(QC_CID_LOCK, &tree->lock);
        ebmb_delete(&conn_id->node);

diff --git a/include/haproxy/quic_conn.h b/include/haproxy/quic_conn.h
index 3c447c73d30c24e772c8d7fd8444863652a5a582..84318c1bfd35fb3f26b33f2ee2a3af630df23c26 100644 (file)
--- a/include/haproxy/quic_conn.h
+++ b/include/haproxy/quic_conn.h
@@ -91,6 +91,12 @@ static inline int qc_is_back(const struct quic_conn *qc)
        return qc->flags & QUIC_FL_CONN_IS_BACK;
 }
 
+static inline enum quic_cid_side qc_cid_side(const struct quic_conn *qc)
+{
+       return !(qc->flags & QUIC_FL_CONN_IS_BACK) ?
+               QUIC_CID_SIDE_FE : QUIC_CID_SIDE_BE;
+}
+
 /* Free the CIDs attached to <conn> QUIC connection. */
 static inline void free_quic_conn_cids(struct quic_conn *conn)
 {

diff --git a/src/quic_cid.c b/src/quic_cid.c
index 166d9074d06dfa574621868f4ccc2625b03d3e42..28164c7d1fef5be91d2dd29741b1811d25feb716 100644 (file)
--- a/src/quic_cid.c
+++ b/src/quic_cid.c
@@ -17,11 +17,12 @@
  *
  * . CID global storage
  * CIDs generated by haproxy and reuse by the peer as DCID are stored in a
- * global tree. Tree access must only be done under lock protection.
+ * global tree. Tree access must only be done under lock protection. Separate
+ * trees are used on frontend and backend sides.
  *
  * . CID global tree splitting
- * To reduce thread contention, global CID tree is in reality split into 256
- * distinct tree instances. Each CID is assigned to a single tree instance
+ * To reduce the thread contention, a global CID tree is in reality splitted
+ * into 256 distinct instances. Each CID is assigned to a single tree instance
  * based on its content. Use quic_cid_tree_idx() to retrieve the expected tree
  * location for a CID.
  *
@@ -35,7 +36,8 @@
  */
 
 #define QUIC_CID_TREES_CNT 256
-struct quic_cid_tree *quic_cid_trees;
+struct quic_cid_tree *quic_fe_cid_trees;
+struct quic_cid_tree *quic_be_cid_trees;
 
 /* Initialize the stateless reset token attached to <conn_id> connection ID.
  * Returns 1 if succeeded, 0 if not.
@@ -120,7 +122,7 @@ static struct quic_cid quic_derive_cid(const struct quic_cid *orig,
  *
  * Returns the CID or NULL on allocation failure.
  */
-struct quic_connection_id *quic_cid_alloc(void)
+struct quic_connection_id *quic_cid_alloc(enum quic_cid_side side)
 {
        struct quic_connection_id *conn_id;
 
@@ -135,6 +137,7 @@ struct quic_connection_id *quic_cid_alloc(void)
 
        HA_ATOMIC_STORE(&conn_id->tid, tid);
        conn_id->qc = NULL;
+       conn_id->side = side;
 
        TRACE_LEAVE(QUIC_EV_CONN_TXPKT);
        return conn_id;
@@ -280,8 +283,8 @@ int quic_cid_insert(struct quic_connection_id *conn_id, int *new_tid)
 
        if (new_tid)
                *new_tid = -1;
-       tree = &quic_cid_trees[quic_cid_tree_idx(&conn_id->cid)];
 
+       tree = quic_cid_get_tree(conn_id);
        HA_RWLOCK_WRLOCK(QC_CID_LOCK, &tree->lock);
        node = ebmb_insert(&tree->root, &conn_id->node, conn_id->cid.len);
        if (node != &conn_id->node) {
@@ -312,7 +315,8 @@ int quic_cmp_cid_conn(const unsigned char *cid, size_t cid_len,
        struct ebmb_node *node;
        int ret = 0;
 
-       tree = &quic_cid_trees[_quic_cid_tree_idx(cid)];
+       /* This function is only used on frontend side. */
+       tree = &quic_fe_cid_trees[_quic_cid_tree_idx(cid)];
        HA_RWLOCK_RDLOCK(QC_CID_LOCK, &tree->lock);
        node = ebmb_lookup(&tree->root, cid, cid_len);
        if (node) {
@@ -342,7 +346,8 @@ int quic_get_cid_tid(const unsigned char *cid, size_t cid_len,
        struct ebmb_node *node;
        int cid_tid = -1;
 
-       tree = &quic_cid_trees[_quic_cid_tree_idx(cid)];
+       /* This function is only used on frontend side. */
+       tree = &quic_fe_cid_trees[_quic_cid_tree_idx(cid)];
        HA_RWLOCK_RDLOCK(QC_CID_LOCK, &tree->lock);
        node = ebmb_lookup(&tree->root, cid, cid_len);
        if (node) {
@@ -371,7 +376,7 @@ int quic_get_cid_tid(const unsigned char *cid, size_t cid_len,
                orig.len = cid_len;
                derive_cid = quic_derive_cid(&orig, cli_addr);
 
-               tree = &quic_cid_trees[quic_cid_tree_idx(&derive_cid)];
+               tree = &quic_fe_cid_trees[quic_cid_tree_idx(&derive_cid)];
                HA_RWLOCK_RDLOCK(QC_CID_LOCK, &tree->lock);
                node = ebmb_lookup(&tree->root, cid, cid_len);
                if (node) {
@@ -404,8 +409,10 @@ struct quic_conn *retrieve_qc_conn_from_cid(struct quic_rx_packet *pkt,
        TRACE_ENTER(QUIC_EV_CONN_RXPKT);
        *new_tid = -1;
 
-       /* First look into DCID tree. */
-       tree = &quic_cid_trees[_quic_cid_tree_idx(pkt->dcid.data)];
+       /* First look into DCID tree.
+        * This function is only used on frontend side.
+        */
+       tree = &quic_fe_cid_trees[_quic_cid_tree_idx(pkt->dcid.data)];
        HA_RWLOCK_RDLOCK(QC_CID_LOCK, &tree->lock);
        node = ebmb_lookup(&tree->root, pkt->dcid.data, pkt->dcid.len);
 
@@ -419,7 +426,7 @@ struct quic_conn *retrieve_qc_conn_from_cid(struct quic_rx_packet *pkt,
 
                HA_RWLOCK_RDUNLOCK(QC_CID_LOCK, &tree->lock);
 
-               tree = &quic_cid_trees[quic_cid_tree_idx(&derive_cid)];
+               tree = &quic_fe_cid_trees[quic_cid_tree_idx(&derive_cid)];
                HA_RWLOCK_RDLOCK(QC_CID_LOCK, &tree->lock);
                node = ebmb_lookup(&tree->root, derive_cid.data, derive_cid.len);
        }
@@ -475,28 +482,54 @@ int qc_build_new_connection_id_frm(struct quic_conn *qc,
        return ret;
 }
 
-static int quic_alloc_global_cid_tree(void)
+static int quic_alloc_global_fe_cid_tree(void)
+{
+       int i;
+
+       quic_fe_cid_trees = calloc(QUIC_CID_TREES_CNT, sizeof(*quic_fe_cid_trees));
+       if (!quic_fe_cid_trees) {
+               ha_alert("Failed to allocate global quic CIDs trees.\n");
+               return 0;
+       }
+
+       for (i = 0; i < QUIC_CID_TREES_CNT; ++i) {
+               HA_RWLOCK_INIT(&quic_fe_cid_trees[i].lock);
+               quic_fe_cid_trees[i].root = EB_ROOT_UNIQUE;
+       }
+
+       return 1;
+}
+REGISTER_POST_CHECK(quic_alloc_global_fe_cid_tree);
+
+static int quic_deallocate_global_fe_cid_tree(void)
+{
+       ha_free(&quic_fe_cid_trees);
+       return 1;
+}
+REGISTER_POST_DEINIT(quic_deallocate_global_fe_cid_tree);
+
+static int quic_alloc_global_be_cid_tree(void)
 {
        int i;
 
-       quic_cid_trees = calloc(QUIC_CID_TREES_CNT, sizeof(*quic_cid_trees));
-       if (!quic_cid_trees) {
+       quic_be_cid_trees = calloc(QUIC_CID_TREES_CNT, sizeof(*quic_be_cid_trees));
+       if (!quic_be_cid_trees) {
                ha_alert("Failed to allocate global quic CIDs trees.\n");
                return 0;
        }
 
        for (i = 0; i < QUIC_CID_TREES_CNT; ++i) {
-               HA_RWLOCK_INIT(&quic_cid_trees[i].lock);
-               quic_cid_trees[i].root = EB_ROOT_UNIQUE;
+               HA_RWLOCK_INIT(&quic_be_cid_trees[i].lock);
+               quic_be_cid_trees[i].root = EB_ROOT_UNIQUE;
        }
 
        return 1;
 }
-REGISTER_POST_CHECK(quic_alloc_global_cid_tree);
+REGISTER_POST_CHECK(quic_alloc_global_be_cid_tree);
 
-static int quic_deallocate_global_cid_tree(void)
+static int quic_deallocate_global_be_cid_tree(void)
 {
-       ha_free(&quic_cid_trees);
+       ha_free(&quic_be_cid_trees);
        return 1;
 }
-REGISTER_POST_DEINIT(quic_deallocate_global_cid_tree);
+REGISTER_POST_DEINIT(quic_deallocate_global_be_cid_tree);

diff --git a/src/quic_conn.c b/src/quic_conn.c
index 249f7a616bc9d6708c893e646a614fc9ed38d0ed..2e15bcb29769427283f0afd8fb10786736861960 100644 (file)
--- a/src/quic_conn.c
+++ b/src/quic_conn.c
@@ -550,7 +550,7 @@ int quic_build_post_handshake_frames(struct quic_conn *qc,
                        goto err;
                }
 
-               conn_id = quic_cid_alloc();
+               conn_id = quic_cid_alloc(qc_cid_side(qc));
                if (!conn_id) {
                        qc_frm_free(qc, &frm);
                        TRACE_ERROR("CID allocation error", QUIC_EV_CONN_IO_CB, qc);

diff --git a/src/quic_rx.c b/src/quic_rx.c
index ca4f916f655a95f6b4e0c563f2a0b15e74dbcb4a..c01cc9d338aa3c3bc6f7850d3000eaf2b1d03d0b 100644 (file)
--- a/src/quic_rx.c
+++ b/src/quic_rx.c
@@ -1052,7 +1052,7 @@ static int qc_parse_pkt_frms(struct quic_conn *qc, struct quic_rx_packet *pkt,
                        pool_free(pool_head_quic_connection_id, conn_id);
                        TRACE_PROTO("CID retired", QUIC_EV_CONN_PSTRM, qc);
 
-                       conn_id = quic_cid_alloc();
+                       conn_id = quic_cid_alloc(qc_cid_side(qc));
                        if (!conn_id) {
                                TRACE_ERROR("CID allocation error", QUIC_EV_CONN_IO_CB, qc);
                                quic_set_connection_close(qc, quic_err_transport(QC_ERR_INTERNAL_ERROR));
@@ -1798,7 +1798,7 @@ static struct quic_conn *quic_rx_pkt_retrieve_conn(struct quic_rx_packet *pkt,
 
                        pkt->saddr = dgram->saddr;
 
-                       conn_id = quic_cid_alloc();
+                       conn_id = quic_cid_alloc(QUIC_CID_SIDE_FE);
                        if (!conn_id) {
                                TRACE_ERROR("error on first CID allocation",
                                            QUIC_EV_CONN_LPKT, NULL, NULL, NULL, pkt->version);

diff --git a/src/xprt_quic.c b/src/xprt_quic.c
index e16274771f1d1aa5fbe089bc3f09129d3deba3e0..c21b1b551f35def82eb2ef5137563335bef0557b 100644 (file)
--- a/src/xprt_quic.c
+++ b/src/xprt_quic.c
@@ -138,7 +138,7 @@ static int qc_conn_init(struct connection *conn, void **xprt_ctx)
                int retry_rand_cid = 3; /* Number of random retries on CID collision. */
                struct server *srv = objt_server(conn->target);
 
-               conn_id = quic_cid_alloc();
+               conn_id = quic_cid_alloc(QUIC_CID_SIDE_BE);
                if (!conn_id) {
                        TRACE_ERROR("error on CID allocation", QUIC_EV_CONN_NEW);
                        goto out;