From: Amaury Denoyelle <adenoyelle@haproxy.com>
Date: Wed, 5 Apr 2023 16:17:51 +0000 (+0200)
Subject: MAJOR: quic: support thread balancing on accept
X-Git-Tag: v2.8-dev8~103
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1acbbca171a7e2d93108edc99945880110a42be7;p=thirdparty%2Fhaproxy.git

MAJOR: quic: support thread balancing on accept

Before this patch, QUIC protocol used a custom add_listener callback.
This was because a quic_conn instance was allocated before accept. Its
thread affinity was fixed and could not be changed after. The thread was
derived itself from the CID selected by the client which prevent an even
repartition of QUIC connections on multiple threads.

A series of patches was introduced with a lot of changes. The most
important ones :
* removal of affinity between an encoded CID and a thread
* possibility to rebind a quic_conn on a new thread

Thanks to this, it's possible to suppress the custom add_listener
callback. Accept is conducted for QUIC protocol as with the others. A
less loaded thread is selected on listener_accept() and the connection
stack is bind on it. This operation implies that quic_conn instance is
moved to the new thread using the set_affinity QUIC protocol callback.

To reactivate quic_conn instance after thread rebind,
qc_finalize_affinity_rebind() is called after accept on the new thread
by qc_xprt_start() through accept_queue_process() / session_accept_fd().

This should be backported up to 2.7 after a period of observation.
---

diff --git a/src/proto_quic.c b/src/proto_quic.c
index 0961fc2104..8c263772dc 100644
--- a/src/proto_quic.c
+++ b/src/proto_quic.c
@@ -62,11 +62,11 @@ struct quic_cid_tree *quic_cid_trees;
 
 DECLARE_STATIC_POOL(pool_head_quic_rxbuf, "quic_rxbuf", QUIC_RX_BUFSZ);
 
-static void quic_add_listener(struct protocol *proto, struct listener *listener);
 static int quic_bind_listener(struct listener *listener, char *errmsg, int errlen);
 static int quic_connect_server(struct connection *conn, int flags);
 static void quic_enable_listener(struct listener *listener);
 static void quic_disable_listener(struct listener *listener);
+static int quic_set_affinity(struct connection *conn, int new_tid);
 
 /* Note: must not be declared <const> as its list will be overwritten */
 struct protocol proto_quic4 = {
@@ -77,7 +77,7 @@ struct protocol proto_quic4 = {
 	.listen         = quic_bind_listener,
 	.enable         = quic_enable_listener,
 	.disable        = quic_disable_listener,
-	.add            = quic_add_listener,
+	.add            = default_add_listener,
 	.unbind         = default_unbind_listener,
 	.suspend        = default_suspend_listener,
 	.resume         = default_resume_listener,
@@ -85,6 +85,7 @@ struct protocol proto_quic4 = {
 	.get_src        = quic_sock_get_src,
 	.get_dst        = quic_sock_get_dst,
 	.connect        = quic_connect_server,
+	.set_affinity   = quic_set_affinity,
 
 	/* binding layer */
 	.rx_suspend     = udp_suspend_receiver,
@@ -117,7 +118,7 @@ struct protocol proto_quic6 = {
 	.listen         = quic_bind_listener,
 	.enable         = quic_enable_listener,
 	.disable        = quic_disable_listener,
-	.add            = quic_add_listener,
+	.add            = default_add_listener,
 	.unbind         = default_unbind_listener,
 	.suspend        = default_suspend_listener,
 	.resume         = default_resume_listener,
@@ -125,6 +126,7 @@ struct protocol proto_quic6 = {
 	.get_src        = quic_sock_get_src,
 	.get_dst        = quic_sock_get_dst,
 	.connect        = quic_connect_server,
+	.set_affinity   = quic_set_affinity,
 
 	/* binding layer */
 	.rx_suspend     = udp_suspend_receiver,
@@ -525,17 +527,6 @@ int quic_connect_server(struct connection *conn, int flags)
 	return SF_ERR_NONE;  /* connection is OK */
 }
 
-/* Add listener <listener> to protocol <proto>. Technically speaking we just
- * initialize a few entries which should be doable during quic_bind_listener().
- * The end of the initialization goes on with the default function.
- */
-static void quic_add_listener(struct protocol *proto, struct listener *listener)
-{
-	listener->rx.flags |= RX_F_LOCAL_ACCEPT;
-
-	default_add_listener(proto, listener);
-}
-
 /* Allocate the RX buffers for <l> listener.
  * Return 1 if succeeded, 0 if not.
  */
@@ -716,6 +707,12 @@ static void quic_disable_listener(struct listener *l)
 		fd_stop_recv(l->rx.fd);
 }
 
+static int quic_set_affinity(struct connection *conn, int new_tid)
+{
+	struct quic_conn *qc = conn->handle.qc;
+	return qc_set_tid_affinity(qc, new_tid);
+}
+
 static int quic_alloc_dghdlrs(void)
 {
 	int i;
diff --git a/src/xprt_quic.c b/src/xprt_quic.c
index 16432dba8e..be9b09c894 100644
--- a/src/xprt_quic.c
+++ b/src/xprt_quic.c
@@ -132,6 +132,8 @@ static int qc_xprt_start(struct connection *conn, void *ctx)
 	qc = conn->handle.qc;
 	TRACE_ENTER(QUIC_EV_CONN_NEW, qc);
 
+	qc_finalize_affinity_rebind(qc);
+
 	/* mux-quic can now be considered ready. */
 	qc->mux_state = QC_MUX_READY;