From: Amaury Denoyelle <adenoyelle@haproxy.com>
Date: Tue, 11 Apr 2023 12:42:31 +0000 (+0200)
Subject: MINOR: quic: properly finalize thread rebinding
X-Git-Tag: v2.8-dev8~104
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=739de3f1191e01c10b3d7bb7ad79686156a05909;p=thirdparty%2Fhaproxy.git

MINOR: quic: properly finalize thread rebinding

When a quic_conn instance is rebinded on a new thread its tasks and
tasklet are destroyed and new ones created. Its socket is also migrated
to a new thread which stop reception on it.

To properly reactivate a quic_conn after rebind, wake up its tasks and
tasklet if they were active before thread rebind. Also reactivate
reading on the socket FD. These operations are implemented on a new
function qc_finalize_affinity_rebind().

This should be backported up to 2.7 after a period of observation.
---

diff --git a/include/haproxy/quic_conn-t.h b/include/haproxy/quic_conn-t.h
index 052b8c8c88..26fd60b8e4 100644
--- a/include/haproxy/quic_conn-t.h
+++ b/include/haproxy/quic_conn-t.h
@@ -629,6 +629,7 @@ enum qc_mux_state {
 #define QUIC_FL_CONN_HALF_OPEN_CNT_DECREMENTED   (1U << 11) /* The half-open connection counter was decremented */
 #define QUIC_FL_CONN_HANDSHAKE_SPEED_UP          (1U << 12) /* Handshake speeding up was done */
 #define QUIC_FL_CONN_ACK_TIMER_FIRED             (1U << 13) /* idle timer triggered for acknowledgements */
+#define QUIC_FL_CONN_IO_TO_REQUEUE               (1U << 14) /* IO handler must be requeued on new thread after connection migration */
 #define QUIC_FL_CONN_TO_KILL                     (1U << 24) /* Unusable connection, to be killed */
 #define QUIC_FL_CONN_TX_TP_RECEIVED              (1U << 25) /* Peer transport parameters have been received (used for the transmitting part) */
 #define QUIC_FL_CONN_FINALIZED                   (1U << 26) /* QUIC connection finalized (functional, ready to send/receive) */
diff --git a/include/haproxy/quic_conn.h b/include/haproxy/quic_conn.h
index a25d5efe81..3764dceb9c 100644
--- a/include/haproxy/quic_conn.h
+++ b/include/haproxy/quic_conn.h
@@ -701,6 +701,7 @@ static inline void quic_handle_stopping(void)
 }
 
 int qc_set_tid_affinity(struct quic_conn *qc, uint tid);
+void qc_finalize_affinity_rebind(struct quic_conn *qc);
 
 #endif /* USE_QUIC */
 #endif /* _HAPROXY_QUIC_CONN_H */
diff --git a/include/haproxy/quic_sock.h b/include/haproxy/quic_sock.h
index 89f7f158b7..acbe45e689 100644
--- a/include/haproxy/quic_sock.h
+++ b/include/haproxy/quic_sock.h
@@ -65,6 +65,7 @@ static inline char qc_test_fd(struct quic_conn *qc)
 void qc_alloc_fd(struct quic_conn *qc, const struct sockaddr_storage *src,
                  const struct sockaddr_storage *dst);
 void qc_release_fd(struct quic_conn *qc, int reinit);
+void qc_want_recv(struct quic_conn *qc);
 
 void quic_accept_push_qc(struct quic_conn *qc);
 
diff --git a/src/quic_conn.c b/src/quic_conn.c
index 31dc1b9793..b6f1467753 100644
--- a/src/quic_conn.c
+++ b/src/quic_conn.c
@@ -8481,6 +8481,8 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
 	}
 
 	/* Reinit IO tasklet. */
+	if (qc->wait_event.tasklet->state & TASK_IN_LIST)
+		qc->flags |= QUIC_FL_CONN_IO_TO_REQUEUE;
 	tasklet_kill(qc->wait_event.tasklet);
 	/* In most cases quic_conn_app_io_cb is used but for 0-RTT quic_conn_io_cb can be still activated. */
 	t3->process = qc->wait_event.tasklet->process;
@@ -8491,8 +8493,8 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
 
 	/* Rebind the connection FD. */
 	if (qc_test_fd(qc)) {
+		/* Reading is reactivated by the new thread. */
 		fd_migrate_on(qc->fd, new_tid);
-		/* TODO need to reactivate reading on the new thread. */
 	}
 
 	/* Remove conn from per-thread list instance. */
@@ -8524,6 +8526,29 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
 	return 1;
 }
 
+/* Must be called after qc_set_tid_affinity() on the new thread. */
+void qc_finalize_affinity_rebind(struct quic_conn *qc)
+{
+	TRACE_ENTER(QUIC_EV_CONN_SET_AFFINITY, qc);
+
+	/* Reactivate FD polling if connection socket is active. */
+	qc_want_recv(qc);
+
+	/* Reactivate timer task if needed. */
+	qc_set_timer(qc);
+
+	/* Idle timer task is always active. */
+	task_queue(qc->idle_timer_task);
+
+	/* Reactivate IO tasklet if needed. */
+	if (qc->flags & QUIC_FL_CONN_IO_TO_REQUEUE) {
+		tasklet_wakeup(qc->wait_event.tasklet);
+		qc->flags &= ~QUIC_FL_CONN_IO_TO_REQUEUE;
+	}
+
+	TRACE_LEAVE(QUIC_EV_CONN_SET_AFFINITY, qc);
+}
+
 /* appctx context used by "show quic" command */
 struct show_quic_ctx {
 	unsigned int epoch;
diff --git a/src/quic_sock.c b/src/quic_sock.c
index ac83b98833..d7d40982ed 100644
--- a/src/quic_sock.c
+++ b/src/quic_sock.c
@@ -878,6 +878,15 @@ void qc_release_fd(struct quic_conn *qc, int reinit)
 	}
 }
 
+/* Wrapper for fd_want_recv(). Safe even if connection does not used its owned
+ * socket.
+ */
+void qc_want_recv(struct quic_conn *qc)
+{
+	if (qc_test_fd(qc))
+		fd_want_recv(qc->fd);
+}
+
 /*********************** QUIC accept queue management ***********************/
 /* per-thread accept queues */
 struct quic_accept_queue *quic_accept_queues;