MINOR: quic: properly finalize thread rebinding

When a quic_conn instance is rebinded on a new thread its tasks and
tasklet are destroyed and new ones created. Its socket is also migrated
to a new thread which stop reception on it.

To properly reactivate a quic_conn after rebind, wake up its tasks and
tasklet if they were active before thread rebind. Also reactivate
reading on the socket FD. These operations are implemented on a new
function qc_finalize_affinity_rebind().

This should be backported up to 2.7 after a period of observation.

diff --git a/include/haproxy/quic_conn-t.h b/include/haproxy/quic_conn-t.h
index 052b8c8c88538cb16bb5c7f699065e04a3f3b78f..26fd60b8e40c59e1079c5938ed94ca51a1370784 100644 (file)
--- a/include/haproxy/quic_conn-t.h
+++ b/include/haproxy/quic_conn-t.h
@@ -629,6 +629,7 @@ enum qc_mux_state {
 #define QUIC_FL_CONN_HALF_OPEN_CNT_DECREMENTED   (1U << 11) /* The half-open connection counter was decremented */
 #define QUIC_FL_CONN_HANDSHAKE_SPEED_UP          (1U << 12) /* Handshake speeding up was done */
 #define QUIC_FL_CONN_ACK_TIMER_FIRED             (1U << 13) /* idle timer triggered for acknowledgements */
+#define QUIC_FL_CONN_IO_TO_REQUEUE               (1U << 14) /* IO handler must be requeued on new thread after connection migration */
 #define QUIC_FL_CONN_TO_KILL                     (1U << 24) /* Unusable connection, to be killed */
 #define QUIC_FL_CONN_TX_TP_RECEIVED              (1U << 25) /* Peer transport parameters have been received (used for the transmitting part) */
 #define QUIC_FL_CONN_FINALIZED                   (1U << 26) /* QUIC connection finalized (functional, ready to send/receive) */

diff --git a/include/haproxy/quic_conn.h b/include/haproxy/quic_conn.h
index a25d5efe818ad1308f4a74efc22a9519c9966642..3764dceb9cfe0b3207f5f7da1e4bc81ea94c801c 100644 (file)
--- a/include/haproxy/quic_conn.h
+++ b/include/haproxy/quic_conn.h
@@ -701,6 +701,7 @@ static inline void quic_handle_stopping(void)
 }
 
 int qc_set_tid_affinity(struct quic_conn *qc, uint tid);
+void qc_finalize_affinity_rebind(struct quic_conn *qc);
 
 #endif /* USE_QUIC */
 #endif /* _HAPROXY_QUIC_CONN_H */

diff --git a/include/haproxy/quic_sock.h b/include/haproxy/quic_sock.h
index 89f7f158b7ba8ef9fe8adca0632e0c3cb0de3694..acbe45e689ad2ee4948af1af73a4bb1c86d7d0e9 100644 (file)
--- a/include/haproxy/quic_sock.h
+++ b/include/haproxy/quic_sock.h
@@ -65,6 +65,7 @@ static inline char qc_test_fd(struct quic_conn *qc)
 void qc_alloc_fd(struct quic_conn *qc, const struct sockaddr_storage *src,
                  const struct sockaddr_storage *dst);
 void qc_release_fd(struct quic_conn *qc, int reinit);
+void qc_want_recv(struct quic_conn *qc);
 
 void quic_accept_push_qc(struct quic_conn *qc);
 

diff --git a/src/quic_conn.c b/src/quic_conn.c
index 31dc1b9793c57589bbac8783f4a456bf5a781a31..b6f1467753424bed620151a6a52b583a36a6ac76 100644 (file)
--- a/src/quic_conn.c
+++ b/src/quic_conn.c
@@ -8481,6 +8481,8 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
        }
 
        /* Reinit IO tasklet. */
+       if (qc->wait_event.tasklet->state & TASK_IN_LIST)
+               qc->flags |= QUIC_FL_CONN_IO_TO_REQUEUE;
        tasklet_kill(qc->wait_event.tasklet);
        /* In most cases quic_conn_app_io_cb is used but for 0-RTT quic_conn_io_cb can be still activated. */
        t3->process = qc->wait_event.tasklet->process;
@@ -8491,8 +8493,8 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
 
        /* Rebind the connection FD. */
        if (qc_test_fd(qc)) {
+               /* Reading is reactivated by the new thread. */
                fd_migrate_on(qc->fd, new_tid);
-               /* TODO need to reactivate reading on the new thread. */
        }
 
        /* Remove conn from per-thread list instance. */
@@ -8524,6 +8526,29 @@ int qc_set_tid_affinity(struct quic_conn *qc, uint new_tid)
        return 1;
 }
 
+/* Must be called after qc_set_tid_affinity() on the new thread. */
+void qc_finalize_affinity_rebind(struct quic_conn *qc)
+{
+       TRACE_ENTER(QUIC_EV_CONN_SET_AFFINITY, qc);
+
+       /* Reactivate FD polling if connection socket is active. */
+       qc_want_recv(qc);
+
+       /* Reactivate timer task if needed. */
+       qc_set_timer(qc);
+
+       /* Idle timer task is always active. */
+       task_queue(qc->idle_timer_task);
+
+       /* Reactivate IO tasklet if needed. */
+       if (qc->flags & QUIC_FL_CONN_IO_TO_REQUEUE) {
+               tasklet_wakeup(qc->wait_event.tasklet);
+               qc->flags &= ~QUIC_FL_CONN_IO_TO_REQUEUE;
+       }
+
+       TRACE_LEAVE(QUIC_EV_CONN_SET_AFFINITY, qc);
+}
+
 /* appctx context used by "show quic" command */
 struct show_quic_ctx {
        unsigned int epoch;

diff --git a/src/quic_sock.c b/src/quic_sock.c
index ac83b988333f1bb6afa5f9f96ac6ead04cb8cfe4..d7d40982edde515f83f54334c57b993948b2da24 100644 (file)
--- a/src/quic_sock.c
+++ b/src/quic_sock.c
@@ -878,6 +878,15 @@ void qc_release_fd(struct quic_conn *qc, int reinit)
        }
 }
 
+/* Wrapper for fd_want_recv(). Safe even if connection does not used its owned
+ * socket.
+ */
+void qc_want_recv(struct quic_conn *qc)
+{
+       if (qc_test_fd(qc))
+               fd_want_recv(qc->fd);
+}
+
 /*********************** QUIC accept queue management ***********************/
 /* per-thread accept queues */
 struct quic_accept_queue *quic_accept_queues;