MEDIUM: mux-quic: properly implement soft-stop

Properly implement support for haproxy soft-stop on QUIC MUX. This code
is similar to H2 MUX :

* on timeout refresh, if stop-stop in progress, schedule the timeout to
  expire with regards to the close-spread-end window.

* after input/output processing, if soft-stop in progress, shutdown the
  connection. This is randomly spread by close-spread-end window. In the
  case of H3 connection, a GOAWAY is emitted and the connection is kept
  until all data are sent for opened streams. If the client tries to use
  new streams, they are rejected in conformance with the GOAWAY
  specification.

This ensures that MUX is able to forward all content properly before
closing the connection. The lower quic-conn layer is then responsible
for retransmission and should be closed when all data are acknowledged.
This will be implemented in the next commit to fully support soft-stop
for QUIC connections.

This should be backported up to 2.7.

diff --git a/src/mux_quic.c b/src/mux_quic.c
index 4b3af328b7a87ffce870fa37e8fccb593507a7a2..a78f8d4a483eb58447361862090ef89304e3fd61 100644 (file)
--- a/src/mux_quic.c
+++ b/src/mux_quic.c
@@ -14,6 +14,7 @@
 #include <haproxy/qmux_trace.h>
 #include <haproxy/quic_conn.h>
 #include <haproxy/quic_frame.h>
+#include <haproxy/quic_sock.h>
 #include <haproxy/quic_stream.h>
 #include <haproxy/quic_tp-t.h>
 #include <haproxy/ssl_sock-t.h>
@@ -258,15 +259,14 @@ static void qcc_refresh_timeout(struct qcc *qcc)
                goto leave;
        }
 
-       /* TODO if connection is idle on frontend and proxy is disabled, remove
-        * it with global close_spread delay applied.
-        */
-
        /* Frontend timeout management
         * - shutdown done -> timeout client-fin
         * - detached streams with data left to send -> default timeout
         * - stream waiting on incomplete request or no stream yet activated -> timeout http-request
         * - idle after stream processing -> timeout http-keep-alive
+        *
+        * If proxy stop-stop in progress, immediate or spread close will be
+        * processed if shutdown already one or connection is idle.
         */
        if (!conn_is_back(qcc->conn)) {
                if (qcc->nb_hreq && !(qcc->flags & QC_CF_APP_SHUT)) {
@@ -305,6 +305,41 @@ static void qcc_refresh_timeout(struct qcc *qcc)
                                TRACE_DEVEL("at least one request achieved but none currently in progress", QMUX_EV_QCC_WAKE, qcc->conn);
                                qcc->task->expire = tick_add_ifset(qcc->idle_start, timeout);
                        }
+
+                       /* If proxy soft-stop in progress and connection is
+                        * inactive, close the connection immediately. If a
+                        * close-spread-time is configured, randomly spread the
+                        * timer over a closing window.
+                        */
+                       if ((qcc->proxy->flags & (PR_FL_DISABLED|PR_FL_STOPPED)) &&
+                           !(global.tune.options & GTUNE_DISABLE_ACTIVE_CLOSE)) {
+
+                               /* Wake timeout task immediately if window already expired. */
+                               int remaining_window = tick_isset(global.close_spread_end) ?
+                                 tick_remain(now_ms, global.close_spread_end) : 0;
+
+                               TRACE_DEVEL("proxy disabled, prepare connection soft-stop", QMUX_EV_QCC_WAKE, qcc->conn);
+                               if (remaining_window) {
+                                       /* We don't need to reset the expire if it would
+                                        * already happen before the close window end.
+                                        */
+                                       if (!tick_isset(qcc->task->expire) ||
+                                           tick_is_le(global.close_spread_end, qcc->task->expire)) {
+                                               /* Set an expire value shorter than the current value
+                                                * because the close spread window end comes earlier.
+                                                */
+                                               qcc->task->expire = tick_add(now_ms,
+                                                                            statistical_prng_range(remaining_window));
+                                       }
+                               }
+                               else {
+                                       /* We are past the soft close window end, wake the timeout
+                                        * task up immediately.
+                                        */
+                                       qcc->task->expire = now_ms;
+                                       task_wakeup(qcc->task, TASK_WOKEN_TIMER);
+                               }
+                       }
                }
        }
 
@@ -1967,6 +2002,46 @@ static int qc_process(struct qcc *qcc)
 {
        qc_purge_streams(qcc);
 
+       /* Check if a soft-stop is in progress.
+        *
+        * TODO this is relevant for frontend connections only.
+        */
+       if (unlikely(qcc->proxy->flags & (PR_FL_DISABLED|PR_FL_STOPPED))) {
+               int close = 1;
+
+               /* If using listener socket, soft-stop is not supported. The
+                * connection must be closed immediately.
+                */
+               if (!qc_test_fd(qcc->conn->handle.qc)) {
+                       TRACE_DEVEL("proxy disabled with listener socket, closing connection", QMUX_EV_QCC_WAKE, qcc->conn);
+                       qcc->conn->flags |= (CO_FL_SOCK_RD_SH|CO_FL_SOCK_WR_SH);
+                       qc_send(qcc);
+                       goto out;
+               }
+
+               TRACE_DEVEL("proxy disabled, prepare connection soft-stop", QMUX_EV_QCC_WAKE, qcc->conn);
+
+               /* If a close-spread-time option is set, we want to avoid
+                * closing all the active HTTP3 connections at once so we add a
+                * random factor that will spread the closing.
+                */
+               if (tick_isset(global.close_spread_end)) {
+                       int remaining_window = tick_remain(now_ms, global.close_spread_end);
+                       if (remaining_window) {
+                               /* This should increase the closing rate the
+                                * further along the window we are. */
+                               close = (remaining_window <= statistical_prng_range(global.close_spread_time));
+                       }
+               }
+               else if (global.tune.options & GTUNE_DISABLE_ACTIVE_CLOSE) {
+                       close = 0; /* let the client close his connection himself */
+               }
+
+               if (close)
+                       qc_shutdown(qcc);
+       }
+
+ out:
        if (qcc_is_dead(qcc))
                return 1;
 
@@ -2457,21 +2532,9 @@ static int qc_wake_some_streams(struct qcc *qcc)
 static int qc_wake(struct connection *conn)
 {
        struct qcc *qcc = conn->ctx;
-       struct proxy *prx = conn->handle.qc->li->bind_conf->frontend;
 
        TRACE_ENTER(QMUX_EV_QCC_WAKE, conn);
 
-       /* Check if a soft-stop is in progress.
-        *
-        * TODO this is relevant for frontend connections only.
-        *
-        * TODO Client should be notified with a H3 GOAWAY and then a
-        * CONNECTION_CLOSE. However, quic-conn uses the listener socket for
-        * sending which at this stage is already closed.
-        */
-       if (unlikely(prx->flags & (PR_FL_DISABLED|PR_FL_STOPPED)))
-               qcc->conn->flags |= (CO_FL_SOCK_RD_SH|CO_FL_SOCK_WR_SH);
-
        if (conn->handle.qc->flags & QUIC_FL_CONN_NOTIFY_CLOSE)
                qcc->conn->flags |= (CO_FL_SOCK_RD_SH|CO_FL_SOCK_WR_SH);