MEDIUM: mux-quic: implement http-request timeout

Implement http-request timeout for QUIC MUX. It is used when the
connection is opened and is triggered if no HTTP request is received in
time. By HTTP request we mean at least a QUIC stream with a full header
section. Then qcs instance is attached to a sedesc and upper layer is
then responsible to wait for the rest of the request.

This timeout is also used when new QUIC streams are opened during the
connection lifetime to wait for full HTTP request on them. As it's
possible to demux multiple streams in parallel with QUIC, each waiting
stream is registered in a list <opening_list> stored in qcc with <start>
as timestamp in qcs for the stream opening. Once a qcs is attached to a
sedesc, it is removed from <opening_list>. When refreshing MUX timeout,
if <opening_list> is not empty, the first waiting stream is used to set
MUX timeout.

This is efficient as streams are stored in the list in their creation
order so CPU usage is minimal. Also, the size of the list is
automatically restricted by flow control limitation so it should not
grow too much.

Streams are insert in <opening_list> by application protocol layer. This
is because only application protocol can differentiate streams for HTTP
messaging from internal usage. A function qcs_wait_http_req() has been
added to register a request stream by app layer. QUIC MUX can then
remove it from the list in qc_attach_sc().

As a side-note, it was necessary to implement attach qcc_app_ops
callback on hq-interop module to be able to insert a stream in waiting
list. Without this, a BUG_ON statement would be triggered when trying to
remove the stream on sedesc attach. This is to ensure that every
requests streams are registered for http-request timeout.

MUX timeout is explicitely refreshed on MAX_STREAM_DATA and STOP_SENDING
frame parsing to schedule http-request timeout if a new stream has been
instantiated. It was already done on STREAM parsing due to a previous
patch.

diff --git a/include/haproxy/mux_quic-t.h b/include/haproxy/mux_quic-t.h
index 38d2af46aa38a43d65d9bb6a7d41b1634eb2c20a..c555ace3c1d7c5b4695610376b943dd53e0db71a 100644 (file)
--- a/include/haproxy/mux_quic-t.h
+++ b/include/haproxy/mux_quic-t.h
@@ -96,6 +96,7 @@ struct qcc {
 
        /* haproxy timeout management */
        struct task *task;
+       struct list opening_list; /* list of not already attached streams (http-request timeout) */
        int timeout;
        int idle_start; /* base time for http-keep-alive timeout */
 
@@ -165,11 +166,14 @@ struct qcs {
        struct qc_stream_desc *stream;
 
        struct list el; /* element of qcc.send_retry_list */
+       struct list el_opening; /* element of qcc.opening_list */
 
        struct wait_event wait_event;
        struct wait_event *subs;
 
        uint64_t err; /* error code to transmit via RESET_STREAM */
+
+       int start; /* base timestamp for http-request timeout */
 };
 
 /* QUIC application layer operations */

diff --git a/include/haproxy/mux_quic.h b/include/haproxy/mux_quic.h
index bf0a18e4721a6290110970faaf3d8bb62c5a3d1e..d18f1a0ebce009b81b6f0d276b477dcf3a5d61db 100644 (file)
--- a/include/haproxy/mux_quic.h
+++ b/include/haproxy/mux_quic.h
@@ -115,9 +115,36 @@ static inline struct stconn *qc_attach_sc(struct qcs *qcs, struct buffer *buf)
        sess->t_handshake = 0;
        sess->t_idle = 0;
 
+       /* A stream must have been registered for HTTP wait before attaching
+        * it to sedesc. See <qcs_wait_http_req> for more info.
+        */
+       BUG_ON_HOT(!LIST_INLIST(&qcs->el_opening));
+       LIST_DELETE(&qcs->el_opening);
+
        return qcs->sd->sc;
 }
 
+/* Register <qcs> stream for http-request timeout. If the stream is not yet
+ * attached in the configured delay, qcc timeout task will be triggered. This
+ * means the full header section was not received in time.
+ *
+ * This function should be called by the application protocol layer on request
+ * streams initialization.
+ */
+static inline void qcs_wait_http_req(struct qcs *qcs)
+{
+       struct qcc *qcc = qcs->qcc;
+
+       /* A stream cannot be registered several times. */
+       BUG_ON_HOT(tick_isset(qcs->start));
+       qcs->start = now_ms;
+
+       /* qcc.opening_list size is limited by flow-control so no custom
+        * restriction is needed here.
+        */
+       LIST_APPEND(&qcc->opening_list, &qcs->el_opening);
+}
+
 #endif /* USE_QUIC */
 
 #endif /* _HAPROXY_MUX_QUIC_H */

diff --git a/src/h3.c b/src/h3.c
index f00281e8d6c9c5e51248c57299e8ee1e7a90e98c..688f9030a8413915beba7d42007bef2752799e84 100644 (file)
--- a/src/h3.c
+++ b/src/h3.c
@@ -1080,6 +1080,7 @@ static int h3_attach(struct qcs *qcs, void *conn_ctx)
        if (quic_stream_is_bidi(qcs->id)) {
                h3s->type = H3S_T_REQ;
                h3s->st_req = H3S_ST_REQ_BEFORE;
+               qcs_wait_http_req(qcs);
        }
        else {
                /* stream type must be decoded for unidirectional streams */

diff --git a/src/hq_interop.c b/src/hq_interop.c
index f2933d6b5747aee886076a168592cda444179ce1..be0287f6f53b3a093369a68d09531cbbfcbee344 100644 (file)
--- a/src/hq_interop.c
+++ b/src/hq_interop.c
@@ -164,7 +164,14 @@ static size_t hq_interop_snd_buf(struct stconn *sc, struct buffer *buf,
        return total;
 }
 
+static int hq_interop_attach(struct qcs *qcs, void *conn_ctx)
+{
+       qcs_wait_http_req(qcs);
+       return 0;
+}
+
 const struct qcc_app_ops hq_interop_ops = {
        .decode_qcs = hq_interop_decode_qcs,
        .snd_buf    = hq_interop_snd_buf,
+       .attach     = hq_interop_attach,
 };

diff --git a/src/mux_quic.c b/src/mux_quic.c
index df6e304d43416edf84f827f8a1b0d2b46f8e2c02..e0d074680d914ff620771030fe14b5b0849e6cd1 100644 (file)
--- a/src/mux_quic.c
+++ b/src/mux_quic.c
@@ -130,6 +130,12 @@ static struct qcs *qcs_new(struct qcc *qcc, uint64_t id, enum qcs_type type)
        qcs->st = QC_SS_IDLE;
        qcs->ctx = NULL;
 
+       /* App callback attach may register the stream for http-request wait.
+        * These fields must be initialed before.
+        */
+       LIST_INIT(&qcs->el_opening);
+       qcs->start = TICK_ETERNITY;
+
        /* Allocate transport layer stream descriptor. Only needed for TX. */
        if (!quic_stream_is_uni(id) || !quic_stream_is_remote(qcc, id)) {
                struct quic_conn *qc = qcc->conn->handle.qc;
@@ -302,18 +308,14 @@ static void qcc_refresh_timeout(struct qcc *qcc)
         * it with global close_spread delay applied.
         */
 
-       /* TODO implement specific timeouts
-        * - http-requset for waiting on incomplete streams
-        * - client-fin for graceful shutdown
-        */
+       /* TODO implement client/server-fin timeout for graceful shutdown */
 
        /* Frontend timeout management
         * - detached streams with data left to send -> default timeout
+        * - stream waiting on incomplete request or no stream yet activated -> timeout http-request
         * - idle after stream processing -> timeout http-keep-alive
         */
        if (!conn_is_back(qcc->conn)) {
-               int timeout;
-
                if (qcc->nb_hreq) {
                        TRACE_DEVEL("one or more requests still in progress", QMUX_EV_QCC_WAKE, qcc->conn);
                        qcc->task->expire = tick_add_ifset(now_ms, qcc->timeout);
@@ -321,12 +323,29 @@ static void qcc_refresh_timeout(struct qcc *qcc)
                        goto leave;
                }
 
-               /* Use http-request timeout if keep-alive timeout not set */
-               timeout = tick_isset(px->timeout.httpka) ?
-                           px->timeout.httpka : px->timeout.httpreq;
+               if (!LIST_ISEMPTY(&qcc->opening_list) || unlikely(!qcc->largest_bidi_r)) {
+                       int timeout = px->timeout.httpreq;
+                       struct qcs *qcs = NULL;
+                       int base_time;
 
-               TRACE_DEVEL("at least one request achieved but none currently in progress", QMUX_EV_QCC_WAKE, qcc->conn);
-               qcc->task->expire = tick_add_ifset(qcc->idle_start, timeout);
+                       /* Use start time of first stream waiting on HTTP or
+                        * qcc idle if no stream not yet used.
+                        */
+                       if (likely(!LIST_ISEMPTY(&qcc->opening_list)))
+                               qcs = LIST_ELEM(qcc->opening_list.n, struct qcs *, el_opening);
+                       base_time = qcs ? qcs->start : qcc->idle_start;
+
+                       TRACE_DEVEL("waiting on http request", QMUX_EV_QCC_WAKE, qcc->conn, qcs);
+                       qcc->task->expire = tick_add_ifset(base_time, timeout);
+               }
+               else {
+                       /* Use http-request timeout if keep-alive timeout not set */
+                       int timeout = tick_isset(px->timeout.httpka) ?
+                                       px->timeout.httpka : px->timeout.httpreq;
+
+                       TRACE_DEVEL("at least one request achieved but none currently in progress", QMUX_EV_QCC_WAKE, qcc->conn);
+                       qcc->task->expire = tick_add_ifset(qcc->idle_start, timeout);
+               }
        }
 
        /* fallback to default timeout if frontend specific undefined or for
@@ -1015,6 +1034,9 @@ int qcc_recv_max_stream_data(struct qcc *qcc, uint64_t id, uint64_t max)
                }
        }
 
+       if (qcc_may_expire(qcc) && !qcc->nb_hreq)
+               qcc_refresh_timeout(qcc);
+
        TRACE_LEAVE(QMUX_EV_QCC_RECV, qcc->conn);
        return 0;
 }
@@ -1064,6 +1086,9 @@ int qcc_recv_stop_sending(struct qcc *qcc, uint64_t id, uint64_t err)
        TRACE_DEVEL("receiving STOP_SENDING on stream", QMUX_EV_QCC_RECV|QMUX_EV_QCS_RECV, qcc->conn, qcs);
        qcc_reset_stream(qcs, err);
 
+       if (qcc_may_expire(qcc) && !qcc->nb_hreq)
+               qcc_refresh_timeout(qcc);
+
  out:
        TRACE_LEAVE(QMUX_EV_QCC_RECV, qcc->conn);
        return 0;
@@ -1931,6 +1956,7 @@ static int qc_init(struct connection *conn, struct proxy *prx,
                qcc->task->expire = tick_add(now_ms, qcc->timeout);
        }
        qcc_reset_idle_start(qcc);
+       LIST_INIT(&qcc->opening_list);
 
        if (!conn_is_back(conn)) {
                if (!LIST_INLIST(&conn->stopping_list)) {