From: Libor Peltan Date: Wed, 1 Nov 2023 13:32:37 +0000 (+0100) Subject: quic: implemented sending replies from sweep... X-Git-Tag: v3.4.0~246^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e683cfb49bd1862b1121bb0b555ff9bdaae2fd32;p=thirdparty%2Fknot-dns.git quic: implemented sending replies from sweep... explicit connection close when too many connections; re-send of lost packets --- diff --git a/src/knot/server/quic-handler.c b/src/knot/server/quic-handler.c index 9526dc299f..0dfa1c4eed 100644 --- a/src/knot/server/quic-handler.c +++ b/src/knot/server/quic-handler.c @@ -26,6 +26,13 @@ #include "libknot/quic/quic.h" #include "libknot/xdp/tcp_iobuf.h" +#define SWEEP_BUF_SIZE 4096 + +typedef union { + struct cmsghdr cmsg; + uint8_t buf[CMSG_SPACE(sizeof(struct in6_pktinfo))]; +} cmsg_pktinfo_t; + static void quic_log_cb(const char *line) { log_fmt(LOG_DEBUG, LOG_SOURCE_QUIC, "QUIC, %s", line); @@ -128,12 +135,77 @@ void quic_reconfigure_table(knot_quic_table_t *table) } } -void quic_sweep_table(knot_quic_table_t *table, knot_sweep_stats_t *stats) +int uq_alloc_sweep(struct knot_quic_reply *r) { - if (table != NULL) { - knot_quic_table_sweep(table, stats); - log_swept(stats, false); + r->out_payload->iov_len = SWEEP_BUF_SIZE; + return KNOT_EOK; +} + +int uq_send_sweep(struct knot_quic_reply *r) +{ + int fd = (int)(size_t)r->sock; + + cmsg_pktinfo_t cmsg = { 0 }; + if (r->ip_loc->ss_family == AF_INET6) { + cmsg.cmsg.cmsg_level = IPPROTO_IPV6; + cmsg.cmsg.cmsg_type = IPV6_PKTINFO; + cmsg.cmsg.cmsg_len = CMSG_LEN(sizeof(struct in6_pktinfo)); + memcpy(&((struct in6_pktinfo *)CMSG_DATA(&cmsg.cmsg))->ipi6_addr, + &((const struct sockaddr_in6 *)r->ip_loc)->sin6_addr, + sizeof(struct in6_addr)); + } else { + cmsg.cmsg.cmsg_level = IPPROTO_IP; + cmsg.cmsg.cmsg_type = IP_PKTINFO; + cmsg.cmsg.cmsg_len = CMSG_LEN(sizeof(struct in_pktinfo)); + memcpy(&((struct in_pktinfo *)CMSG_DATA(&cmsg.cmsg))->ipi_addr, + &((const struct sockaddr_in *)r->ip_loc)->sin_addr, + sizeof(struct in_addr)); + } // this only says "send it with given outgoing IP address" + + assert(r->ip_rem != NULL); + struct msghdr msg = { + .msg_iov = r->out_payload, + .msg_iovlen = 1, + .msg_name = (void *)r->ip_rem, + .msg_namelen = r->ip_rem->ss_family == AF_INET6 ? sizeof(struct sockaddr_in6) : + sizeof(struct sockaddr_in), + .msg_control = &cmsg, + .msg_controllen = sizeof(cmsg), + }; + + int ret = net_msg_send(fd, &msg, 0); + if (ret < 0) { + return ret; + } else if (ret == r->out_payload->iov_len) { + return KNOT_EOK; + } else { + return KNOT_EAGAIN; + } +} + +void uq_free_sweep(struct knot_quic_reply *r) +{ + (void)r; +} + +void quic_sweep_table(knot_quic_table_t *table, knot_sweep_stats_t *stats, int fd) +{ + if (table == NULL) { + return; } + + uint8_t sendbuf[SWEEP_BUF_SIZE]; + struct iovec r_iov = { .iov_base = sendbuf }; + knot_quic_reply_t r = { + .sock = (void *)(size_t)fd, + .out_payload = &r_iov, + .alloc_reply = uq_alloc_sweep, + .send_reply = uq_send_sweep, + .free_reply = uq_free_sweep, + }; + + knot_quic_table_sweep(table, &r, stats); + log_swept(stats, false); } void quic_unmake_table(knot_quic_table_t *table) diff --git a/src/knot/server/quic-handler.h b/src/knot/server/quic-handler.h index 12b2b47c12..e491f0e0dc 100644 --- a/src/knot/server/quic-handler.h +++ b/src/knot/server/quic-handler.h @@ -59,10 +59,13 @@ void quic_reconfigure_table(knot_quic_table_t *table); /*! * \brief Sweep idle or excessive QUIC connections. * + * \note This function cannot be used with XDP. + * * \param table QUIC connection table. * \param stats Statistics to be updated. + * \param fd Standard socket descriptor to send sweep replies through. */ -void quic_sweep_table(knot_quic_table_t *table, knot_sweep_stats_t *stats); +void quic_sweep_table(knot_quic_table_t *table, knot_sweep_stats_t *stats, int fd); /*! * \brief Deallocate QUIC connecton table. diff --git a/src/knot/server/udp-handler.c b/src/knot/server/udp-handler.c index 6ae8b02ff2..3b06fa983d 100644 --- a/src/knot/server/udp-handler.c +++ b/src/knot/server/udp-handler.c @@ -165,10 +165,11 @@ void cmsg_handle(const struct msghdr *rx, struct msghdr *tx, } } -static void udp_sweep(udp_context_t *ctx, _unused_ void *d) +static void udp_sweep(udp_context_t *ctx, void *d) { #ifdef ENABLE_QUIC - quic_sweep_table(ctx->quic_table, &ctx->quic_closed); + int fd = *(int *)d; // NOTE both udp_msg_ctx_t and udp_mmsg_ctx_t have 'fd' as first item + quic_sweep_table(ctx->quic_table, &ctx->quic_closed, fd); quic_reconfigure_table(ctx->quic_table); #endif // ENABLE_QUIC } diff --git a/src/knot/server/xdp-handler.c b/src/knot/server/xdp-handler.c index 1b36558a4c..a81e61476e 100644 --- a/src/knot/server/xdp-handler.c +++ b/src/knot/server/xdp-handler.c @@ -342,7 +342,8 @@ void xdp_handle_send(xdp_handle_ctx_t *ctx) void xdp_handle_sweep(xdp_handle_ctx_t *ctx) { #ifdef ENABLE_QUIC - quic_sweep_table(ctx->quic_table, &ctx->quic_closed); + knot_quic_table_sweep(ctx->quic_table, NULL, &ctx->quic_closed); + log_swept(&ctx->quic_closed, false); quic_reconfigure_table(ctx->quic_table); #endif // ENABLE_QUIC diff --git a/src/libknot/quic/quic.c b/src/libknot/quic/quic.c index 05a2271cfd..40d829250f 100644 --- a/src/libknot/quic/quic.c +++ b/src/libknot/quic/quic.c @@ -53,6 +53,7 @@ #define QUIC_SEND_RETRY NGTCP2_ERR_RETRY #define QUIC_SEND_STATELESS_RESET (-NGTCP2_STATELESS_RESET_TOKENLEN) #define QUIC_SEND_CONN_CLOSE (-KNOT_QUIC_HANDLE_RET_CLOSE) +#define QUIC_SEND_EXCESSIVE_LOAD (-KNOT_QUIC_ERR_EXCESSIVE_LOAD) #define TLS_CALLBACK_ERR (-1) @@ -1054,9 +1055,17 @@ static int send_stream(knot_quic_table_t *quic_table, knot_quic_reply_t *rpl, ngtcp2_vec vec = { .base = data, .len = len }; ngtcp2_pkt_info pi = { 0 }; - ret = ngtcp2_conn_writev_stream(relay->conn, NULL, &pi, rpl->out_payload->iov_base, - rpl->out_payload->iov_len, sent, fl, stream_id, - &vec, (stream_id >= 0 ? 1 : 0), get_timestamp()); + struct sockaddr_storage path_loc = { 0 }, path_rem = { 0 }; + ngtcp2_path path = { .local = { .addr = (struct sockaddr *)&path_loc, .addrlen = sizeof(path_loc) }, + .remote = { .addr = (struct sockaddr *)&path_rem, .addrlen = sizeof(path_rem) }, + .user_data = NULL }; + bool find_path = (rpl->ip_rem == NULL); + assert(find_path == (bool)(rpl->ip_loc == NULL)); + + ret = ngtcp2_conn_writev_stream(relay->conn, find_path ? &path : NULL, &pi, + rpl->out_payload->iov_base, rpl->out_payload->iov_len, + sent, fl, stream_id, &vec, + (stream_id >= 0 ? 1 : 0), get_timestamp()); if (ret <= 0) { rpl->free_reply(rpl); return ret; @@ -1067,7 +1076,15 @@ static int send_stream(knot_quic_table_t *quic_table, knot_quic_reply_t *rpl, rpl->out_payload->iov_len = ret; rpl->ecn = pi.ecn; + if (find_path) { + rpl->ip_loc = &path_loc; + rpl->ip_rem = &path_rem; + } ret = rpl->send_reply(rpl); + if (find_path) { + rpl->ip_loc = NULL; + rpl->ip_rem = NULL; + } if (ret == KNOT_EOK) { return 1; } @@ -1086,10 +1103,11 @@ static int send_special(knot_quic_table_t *quic_table, knot_quic_reply_t *rpl, ngtcp2_version_cid decoded_cids = { 0 }; ngtcp2_cid scid = { 0 }, dcid = { 0 }; - int dvc_ret = ngtcp2_pkt_decode_version_cid(&decoded_cids, - rpl->in_payload->iov_base, - rpl->in_payload->iov_len, - SERVER_DEFAULT_SCIDLEN); + int dvc_ret = rpl->in_payload == NULL ? NGTCP2_ERR_FATAL : + ngtcp2_pkt_decode_version_cid(&decoded_cids, + rpl->in_payload->iov_base, + rpl->in_payload->iov_len, + SERVER_DEFAULT_SCIDLEN); uint8_t rnd = 0; dnssec_random_buffer(&rnd, sizeof(rnd)); @@ -1103,6 +1121,14 @@ static int send_special(knot_quic_table_t *quic_table, knot_quic_reply_t *rpl, ngtcp2_ccerr_default(&ccerr); ngtcp2_pkt_info pi = { 0 }; + struct sockaddr_storage path_loc = { 0 }, path_rem = { 0 }; + ngtcp2_path path = { .local = { .addr = (struct sockaddr *)&path_loc, .addrlen = sizeof(path_loc) }, + .remote = { .addr = (struct sockaddr *)&path_rem, .addrlen = sizeof(path_rem) }, + .user_data = NULL }; + bool find_path = (rpl->ip_rem == NULL); + assert(find_path == (bool)(rpl->ip_loc == NULL)); + assert(!find_path || rpl->handle_ret == -QUIC_SEND_EXCESSIVE_LOAD); + switch (rpl->handle_ret) { case -QUIC_SEND_VERSION_NEGOTIATION: if (dvc_ret != NGTCP2_ERR_VERSION_NEGOTIATION) { @@ -1145,7 +1171,16 @@ static int send_special(knot_quic_table_t *quic_table, knot_quic_reply_t *rpl, break; case -QUIC_SEND_CONN_CLOSE: ret = ngtcp2_conn_write_connection_close( - relay->conn, NULL, &pi, rpl->out_payload->iov_base, rpl->out_payload->iov_len, &ccerr, now + relay->conn, NULL, &pi, rpl->out_payload->iov_base, + rpl->out_payload->iov_len, &ccerr, now + ); + break; + case -QUIC_SEND_EXCESSIVE_LOAD: + ccerr.type = NGTCP2_CCERR_TYPE_APPLICATION; + ccerr.error_code = KNOT_QUIC_ERR_EXCESSIVE_LOAD; + ret = ngtcp2_conn_write_connection_close( + relay->conn, find_path ? &path : NULL, &pi, rpl->out_payload->iov_base, + rpl->out_payload->iov_len, &ccerr, now ); break; default: @@ -1158,7 +1193,15 @@ static int send_special(knot_quic_table_t *quic_table, knot_quic_reply_t *rpl, } else { rpl->out_payload->iov_len = ret; rpl->ecn = pi.ecn; + if (find_path) { + rpl->ip_loc = &path_loc; + rpl->ip_rem = &path_rem; + } ret = rpl->send_reply(rpl); + if (find_path) { + rpl->ip_loc = NULL; + rpl->ip_rem = NULL; + } } return ret; } diff --git a/src/libknot/quic/quic.h b/src/libknot/quic/quic.h index 4163a5097a..3f6b6bf3e7 100644 --- a/src/libknot/quic/quic.h +++ b/src/libknot/quic/quic.h @@ -32,6 +32,11 @@ #define KNOT_QUIC_PIN_LEN 32 +#define KNOT_QUIC_HANDLE_RET_CLOSE 2000 + +// RFC 9250 +#define KNOT_QUIC_ERR_EXCESSIVE_LOAD 0x4 + struct gnutls_x509_crt_int; struct knot_quic_creds; struct knot_quic_session; @@ -58,8 +63,6 @@ typedef struct knot_quic_reply { void (*free_reply)(struct knot_quic_reply *); } knot_quic_reply_t; -#define KNOT_QUIC_HANDLE_RET_CLOSE (2000) - /*! * \brief Check if session ticket can be taken out of this connection. */ diff --git a/src/libknot/quic/quic_conn.c b/src/libknot/quic/quic_conn.c index 7dfa3897fd..4b3d8fede6 100644 --- a/src/libknot/quic/quic_conn.c +++ b/src/libknot/quic/quic_conn.c @@ -97,8 +97,18 @@ void knot_quic_table_free(knot_quic_table_t *table) } } +static void send_excessive_load(knot_quic_conn_t *conn, struct knot_quic_reply *reply, + knot_quic_table_t *table) +{ + if (reply != NULL) { + reply->handle_ret = KNOT_QUIC_ERR_EXCESSIVE_LOAD; + (void)knot_quic_send(table, conn, reply, 0, 0); + } +} + _public_ -void knot_quic_table_sweep(knot_quic_table_t *table, struct knot_sweep_stats *stats) +void knot_quic_table_sweep(knot_quic_table_t *table, struct knot_quic_reply *sweep_reply, + struct knot_sweep_stats *stats) { uint64_t now = 0; while (!EMPTY_HEAP(table->expiry_heap)) { @@ -107,17 +117,15 @@ void knot_quic_table_sweep(knot_quic_table_t *table, struct knot_sweep_stats *st break; // highly inprobable } else if (table->usage > table->max_conns) { knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_LIMIT_CONN); + send_excessive_load(c, sweep_reply, table); knot_quic_table_rem(c, table); - // NOTE here it would be correct to send Immediate close - // with DoQ errcode DOQ_EXCESSIVE_LOAD - // nowever, we don't do this for the sake of simplicty - // it would be possible to send by using ngtcp2_conn_get_path()... - // (also applies to below case) } else if (table->obufs_size > table->obufs_max) { knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_LIMIT_OBUF); + send_excessive_load(c, sweep_reply, table); knot_quic_table_rem(c, table); } else if (table->ibufs_size > table->ibufs_max) { knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_LIMIT_IBUF); + send_excessive_load(c, sweep_reply, table); knot_quic_table_rem(c, table); } else if (quic_conn_timeout(c, &now)) { int ret = ngtcp2_conn_handle_expiry(c->conn, now); @@ -125,6 +133,10 @@ void knot_quic_table_sweep(knot_quic_table_t *table, struct knot_sweep_stats *st knot_sweep_stats_incr(stats, KNOT_SWEEP_CTR_TIMEOUT); knot_quic_table_rem(c, table); } else { + if (sweep_reply != NULL) { + sweep_reply->handle_ret = KNOT_EOK; + (void)knot_quic_send(table, c, sweep_reply, 0, 0); + } quic_conn_mark_used(c, table); } } diff --git a/src/libknot/quic/quic_conn.h b/src/libknot/quic/quic_conn.h index 016f4dce3e..ca21a5b193 100644 --- a/src/libknot/quic/quic_conn.h +++ b/src/libknot/quic/quic_conn.h @@ -35,6 +35,7 @@ struct ngtcp2_cid; // declaration taken from wherever in ngtcp2 struct knot_quic_creds; +struct knot_quic_reply; struct knot_sweep_stats; // those are equivalent to contrib/ucw/lists.h , just must not be included. @@ -148,9 +149,11 @@ void knot_quic_table_free(knot_quic_table_t *table); * \brief Close timed out connections and some oldest ones if table full. * * \param table QUIC table to be cleaned up. + * \param sweep_reply Optional: reply structure to send sweep-initiated packets to the client. * \param stats Out: sweep statistics. */ -void knot_quic_table_sweep(knot_quic_table_t *table, struct knot_sweep_stats *stats); +void knot_quic_table_sweep(knot_quic_table_t *table, struct knot_quic_reply *sweep_reply, + struct knot_sweep_stats *stats); /*! * \brief Add new connection/CID link to table. diff --git a/src/utils/kxdpgun/main.c b/src/utils/kxdpgun/main.c index 5a5e94c463..a7c8e87bb9 100644 --- a/src/utils/kxdpgun/main.c +++ b/src/utils/kxdpgun/main.c @@ -877,7 +877,7 @@ void *xdp_gun_thread(void *_ctx) #ifdef ENABLE_QUIC if (ctx->quic) { - (void)knot_quic_table_sweep(quic_table, &sweep_stats); + (void)knot_quic_table_sweep(quic_table, NULL, &sweep_stats); } #endif // ENABLE_QUIC