From: Olivier Houchard Date: Thu, 2 Jul 2026 12:55:26 +0000 (+0200) Subject: MEDIUM: fd: Remove fdinfo X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=02b7685013d8abe8ba28a4621217591652d60157;p=thirdparty%2Fhaproxy.git MEDIUM: fd: Remove fdinfo fdinfo is a global table, indexed by the file descriptor, whose purpose is just to remember if a port was allocated in a port range, so that that port may be released when the file descriptor is closed. But that would no longer work if we split file descriptors per thread-group, and while we may just have one fdinfo table per thread group, this sounds like wasted memory with little benefits. So instead, abuse fdtab to store the relevant information. "state" now has two more flags, FD_HAS_PORT and FD_OWNER_PR. FD_HAS_PORT means a port has been explicitly allocated for that file descriptor. FD_OWNER_PR means the owner field now points to the struct port_range from which the port was allocated, and where it should be released. The local port is now obtained from getsockname(), instead of storing it anywhere. For TCP, as fd_delete() is called from sock_conn_ctrl_close(), which itself is called when the connection is about to be destroyed, we can get the port range used as it is the one provided by the server, which is available as conn->target. For QUIC, unfortunately, the connection may be gone already, so instead, we just store it in the quic_conn. We have to wait until _fd_delete_orphan() is called in order to release the port, as in some rare cases, fd_delete() will not call _fd_delete_orphan(), and we want to make sure we only release the port just as we're closing the file descriptor. --- diff --git a/include/haproxy/fd-t.h b/include/haproxy/fd-t.h index 6abd763e6..a57fd6e8f 100644 --- a/include/haproxy/fd-t.h +++ b/include/haproxy/fd-t.h @@ -72,6 +72,8 @@ enum { #define FD_EXCL_SYSCALL_BIT 21 /* a syscall claims exclusivity on this FD */ #define FD_DISOWN_BIT 22 /* this fd will be closed by some external code */ #define FD_MUST_CLOSE_BIT 23 /* this fd will be closed by some external code */ +#define FD_HAS_PORT_BIT 24 /* This fd had a port allocated from a port range */ +#define FD_OWNER_PR_BIT 27 /* The owner is actually a port range */ /* and flag values */ @@ -114,6 +116,8 @@ enum { #define FD_EXCL_SYSCALL (1U << FD_EXCL_SYSCALL_BIT) #define FD_DISOWN (1U << FD_DISOWN_BIT) #define FD_MUST_CLOSE (1U << FD_MUST_CLOSE_BIT) +#define FD_HAS_PORT (1U << FD_HAS_PORT_BIT) +#define FD_OWNER_PR (1U << FD_OWNER_PR_BIT) /* This function is used to report flags in debugging tools. Please reflect * below any single-bit flag addition above in the same order via the @@ -210,12 +214,6 @@ struct polled_mask { unsigned long poll_send; }; -/* less often used information */ -struct fdinfo { - struct port_range *port_range; /* optional port range to bind to */ - int local_port; /* optional local port */ -}; - /* * Poller descriptors. * - is initialized by the poller's register() function, and should not diff --git a/include/haproxy/fd.h b/include/haproxy/fd.h index 07e132523..9d2ac0586 100644 --- a/include/haproxy/fd.h +++ b/include/haproxy/fd.h @@ -39,7 +39,6 @@ extern struct poller cur_poller; /* the current poller */ extern int nbpollers; extern struct poller pollers[MAX_POLLERS]; /* all registered pollers */ extern struct fdtab *fdtab; /* array of all the file descriptors */ -extern struct fdinfo *fdinfo; /* less-often used infos for file descriptors */ extern int totalconn; /* total # of terminated sessions */ extern int actconn; /* # of active sessions */ diff --git a/include/haproxy/quic_conn-t.h b/include/haproxy/quic_conn-t.h index c9d51d09a..c594fd835 100644 --- a/include/haproxy/quic_conn-t.h +++ b/include/haproxy/quic_conn-t.h @@ -310,6 +310,7 @@ struct qcc_app_ops; /* QUIC connection level counters */ \ struct quic_conn_cntrs cntrs; \ struct connection *conn; \ + struct port_range *sport_range; \ } struct quic_conn { diff --git a/src/fd.c b/src/fd.c index 7df4ad0ae..5e5d759ce 100644 --- a/src/fd.c +++ b/src/fd.c @@ -100,7 +100,6 @@ struct fdtab *fdtab __read_mostly = NULL; /* array of all the file descriptors */ struct polled_mask *polled_mask __read_mostly = NULL; /* Array for the polled_mask of each fd */ -struct fdinfo *fdinfo __read_mostly = NULL; /* less-often used infos for file descriptors */ int totalconn; /* total # of terminated sessions */ int actconn; /* # of active sessions */ @@ -340,7 +339,26 @@ void _fd_delete_orphan(int fd) if (fd_nbupdt > 0 && fd_updt[fd_nbupdt - 1] == fd) fd_nbupdt--; - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); + if (unlikely(fdtab[fd].state & FD_HAS_PORT)) { + struct sockaddr_storage sa; + socklen_t addrlen = sizeof(sa); + int port; + + BUG_ON(!(fdtab[fd].state & FD_OWNER_PR)); + /* + * We have to release the port, let's use getsockname() + * to figure out what the port was. + */ + BUG_ON(getsockname(fd, (struct sockaddr *)&sa, &addrlen) != 0); + if (sa.ss_family == AF_INET) + port = ((struct sockaddr_in *)&sa)->sin_port; + else if (sa.ss_family == AF_INET6) + port = ((struct sockaddr_in6 *)&sa)->sin6_port; + else + ABORT_NOW(); + port_range_release_port(fdtab[fd].owner, port); + } + polled_mask[fd].poll_recv = polled_mask[fd].poll_send = 0; fdtab[fd].state = 0; @@ -348,7 +366,6 @@ void _fd_delete_orphan(int fd) #ifdef DEBUG_FD fdtab[fd].event_count = 0; #endif - fdinfo[fd].port_range = NULL; fdtab[fd].owner = NULL; /* perform the close() call last as it's what unlocks the instant reuse @@ -1188,12 +1205,6 @@ int init_pollers() } vma_set_name(polled_mask, global.maxsock * sizeof(*polled_mask), "fd", "polled_mask"); - if ((fdinfo = calloc(global.maxsock, sizeof(*fdinfo))) == NULL) { - ha_alert("Not enough memory to allocate %d entries for fdinfo!\n", global.maxsock); - goto fail_info; - } - vma_set_name(fdinfo, global.maxsock * sizeof(*fdinfo), "fd", "fdinfo"); - for (p = 0; p < MAX_TGROUPS; p++) update_list[p].first = update_list[p].last = -1; @@ -1217,7 +1228,6 @@ int init_pollers() } } while (!bp || bp->pref == 0); - free(fdinfo); fail_info: free(polled_mask); fail_polledmask: @@ -1241,7 +1251,6 @@ void deinit_pollers() { bp->term(bp); } - ha_free(&fdinfo); ha_aligned_free(fdtab); ha_free(&polled_mask); } diff --git a/src/proto_quic.c b/src/proto_quic.c index 547e6d9de..b0f795298 100644 --- a/src/proto_quic.c +++ b/src/proto_quic.c @@ -286,6 +286,7 @@ int quic_connect_server(struct connection *conn, int flags) struct sockaddr_storage *addr, saddr; struct quic_conn *qc = conn->handle.qc; socklen_t saddr_len; + int local_port = 0; BUG_ON(qc->fd != -1); BUG_ON(!conn->dst); @@ -353,21 +354,22 @@ int quic_connect_server(struct connection *conn, int flags) /* note: in case of retry, we may have to release a previously * allocated port, hence this loop's construct. */ - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; + port_range_release_port(src->sport_range, local_port); + local_port = 0; + qc->sport_range = NULL; if (!attempts) break; attempts--; - fdinfo[fd].local_port = port_range_alloc_port(src->sport_range); - if (!fdinfo[fd].local_port) { + local_port = port_range_alloc_port(src->sport_range); + if (!local_port) { conn->err_code = CO_ER_PORT_RANGE; break; } - fdinfo[fd].port_range = src->sport_range; - set_host_port(&sa, fdinfo[fd].local_port); + set_host_port(&sa, local_port); + qc->sport_range = src->sport_range; ret = quic_bind_socket(fd, flags, &sa, conn->src); if (ret != 0) @@ -385,8 +387,7 @@ int quic_connect_server(struct connection *conn, int flags) } if (unlikely(ret != 0)) { - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; + port_range_release_port(src->sport_range, local_port); close(fd); if (ret == 1) { @@ -415,8 +416,7 @@ int quic_connect_server(struct connection *conn, int flags) addr = (conn->flags & CO_FL_SOCKS4) ? &srv->socks4_addr : conn->dst; if (connect(fd, (const struct sockaddr *)addr, get_addr_len(addr)) == -1) { - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; + port_range_release_port(src ? src->sport_range : NULL, local_port); close(fd); conn->flags |= CO_FL_ERROR; return SF_ERR_SRVCL; @@ -433,6 +433,8 @@ int quic_connect_server(struct connection *conn, int flags) fd_want_recv(fd); conn_ctrl_init(conn); + if (local_port != 0) + _HA_ATOMIC_OR(&fdtab[fd].state, FD_HAS_PORT); return SF_ERR_NONE; /* connection is OK */ } diff --git a/src/proto_tcp.c b/src/proto_tcp.c index 932e7a53b..71ce3faa5 100644 --- a/src/proto_tcp.c +++ b/src/proto_tcp.c @@ -370,6 +370,7 @@ int tcp_connect_server(struct connection *conn, int flags) struct proxy *be; struct conn_src *src; int use_fastopen = 0; + int local_port = 0; struct sockaddr_storage *addr; BUG_ON(!conn->dst); @@ -462,24 +463,23 @@ int tcp_connect_server(struct connection *conn, int flags) memcpy(&sa, &src->source_addr, sizeof(sa)); do { + /* note: in case of retry, we may have to release a previously * allocated port, hence this loop's construct. */ - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; - + port_range_release_port(src->sport_range, local_port); + local_port = 0; if (!attempts) break; attempts--; - fdinfo[fd].local_port = port_range_alloc_port(src->sport_range); - if (!fdinfo[fd].local_port) { + local_port = port_range_alloc_port(src->sport_range); + if (!local_port) { conn->err_code = CO_ER_PORT_RANGE; break; } - fdinfo[fd].port_range = src->sport_range; - set_host_port(&sa, fdinfo[fd].local_port); + set_host_port(&sa, local_port); ret = tcp_bind_socket(fd, flags, &sa, conn->src); if (ret != 0) @@ -497,8 +497,7 @@ int tcp_connect_server(struct connection *conn, int flags) } if (unlikely(ret != 0)) { - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; + port_range_release_port(src->sport_range, local_port); close(fd); if (ret == 1) { @@ -606,16 +605,14 @@ int tcp_connect_server(struct connection *conn, int flags) } qfprintf(stderr,"Connect() failed for backend %s: %s.\n", be->id, msg); - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; + port_range_release_port(src ? src->sport_range : NULL, local_port); close(fd); send_log(be, LOG_ERR, "Connect() failed for backend %s: %s.\n", be->id, msg); conn->flags |= CO_FL_ERROR; return SF_ERR_RESOURCE; } else if (errno == ETIMEDOUT) { //qfprintf(stderr,"Connect(): ETIMEDOUT"); - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; + port_range_release_port(src ? src->sport_range : NULL, local_port); close(fd); conn->err_code = CO_ER_SOCK_ERR; conn->flags |= CO_FL_ERROR; @@ -623,8 +620,7 @@ int tcp_connect_server(struct connection *conn, int flags) } else { // (errno == ECONNREFUSED || errno == ENETUNREACH || errno == EACCES || errno == EPERM) //qfprintf(stderr,"Connect(): %d", errno); - port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); - fdinfo[fd].port_range = NULL; + port_range_release_port(src ? src->sport_range : NULL, local_port); close(fd); conn->err_code = CO_ER_SOCK_ERR; conn->flags |= CO_FL_ERROR; @@ -637,6 +633,8 @@ int tcp_connect_server(struct connection *conn, int flags) } conn_ctrl_init(conn); /* registers the FD */ + if (local_port != 0) + _HA_ATOMIC_OR(&fdtab[fd].state, FD_HAS_PORT); HA_ATOMIC_OR(&fdtab[fd].state, FD_LINGER_RISK); /* close hard if needed */ if (conn->flags & CO_FL_WAIT_L4_CONN) { diff --git a/src/quic_conn.c b/src/quic_conn.c index d27eeb323..7ecc824a5 100644 --- a/src/quic_conn.c +++ b/src/quic_conn.c @@ -1165,6 +1165,7 @@ struct quic_conn *qc_new_conn(void *target, qc->qcc = NULL; qc->strm_reject = NULL; qc->path = NULL; + qc->sport_range = NULL; /* Keyupdate: required to safely call quic_tls_ku_free() from * quic_conn_release(). diff --git a/src/quic_sock.c b/src/quic_sock.c index b3e903dc9..d8ebf4cd6 100644 --- a/src/quic_sock.c +++ b/src/quic_sock.c @@ -987,6 +987,10 @@ void qc_alloc_fd(struct quic_conn *qc, const struct sockaddr_storage *src, void qc_release_fd(struct quic_conn *qc, int reinit) { if (qc_test_fd(qc)) { + if (unlikely(fdtab[qc->fd].state & FD_HAS_PORT)) { + fdtab[qc->fd].owner = qc->sport_range; + _HA_ATOMIC_OR(&fdtab[qc->fd].state, FD_OWNER_PR); + } fd_delete(qc->fd); qc->fd = DEAD_FD_MAGIC; diff --git a/src/sock.c b/src/sock.c index df387e4e8..8474c1c22 100644 --- a/src/sock.c +++ b/src/sock.c @@ -901,6 +901,22 @@ void sock_conn_ctrl_init(struct connection *conn) void sock_conn_ctrl_close(struct connection *conn) { BUG_ON(conn->flags & CO_FL_FDLESS); + if (unlikely(fdtab[conn->handle.fd].state & FD_HAS_PORT)) { + struct server *srv = objt_server(conn->target); + struct proxy *be; + struct port_range *port_range; + + BUG_ON(srv == NULL); + be = srv->proxy; + if (srv->conn_src.opts & CO_SRC_BIND) + port_range = srv->conn_src.sport_range; + else if (be->conn_src.opts & CO_SRC_BIND) + port_range = be->conn_src.sport_range; + else + ABORT_NOW(); + _HA_ATOMIC_OR(&fdtab[conn->handle.fd].state, FD_OWNER_PR); + fdtab[conn->handle.fd].owner = port_range; + } fd_delete(conn->handle.fd); conn->handle.fd = DEAD_FD_MAGIC; }