MEDIUM: fd: Remove fdinfo

fdinfo is a global table, indexed by the file descriptor, whose purpose
is just to remember if a port was allocated in a port range, so that
that port may be released when the file descriptor is closed. But that
would no longer work if we split file descriptors per thread-group, and
while we may just have one fdinfo table per thread group, this sounds
like wasted memory with little benefits.
So instead, abuse fdtab to store the relevant information. "state" now
has two more flags, FD_HAS_PORT and FD_OWNER_PR. FD_HAS_PORT means a
port has been explicitly allocated for that file descriptor.
FD_OWNER_PR means the owner field now points to the struct port_range
from which the port was allocated, and where it should be released.
The local port is now obtained from getsockname(), instead of storing it
anywhere.
For TCP, as fd_delete() is called from sock_conn_ctrl_close(), which
itself is called when the connection is about to be destroyed, we can
get the port range used as it is the one provided by the server, which
is available as conn->target.
For QUIC, unfortunately, the connection may be gone already, so instead,
we just store it in the quic_conn.
We have to wait until _fd_delete_orphan() is called in order to release
the port, as in some rare cases, fd_delete() will not call
_fd_delete_orphan(), and we want to make sure we only release the port
just as we're closing the file descriptor.

diff --git a/include/haproxy/fd-t.h b/include/haproxy/fd-t.h
index 6abd763e6f565333f659b72dd58ccea1e01e564a..a57fd6e8fab893c3242d8520e365c1d7aeb6056e 100644 (file)
--- a/include/haproxy/fd-t.h
+++ b/include/haproxy/fd-t.h
@@ -72,6 +72,8 @@ enum {
 #define FD_EXCL_SYSCALL_BIT 21 /* a syscall claims exclusivity on this FD */
 #define FD_DISOWN_BIT      22  /* this fd will be closed by some external code */
 #define FD_MUST_CLOSE_BIT  23  /* this fd will be closed by some external code */
+#define FD_HAS_PORT_BIT    24  /* This fd had a port allocated from a port range */
+#define FD_OWNER_PR_BIT    27 /* The owner is actually a port range */
 
 
 /* and flag values */
@@ -114,6 +116,8 @@ enum {
 #define FD_EXCL_SYSCALL     (1U << FD_EXCL_SYSCALL_BIT)
 #define FD_DISOWN           (1U << FD_DISOWN_BIT)
 #define FD_MUST_CLOSE       (1U << FD_MUST_CLOSE_BIT)
+#define FD_HAS_PORT         (1U << FD_HAS_PORT_BIT)
+#define FD_OWNER_PR         (1U << FD_OWNER_PR_BIT)
 
 /* This function is used to report flags in debugging tools. Please reflect
  * below any single-bit flag addition above in the same order via the
@@ -210,12 +214,6 @@ struct polled_mask {
        unsigned long poll_send;
 };
 
-/* less often used information */
-struct fdinfo {
-       struct port_range *port_range;       /* optional port range to bind to */
-       int local_port;                      /* optional local port */
-};
-
 /*
  * Poller descriptors.
  *  - <name> is initialized by the poller's register() function, and should not

diff --git a/include/haproxy/fd.h b/include/haproxy/fd.h
index 07e1325239f85cb5e16d11832e67351c0c48346a..9d2ac0586d7d922580e80eeb815dacbb04be56ce 100644 (file)
--- a/include/haproxy/fd.h
+++ b/include/haproxy/fd.h
@@ -39,7 +39,6 @@ extern struct poller cur_poller; /* the current poller */
 extern int nbpollers;
 extern struct poller pollers[MAX_POLLERS];   /* all registered pollers */
 extern struct fdtab *fdtab;             /* array of all the file descriptors */
-extern struct fdinfo *fdinfo;           /* less-often used infos for file descriptors */
 extern int totalconn;                   /* total # of terminated sessions */
 extern int actconn;                     /* # of active sessions */
 

diff --git a/include/haproxy/quic_conn-t.h b/include/haproxy/quic_conn-t.h
index c9d51d09a66366b611e9188c6f8f538e164f1572..c594fd835ac263333a78cf1be0b812d464571c4b 100644 (file)
--- a/include/haproxy/quic_conn-t.h
+++ b/include/haproxy/quic_conn-t.h
@@ -310,6 +310,7 @@ struct qcc_app_ops;
         /* QUIC connection level counters */                                   \
         struct quic_conn_cntrs cntrs;                                          \
         struct connection *conn;                                               \
+        struct port_range *sport_range;                                        \
     }
 
 struct quic_conn {

diff --git a/src/fd.c b/src/fd.c
index 7df4ad0aeb8dab64dc110659f7dee877208b6c09..5e5d759ce4ad4d587d4a3cca97fd31f4a9e8591d 100644 (file)
--- a/src/fd.c
+++ b/src/fd.c
@@ -100,7 +100,6 @@
 
 struct fdtab *fdtab             __read_mostly = NULL;  /* array of all the file descriptors */
 struct polled_mask *polled_mask __read_mostly = NULL;  /* Array for the polled_mask of each fd */
-struct fdinfo *fdinfo           __read_mostly = NULL;  /* less-often used infos for file descriptors */
 int totalconn;                  /* total # of terminated sessions */
 int actconn;                    /* # of active sessions */
 
@@ -340,7 +339,26 @@ void _fd_delete_orphan(int fd)
        if (fd_nbupdt > 0 && fd_updt[fd_nbupdt - 1] == fd)
                fd_nbupdt--;
 
-       port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
+       if (unlikely(fdtab[fd].state & FD_HAS_PORT)) {
+               struct sockaddr_storage sa;
+               socklen_t addrlen = sizeof(sa);
+               int port;
+
+               BUG_ON(!(fdtab[fd].state & FD_OWNER_PR));
+               /*
+                * We have to release the port, let's use getsockname()
+                * to figure out what the port was.
+                */
+               BUG_ON(getsockname(fd, (struct sockaddr *)&sa, &addrlen) != 0);
+               if (sa.ss_family == AF_INET)
+                       port = ((struct sockaddr_in *)&sa)->sin_port;
+               else if (sa.ss_family == AF_INET6)
+                       port = ((struct sockaddr_in6 *)&sa)->sin6_port;
+               else
+                       ABORT_NOW();
+               port_range_release_port(fdtab[fd].owner, port);
+       }
+
        polled_mask[fd].poll_recv = polled_mask[fd].poll_send = 0;
 
        fdtab[fd].state = 0;
@@ -348,7 +366,6 @@ void _fd_delete_orphan(int fd)
 #ifdef DEBUG_FD
        fdtab[fd].event_count = 0;
 #endif
-       fdinfo[fd].port_range = NULL;
        fdtab[fd].owner = NULL;
 
        /* perform the close() call last as it's what unlocks the instant reuse
@@ -1188,12 +1205,6 @@ int init_pollers()
        }
        vma_set_name(polled_mask, global.maxsock * sizeof(*polled_mask), "fd", "polled_mask");
 
-       if ((fdinfo = calloc(global.maxsock, sizeof(*fdinfo))) == NULL) {
-               ha_alert("Not enough memory to allocate %d entries for fdinfo!\n", global.maxsock);
-               goto fail_info;
-       }
-       vma_set_name(fdinfo, global.maxsock * sizeof(*fdinfo), "fd", "fdinfo");
-
        for (p = 0; p < MAX_TGROUPS; p++)
                update_list[p].first = update_list[p].last = -1;
 
@@ -1217,7 +1228,6 @@ int init_pollers()
                }
        } while (!bp || bp->pref == 0);
 
-       free(fdinfo);
  fail_info:
        free(polled_mask);
  fail_polledmask:
@@ -1241,7 +1251,6 @@ void deinit_pollers() {
                        bp->term(bp);
        }
 
-       ha_free(&fdinfo);
        ha_aligned_free(fdtab);
        ha_free(&polled_mask);
 }

diff --git a/src/proto_quic.c b/src/proto_quic.c
index 547e6d9de233f0bc82823ab157a6828ef4d8027e..b0f795298bad9060c996f91fbf624742d3ea60de 100644 (file)
--- a/src/proto_quic.c
+++ b/src/proto_quic.c
@@ -286,6 +286,7 @@ int quic_connect_server(struct connection *conn, int flags)
        struct sockaddr_storage *addr, saddr;
        struct quic_conn *qc = conn->handle.qc;
        socklen_t saddr_len;
+       int local_port = 0;
 
        BUG_ON(qc->fd != -1);
        BUG_ON(!conn->dst);
@@ -353,21 +354,22 @@ int quic_connect_server(struct connection *conn, int flags)
                                /* note: in case of retry, we may have to release a previously
                                 * allocated port, hence this loop's construct.
                                 */
-                               port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-                               fdinfo[fd].port_range = NULL;
+                               port_range_release_port(src->sport_range, local_port);
+                               local_port = 0;
+                               qc->sport_range = NULL;
 
                                if (!attempts)
                                        break;
                                attempts--;
 
-                               fdinfo[fd].local_port = port_range_alloc_port(src->sport_range);
-                               if (!fdinfo[fd].local_port) {
+                               local_port = port_range_alloc_port(src->sport_range);
+                               if (!local_port) {
                                        conn->err_code = CO_ER_PORT_RANGE;
                                        break;
                                }
 
-                               fdinfo[fd].port_range = src->sport_range;
-                               set_host_port(&sa, fdinfo[fd].local_port);
+                               set_host_port(&sa, local_port);
+                               qc->sport_range = src->sport_range;
 
                                ret = quic_bind_socket(fd, flags, &sa, conn->src);
                                if (ret != 0)
@@ -385,8 +387,7 @@ int quic_connect_server(struct connection *conn, int flags)
                }
 
                if (unlikely(ret != 0)) {
-                       port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-                       fdinfo[fd].port_range = NULL;
+                       port_range_release_port(src->sport_range, local_port);
                        close(fd);
 
                        if (ret == 1) {
@@ -415,8 +416,7 @@ int quic_connect_server(struct connection *conn, int flags)
 
        addr = (conn->flags & CO_FL_SOCKS4) ? &srv->socks4_addr : conn->dst;
        if (connect(fd, (const struct sockaddr *)addr, get_addr_len(addr)) == -1) {
-               port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-               fdinfo[fd].port_range = NULL;
+               port_range_release_port(src ? src->sport_range : NULL, local_port);
                close(fd);
                conn->flags |= CO_FL_ERROR;
                return SF_ERR_SRVCL;
@@ -433,6 +433,8 @@ int quic_connect_server(struct connection *conn, int flags)
        fd_want_recv(fd);
 
        conn_ctrl_init(conn);
+       if (local_port != 0)
+               _HA_ATOMIC_OR(&fdtab[fd].state, FD_HAS_PORT);
        return SF_ERR_NONE;  /* connection is OK */
 }
 

diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 932e7a53b87fa81995685655bf81ac4d80bb3f2c..71ce3faa51d1c1d039bb6f36bcb364e94ee4e28f 100644 (file)
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -370,6 +370,7 @@ int tcp_connect_server(struct connection *conn, int flags)
        struct proxy *be;
        struct conn_src *src;
        int use_fastopen = 0;
+       int local_port = 0;
        struct sockaddr_storage *addr;
 
        BUG_ON(!conn->dst);
@@ -462,24 +463,23 @@ int tcp_connect_server(struct connection *conn, int flags)
                        memcpy(&sa, &src->source_addr, sizeof(sa));
 
                        do {
+
                                /* note: in case of retry, we may have to release a previously
                                 * allocated port, hence this loop's construct.
                                 */
-                               port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-                               fdinfo[fd].port_range = NULL;
-
+                               port_range_release_port(src->sport_range, local_port);
+                               local_port = 0;
                                if (!attempts)
                                        break;
                                attempts--;
 
-                               fdinfo[fd].local_port = port_range_alloc_port(src->sport_range);
-                               if (!fdinfo[fd].local_port) {
+                               local_port = port_range_alloc_port(src->sport_range);
+                               if (!local_port) {
                                        conn->err_code = CO_ER_PORT_RANGE;
                                        break;
                                }
 
-                               fdinfo[fd].port_range = src->sport_range;
-                               set_host_port(&sa, fdinfo[fd].local_port);
+                               set_host_port(&sa, local_port);
 
                                ret = tcp_bind_socket(fd, flags, &sa, conn->src);
                                if (ret != 0)
@@ -497,8 +497,7 @@ int tcp_connect_server(struct connection *conn, int flags)
                }
 
                if (unlikely(ret != 0)) {
-                       port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-                       fdinfo[fd].port_range = NULL;
+                       port_range_release_port(src->sport_range, local_port);
                        close(fd);
 
                        if (ret == 1) {
@@ -606,16 +605,14 @@ int tcp_connect_server(struct connection *conn, int flags)
                        }
 
                        qfprintf(stderr,"Connect() failed for backend %s: %s.\n", be->id, msg);
-                       port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-                       fdinfo[fd].port_range = NULL;
+                       port_range_release_port(src ? src->sport_range : NULL, local_port);
                        close(fd);
                        send_log(be, LOG_ERR, "Connect() failed for backend %s: %s.\n", be->id, msg);
                        conn->flags |= CO_FL_ERROR;
                        return SF_ERR_RESOURCE;
                } else if (errno == ETIMEDOUT) {
                        //qfprintf(stderr,"Connect(): ETIMEDOUT");
-                       port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-                       fdinfo[fd].port_range = NULL;
+                       port_range_release_port(src ? src->sport_range : NULL, local_port);
                        close(fd);
                        conn->err_code = CO_ER_SOCK_ERR;
                        conn->flags |= CO_FL_ERROR;
@@ -623,8 +620,7 @@ int tcp_connect_server(struct connection *conn, int flags)
                } else {
                        // (errno == ECONNREFUSED || errno == ENETUNREACH || errno == EACCES || errno == EPERM)
                        //qfprintf(stderr,"Connect(): %d", errno);
-                       port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
-                       fdinfo[fd].port_range = NULL;
+                       port_range_release_port(src ? src->sport_range : NULL, local_port);
                        close(fd);
                        conn->err_code = CO_ER_SOCK_ERR;
                        conn->flags |= CO_FL_ERROR;
@@ -637,6 +633,8 @@ int tcp_connect_server(struct connection *conn, int flags)
        }
 
        conn_ctrl_init(conn);       /* registers the FD */
+       if (local_port != 0)
+               _HA_ATOMIC_OR(&fdtab[fd].state, FD_HAS_PORT);
        HA_ATOMIC_OR(&fdtab[fd].state, FD_LINGER_RISK);  /* close hard if needed */
 
        if (conn->flags & CO_FL_WAIT_L4_CONN) {

diff --git a/src/quic_conn.c b/src/quic_conn.c
index d27eeb323f8b39f6224edc81bf14faf350d13da0..7ecc824a5ec4f0cb64fd3eb8b0dee435afb6bde5 100644 (file)
--- a/src/quic_conn.c
+++ b/src/quic_conn.c
@@ -1165,6 +1165,7 @@ struct quic_conn *qc_new_conn(void *target,
        qc->qcc = NULL;
        qc->strm_reject = NULL;
        qc->path = NULL;
+       qc->sport_range = NULL;
 
        /* Keyupdate: required to safely call quic_tls_ku_free() from
         * quic_conn_release().

diff --git a/src/quic_sock.c b/src/quic_sock.c
index b3e903dc97c79280c91e48a9d288b0ad709fbe5a..d8ebf4cd676606b4a08ee1aa39103a8e8b1e234c 100644 (file)
--- a/src/quic_sock.c
+++ b/src/quic_sock.c
@@ -987,6 +987,10 @@ void qc_alloc_fd(struct quic_conn *qc, const struct sockaddr_storage *src,
 void qc_release_fd(struct quic_conn *qc, int reinit)
 {
        if (qc_test_fd(qc)) {
+               if (unlikely(fdtab[qc->fd].state & FD_HAS_PORT)) {
+                       fdtab[qc->fd].owner = qc->sport_range;
+                       _HA_ATOMIC_OR(&fdtab[qc->fd].state, FD_OWNER_PR);
+               }
                fd_delete(qc->fd);
                qc->fd = DEAD_FD_MAGIC;
 

diff --git a/src/sock.c b/src/sock.c
index df387e4e87d2e755ce597a89d92e4b937f67531f..8474c1c220c10aaad38b070d5fa4dd4c7545acff 100644 (file)
--- a/src/sock.c
+++ b/src/sock.c
@@ -901,6 +901,22 @@ void sock_conn_ctrl_init(struct connection *conn)
 void sock_conn_ctrl_close(struct connection *conn)
 {
        BUG_ON(conn->flags & CO_FL_FDLESS);
+       if (unlikely(fdtab[conn->handle.fd].state & FD_HAS_PORT)) {
+               struct server *srv = objt_server(conn->target);
+               struct proxy *be;
+               struct port_range *port_range;
+
+               BUG_ON(srv == NULL);
+               be = srv->proxy;
+               if (srv->conn_src.opts & CO_SRC_BIND)
+                       port_range = srv->conn_src.sport_range;
+               else if (be->conn_src.opts & CO_SRC_BIND)
+                       port_range = be->conn_src.sport_range;
+               else
+                       ABORT_NOW();
+               _HA_ATOMIC_OR(&fdtab[conn->handle.fd].state, FD_OWNER_PR);
+               fdtab[conn->handle.fd].owner = port_range;
+       }
        fd_delete(conn->handle.fd);
        conn->handle.fd = DEAD_FD_MAGIC;
 }