From: pponakan <52429302+pponakan@users.noreply.github.com>
Date: Mon, 22 Aug 2022 22:28:34 +0000 (+0000)
Subject: Optimize ephemeral port reuse with IP_BIND_ADDRESS_NO_PORT (#1115)
X-Git-Tag: SQUID_6_0_1~125
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=74d7d19;p=thirdparty%2Fsquid.git

Optimize ephemeral port reuse with IP_BIND_ADDRESS_NO_PORT (#1115)

    commBind: Cannot bind socket ... : (98) Address already in use

When opening a TCP connection from a specific IP address (e.g., set by
tcp_outgoing_address), Squid usually lets the OS select the source port
by performing the traditional bind(2)+connect(2) sequence with zero
source port. This sequence consumes ephemeral ports at the connection
opening rate even if many destinations are unique (and, hence, could
reuse the same source port) because the OS must pick a source port at
bind(2) time, before connect(2) supplies the destination address.

Starting with v4.2, Linux supports IP_BIND_ADDRESS_NO_PORT socket option
designed to decrease ephemeral port consumption by delaying port binding
until connect(2) time. If available, Squid now uses that option when
opening any outgoing TCP connection bound to a source IP address.
---

diff --git a/src/comm.cc b/src/comm.cc
index e3a7a9b631..f7516ea3d9 100644
--- a/src/comm.cc
+++ b/src/comm.cc
@@ -57,6 +57,7 @@
  */
 
 static IOCB commHalfClosedReader;
+static int comm_openex(int sock_type, int proto, Ip::Address &, int flags, const char *note);
 static void comm_init_opened(const Comm::ConnectionPointer &conn, const char *note, struct addrinfo *AI);
 static int comm_apply_flags(int new_socket, Ip::Address &addr, int flags, struct addrinfo *AI);
 
@@ -74,6 +75,7 @@ static EVH commHalfClosedCheck;
 static void commPlanHalfClosedCheck();
 
 static Comm::Flag commBind(int s, struct addrinfo &);
+static void commSetBindAddressNoPort(int);
 static void commSetReuseAddr(int);
 static void commSetNoLinger(int);
 #ifdef TCP_NODELAY
@@ -200,6 +202,22 @@ comm_local_port(int fd)
     return F->local_addr.port();
 }
 
+/// sets the IP_BIND_ADDRESS_NO_PORT socket option to optimize ephemeral port
+/// reuse by outgoing TCP connections that must bind(2) to a source IP address
+static void
+commSetBindAddressNoPort(const int fd)
+{
+#if defined(IP_BIND_ADDRESS_NO_PORT)
+    int flag = 1;
+    if (setsockopt(fd, IPPROTO_IP, IP_BIND_ADDRESS_NO_PORT, reinterpret_cast<char*>(&flag), sizeof(flag)) < 0) {
+        const auto savedErrno = errno;
+        debugs(50, DBG_IMPORTANT, "ERROR: setsockopt(IP_BIND_ADDRESS_NO_PORT) failure: " << xstrerr(savedErrno));
+    }
+#else
+    (void)fd;
+#endif
+}
+
 static Comm::Flag
 commBind(int s, struct addrinfo &inaddr)
 {
@@ -226,6 +244,10 @@ comm_open(int sock_type,
           int flags,
           const char *note)
 {
+    // assume zero-port callers do not need to know the assigned port right away
+    if (sock_type == SOCK_STREAM && addr.port() == 0 && ((flags & COMM_DOBIND) || !addr.isAnyAddr()))
+        flags |= COMM_DOBIND_PORT_LATER;
+
     return comm_openex(sock_type, proto, addr, flags, note);
 }
 
@@ -327,7 +349,7 @@ comm_set_transparent(int fd)
  * Create a socket. Default is blocking, stream (TCP) socket.  IO_TYPE
  * is OR of flags specified in defines.h:COMM_*
  */
-int
+static int
 comm_openex(int sock_type,
             int proto,
             Ip::Address &addr,
@@ -482,6 +504,10 @@ comm_apply_flags(int new_socket,
             }
         }
 #endif
+
+        if ((flags & COMM_DOBIND_PORT_LATER))
+            commSetBindAddressNoPort(new_socket);
+
         if (commBind(new_socket, *AI) != Comm::OK) {
             comm_close(new_socket);
             return -1;
diff --git a/src/comm.h b/src/comm.h
index fa4c62a21c..4058573056 100644
--- a/src/comm.h
+++ b/src/comm.h
@@ -39,7 +39,6 @@ void comm_import_opened(const Comm::ConnectionPointer &, const char *note, struc
 
 /**
  * Open a port specially bound for listening or sending through a specific port.
- * This is a wrapper providing IPv4/IPv6 failover around comm_openex().
  * Please use for all listening sockets and bind() outbound sockets.
  *
  * It will open a socket bound for:
@@ -55,7 +54,6 @@ void comm_import_opened(const Comm::ConnectionPointer &, const char *note, struc
 int comm_open_listener(int sock_type, int proto, Ip::Address &addr, int flags, const char *note);
 void comm_open_listener(int sock_type, int proto, Comm::ConnectionPointer &conn, const char *note);
 
-int comm_openex(int, int, Ip::Address &, int, const char *);
 unsigned short comm_local_port(int fd);
 
 int comm_udp_sendto(int sock, const Ip::Address &to, const void *buf, int buflen);
diff --git a/src/comm/ConnOpener.cc b/src/comm/ConnOpener.cc
index f1df32da3c..4a63cd5bd2 100644
--- a/src/comm/ConnOpener.cc
+++ b/src/comm/ConnOpener.cc
@@ -284,7 +284,7 @@ Comm::ConnOpener::createFd()
     if (callback_ == nullptr || callback_->canceled())
         return false;
 
-    temporaryFd_ = comm_openex(SOCK_STREAM, IPPROTO_TCP, conn_->local, conn_->flags, host_);
+    temporaryFd_ = comm_open(SOCK_STREAM, IPPROTO_TCP, conn_->local, conn_->flags, host_);
     if (temporaryFd_ < 0) {
         sendAnswer(Comm::ERR_CONNECT, 0, "Comm::ConnOpener::createFd");
         return false;
diff --git a/src/comm/Connection.h b/src/comm/Connection.h
index 9da1ff47b0..26cf784c29 100644
--- a/src/comm/Connection.h
+++ b/src/comm/Connection.h
@@ -52,6 +52,8 @@ namespace Comm
 #define COMM_REUSEPORT          0x40 //< needs SO_REUSEPORT
 /// not registered with Comm and not owned by any connection-closing code
 #define COMM_ORPHANED           0x80
+/// Internal Comm optimization: Keep the source port unassigned until connect(2)
+#define COMM_DOBIND_PORT_LATER 0x100
 
 /**
  * Store data about the physical and logical attributes of a connection.
diff --git a/src/tests/stub_comm.cc b/src/tests/stub_comm.cc
index 25de5d0640..ba398f18cd 100644
--- a/src/tests/stub_comm.cc
+++ b/src/tests/stub_comm.cc
@@ -43,7 +43,6 @@ int comm_open_uds(int, int, struct sockaddr_un*, int) STUB_RETVAL(-1)
 void comm_import_opened(const Comm::ConnectionPointer &, const char *, struct addrinfo *) STUB
 int comm_open_listener(int, int, Ip::Address &, int, const char *) STUB_RETVAL(-1)
 void comm_open_listener(int, int, Comm::ConnectionPointer &, const char *) STUB
-// int comm_openex(int, int, Ip::Address &, int, tos_t, nfmark_t, const char *) STUB_RETVAL(-1)
 unsigned short comm_local_port(int) STUB_RETVAL(0)
 int comm_udp_sendto(int, const Ip::Address &, const void *, int) STUB_RETVAL(-1)
 void commCallCloseHandlers(int) STUB