Optimize ephemeral port reuse with IP_BIND_ADDRESS_NO_PORT (#1115)

    commBind: Cannot bind socket ... : (98) Address already in use

When opening a TCP connection from a specific IP address (e.g., set by
tcp_outgoing_address), Squid usually lets the OS select the source port
by performing the traditional bind(2)+connect(2) sequence with zero
source port. This sequence consumes ephemeral ports at the connection
opening rate even if many destinations are unique (and, hence, could
reuse the same source port) because the OS must pick a source port at
bind(2) time, before connect(2) supplies the destination address.

Starting with v4.2, Linux supports IP_BIND_ADDRESS_NO_PORT socket option
designed to decrease ephemeral port consumption by delaying port binding
until connect(2) time. If available, Squid now uses that option when
opening any outgoing TCP connection bound to a source IP address.

diff --git a/src/comm.cc b/src/comm.cc
index e3a7a9b6313395c810f44b9bd41ed0c208941561..f7516ea3d9138edc51452440b4be97237374aa2e 100644 (file)
--- a/src/comm.cc
+++ b/src/comm.cc
@@ -57,6 +57,7 @@
  */
 
 static IOCB commHalfClosedReader;
+static int comm_openex(int sock_type, int proto, Ip::Address &, int flags, const char *note);
 static void comm_init_opened(const Comm::ConnectionPointer &conn, const char *note, struct addrinfo *AI);
 static int comm_apply_flags(int new_socket, Ip::Address &addr, int flags, struct addrinfo *AI);
 
@@ -74,6 +75,7 @@ static EVH commHalfClosedCheck;
 static void commPlanHalfClosedCheck();
 
 static Comm::Flag commBind(int s, struct addrinfo &);
+static void commSetBindAddressNoPort(int);
 static void commSetReuseAddr(int);
 static void commSetNoLinger(int);
 #ifdef TCP_NODELAY
@@ -200,6 +202,22 @@ comm_local_port(int fd)
     return F->local_addr.port();
 }
 
+/// sets the IP_BIND_ADDRESS_NO_PORT socket option to optimize ephemeral port
+/// reuse by outgoing TCP connections that must bind(2) to a source IP address
+static void
+commSetBindAddressNoPort(const int fd)
+{
+#if defined(IP_BIND_ADDRESS_NO_PORT)
+    int flag = 1;
+    if (setsockopt(fd, IPPROTO_IP, IP_BIND_ADDRESS_NO_PORT, reinterpret_cast<char*>(&flag), sizeof(flag)) < 0) {
+        const auto savedErrno = errno;
+        debugs(50, DBG_IMPORTANT, "ERROR: setsockopt(IP_BIND_ADDRESS_NO_PORT) failure: " << xstrerr(savedErrno));
+    }
+#else
+    (void)fd;
+#endif
+}
+
 static Comm::Flag
 commBind(int s, struct addrinfo &inaddr)
 {
@@ -226,6 +244,10 @@ comm_open(int sock_type,
           int flags,
           const char *note)
 {
+    // assume zero-port callers do not need to know the assigned port right away
+    if (sock_type == SOCK_STREAM && addr.port() == 0 && ((flags & COMM_DOBIND) || !addr.isAnyAddr()))
+        flags |= COMM_DOBIND_PORT_LATER;
+
     return comm_openex(sock_type, proto, addr, flags, note);
 }
 
@@ -327,7 +349,7 @@ comm_set_transparent(int fd)
  * Create a socket. Default is blocking, stream (TCP) socket.  IO_TYPE
  * is OR of flags specified in defines.h:COMM_*
  */
-int
+static int
 comm_openex(int sock_type,
             int proto,
             Ip::Address &addr,
@@ -482,6 +504,10 @@ comm_apply_flags(int new_socket,
             }
         }
 #endif
+
+        if ((flags & COMM_DOBIND_PORT_LATER))
+            commSetBindAddressNoPort(new_socket);
+
         if (commBind(new_socket, *AI) != Comm::OK) {
             comm_close(new_socket);
             return -1;

diff --git a/src/comm.h b/src/comm.h
index fa4c62a21cfd669992e3ed93ef6e50b2f51d257e..40585730568668f74b8ffee4f77764f578282713 100644 (file)
--- a/src/comm.h
+++ b/src/comm.h
@@ -39,7 +39,6 @@ void comm_import_opened(const Comm::ConnectionPointer &, const char *note, struc
 
 /**
  * Open a port specially bound for listening or sending through a specific port.
- * This is a wrapper providing IPv4/IPv6 failover around comm_openex().
  * Please use for all listening sockets and bind() outbound sockets.
  *
  * It will open a socket bound for:
@@ -55,7 +54,6 @@ void comm_import_opened(const Comm::ConnectionPointer &, const char *note, struc
 int comm_open_listener(int sock_type, int proto, Ip::Address &addr, int flags, const char *note);
 void comm_open_listener(int sock_type, int proto, Comm::ConnectionPointer &conn, const char *note);
 
-int comm_openex(int, int, Ip::Address &, int, const char *);
 unsigned short comm_local_port(int fd);
 
 int comm_udp_sendto(int sock, const Ip::Address &to, const void *buf, int buflen);

diff --git a/src/comm/ConnOpener.cc b/src/comm/ConnOpener.cc
index f1df32da3c12c6b2ff08e6a93daea56c3c4e712b..4a63cd5bd2d838245500f9bc5290c3111570b447 100644 (file)
--- a/src/comm/ConnOpener.cc
+++ b/src/comm/ConnOpener.cc
@@ -284,7 +284,7 @@ Comm::ConnOpener::createFd()
     if (callback_ == nullptr || callback_->canceled())
         return false;
 
-    temporaryFd_ = comm_openex(SOCK_STREAM, IPPROTO_TCP, conn_->local, conn_->flags, host_);
+    temporaryFd_ = comm_open(SOCK_STREAM, IPPROTO_TCP, conn_->local, conn_->flags, host_);
     if (temporaryFd_ < 0) {
         sendAnswer(Comm::ERR_CONNECT, 0, "Comm::ConnOpener::createFd");
         return false;

diff --git a/src/comm/Connection.h b/src/comm/Connection.h
index 9da1ff47b062f391b7a2dcf3f000b254735ce94a..26cf784c29a52b713c7cd7474d5237b96f740a9d 100644 (file)
--- a/src/comm/Connection.h
+++ b/src/comm/Connection.h
@@ -52,6 +52,8 @@ namespace Comm
 #define COMM_REUSEPORT          0x40 //< needs SO_REUSEPORT
 /// not registered with Comm and not owned by any connection-closing code
 #define COMM_ORPHANED           0x80
+/// Internal Comm optimization: Keep the source port unassigned until connect(2)
+#define COMM_DOBIND_PORT_LATER 0x100
 
 /**
  * Store data about the physical and logical attributes of a connection.

diff --git a/src/tests/stub_comm.cc b/src/tests/stub_comm.cc
index 25de5d06405e7f2a977318ceb1467d315192210d..ba398f18cd0027131abed86dfb8e304b6af1e37e 100644 (file)
--- a/src/tests/stub_comm.cc
+++ b/src/tests/stub_comm.cc
@@ -43,7 +43,6 @@ int comm_open_uds(int, int, struct sockaddr_un*, int) STUB_RETVAL(-1)
 void comm_import_opened(const Comm::ConnectionPointer &, const char *, struct addrinfo *) STUB
 int comm_open_listener(int, int, Ip::Address &, int, const char *) STUB_RETVAL(-1)
 void comm_open_listener(int, int, Comm::ConnectionPointer &, const char *) STUB
-// int comm_openex(int, int, Ip::Address &, int, tos_t, nfmark_t, const char *) STUB_RETVAL(-1)
 unsigned short comm_local_port(int) STUB_RETVAL(0)
 int comm_udp_sendto(int, const Ip::Address &, const void *, int) STUB_RETVAL(-1)
 void commCallCloseHandlers(int) STUB