Split reusing the addr/port and load-balancing socket options

The SO_REUSEADDR, SO_REUSEPORT and SO_REUSEPORT_LB has different meaning
on different platform. In this commit, we split the function to set the
reuse of address/port and setting the load-balancing into separate
functions.

The libuv library already have multiplatform support for setting
SO_REUSEADDR and SO_REUSEPORT that allows binding to the same address
and port, but unfortunately, when used after the load-balancing socket
options have been already set, it overrides the previous setting, so we
need our own helper function to enable the SO_REUSEADDR/SO_REUSEPORT
first and then enable the load-balancing socket option.

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index 1ea281e251f750b18613fdcefe52341232360dfe..ceeb5cbb276ad4681017c14009576a046706263a 100644 (file)
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -840,9 +840,15 @@ isc__nm_socket_freebind(uv_os_sock_t fd, sa_family_t sa_family);
  */
 
 isc_result_t
-isc__nm_socket_reuseport(uv_os_sock_t fd);
+isc__nm_socket_reuse(uv_os_sock_t fd);
 /*%<
- * Set the SO_REUSEPORT (or equivalent) socket option on the fd
+ * Set the SO_REUSEADDR or SO_REUSEPORT (or equivalent) socket option on the fd
+ */
+
+isc_result_t
+isc__nm_socket_reuse_lb(uv_os_sock_t fd);
+/*%<
+ * Set the SO_REUSEPORT_LB (or equivalent) socket option on the fd
  */
 
 isc_result_t

diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index 7567ee30ffcdcc3ae4a3146c8d04ef4c8899548d..28dc6f22e11f271cfb770d5dd01c1af94efbeacd 100644 (file)
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -1625,46 +1625,66 @@ isc__nm_socket_freebind(uv_os_sock_t fd, sa_family_t sa_family) {
 }
 
 isc_result_t
-isc__nm_socket_reuseport(uv_os_sock_t fd) {
+isc__nm_socket_reuse(uv_os_sock_t fd) {
        /*
-        * This is SO_REUSE**** hell:
-        *
         * Generally, the SO_REUSEADDR socket option allows reuse of
-        * local addresses.  On Windows, it also allows a socket to
-        * forcibly bind to a port in use by another socket.
+        * local addresses.
         *
-        * On Linux, SO_REUSEPORT socket option allows sockets to be
-        * bound to an identical socket address. For UDP sockets, the
-        * use of this option can provide better distribution of
-        * incoming datagrams to multiple processes (or threads) as
-        * compared to the traditional technique of having multiple
-        * processes compete to receive datagrams on the same socket.
+        * On the BSDs, SO_REUSEPORT implies SO_REUSEADDR but with some
+        * additional refinements for programs that use multicast.
         *
-        * On FreeBSD 12+, the same thing is achieved with SO_REUSEPORT_LB.
+        * On Linux, SO_REUSEPORT has different semantics: it _shares_ the port
+        * rather than steal it from the current listener, so we don't use it
+        * here, but rather in isc__nm_socket_reuse_lb().
         *
+        * On Windows, it also allows a socket to forcibly bind to a port in use
+        * by another socket.
         */
-       isc_result_t result = ISC_R_NOTIMPLEMENTED;
-#if defined(SO_REUSEADDR)
+
+#if defined(SO_REUSEPORT) && !defined(__linux__)
+       if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT) == -1) {
+               return (ISC_R_FAILURE);
+       }
+       return (ISC_R_SUCCESS);
+#elif defined(SO_REUSEADDR)
        if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEADDR) == -1) {
                return (ISC_R_FAILURE);
-       } else {
-               result = ISC_R_SUCCESS;
        }
+       return (ISC_R_SUCCESS);
+#else
+       UNUSED(fd);
+       return (ISC_R_NOTIMPLEMENTED);
 #endif
+}
+
+isc_result_t
+isc__nm_socket_reuse_lb(uv_os_sock_t fd) {
+       /*
+        * On FreeBSD 12+, SO_REUSEPORT_LB socket option allows sockets to be
+        * bound to an identical socket address. For UDP sockets, the use of
+        * this option can provide better distribution of incoming datagrams to
+        * multiple processes (or threads) as compared to the traditional
+        * technique of having multiple processes compete to receive datagrams
+        * on the same socket.
+        *
+        * On Linux, the same thing is achieved simply with SO_REUSEPORT.
+        */
 #if defined(SO_REUSEPORT_LB)
        if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT_LB) == -1) {
                return (ISC_R_FAILURE);
        } else {
-               result = ISC_R_SUCCESS;
+               return (ISC_R_SUCCESS);
        }
-#elif defined(SO_REUSEPORT)
+#elif defined(SO_REUSEPORT) && defined(__linux__)
        if (setsockopt_on(fd, SOL_SOCKET, SO_REUSEPORT) == -1) {
                return (ISC_R_FAILURE);
        } else {
-               result = ISC_R_SUCCESS;
+               return (ISC_R_SUCCESS);
        }
+#else
+       UNUSED(fd);
+       return (ISC_R_NOTIMPLEMENTED);
 #endif
-       return (result);
 }
 
 isc_result_t

diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c
index e531857e1c66aefea281bf4249e8d012bd330925..7056a29bf27c1f1e5f645e5764e273ba76f61aab 100644 (file)
--- a/lib/isc/netmgr/udp.c
+++ b/lib/isc/netmgr/udp.c
@@ -82,7 +82,11 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
                csock->fd = socket(family, SOCK_DGRAM, 0);
                RUNTIME_CHECK(csock->fd >= 0);
 
-               result = isc__nm_socket_reuseport(csock->fd);
+               result = isc__nm_socket_reuse(csock->fd);
+               RUNTIME_CHECK(result == ISC_R_SUCCESS ||
+                             result == ISC_R_NOTIMPLEMENTED);
+
+               result = isc__nm_socket_reuse_lb(csock->fd);
                RUNTIME_CHECK(result == ISC_R_SUCCESS ||
                              result == ISC_R_NOTIMPLEMENTED);