lib/socket: retry generate local port in nl_connect on ADDRINUSE

It can easily happen that the generated local netlink port is alrady in
use. In that case bind will fail with ADDRINUSE.

Users of libnl3 could workaround this, by managing the local ports
themselves, but sometimes these users are libraries too and they also
don't know which ports might be used by other components.

This patch changes that nl_socket_alloc() no longer initilizes the local
port id immediately. Instead it will be initialized when the user calls
nl_socket_get_local_port() the first time and thereby shows interest in
the value.

If bind() fails with ADDRINUSE, check if the user ever cared about the
local port, i.e. whether the local port is still unset. If it is still
unset, assume that libnl should choose a suitable port and retry until
an unused port can be found.

Signed-off-by: Thomas Haller <thaller@redhat.com>

diff --git a/include/Makefile.am b/include/Makefile.am
index a27857c3ff199d66b8e524eeb0b5d3d09fad3608..73d7c15d7fdee1445b1ee8e5ab9ae856f581c9cf 100644 (file)
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -142,6 +142,7 @@ noinst_HEADERS = \
        linux/tc_ematch/tc_em_meta.h \
        netlink-private/genl.h \
        netlink-private/netlink.h \
+       netlink-private/socket.h \
        netlink-private/tc.h \
        netlink-private/types.h \
        netlink-private/cache-api.h \

diff --git a/include/netlink-private/socket.h b/include/netlink-private/socket.h
new file mode 100644 (file)
index 0000000..86a440c
--- /dev/null
+++ b/include/netlink-private/socket.h
@@ -0,0 +1,31 @@
+/*
+ * netlink-private/socket.h            Private declarations for socket
+ *
+ *     This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU Lesser General Public
+ *     License as published by the Free Software Foundation version 2.1
+ *     of the License.
+ *
+ * Copyright (c) 2014 Thomas Graf <tgraf@suug.ch>
+ */
+
+#ifndef NETLINK_SOCKET_PRIV_H_
+#define NETLINK_SOCKET_PRIV_H_
+
+#include <netlink-private/netlink.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int _nl_socket_is_local_port_unspecified (struct nl_sock *sk);
+uint32_t _nl_socket_generate_local_port_no_release(struct nl_sock *sk);
+
+void _nl_socket_used_ports_release_all(const uint32_t *used_ports);
+void _nl_socket_used_ports_set(uint32_t *used_ports, uint32_t port);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

diff --git a/include/netlink/utils.h b/include/netlink/utils.h
index 8faf9177807ebb0b72b881ff85e305b23d360d2d..6b4b7874e3eaa9e3aa0e9973d31087d4b6bab004 100644 (file)
--- a/include/netlink/utils.h
+++ b/include/netlink/utils.h
@@ -104,6 +104,15 @@ enum {
        NL_CAPABILITY_ROUTE_LINK_CLS_ADD_ACT_OWN_REFERENCE = 3,
 #define NL_CAPABILITY_ROUTE_LINK_CLS_ADD_ACT_OWN_REFERENCE NL_CAPABILITY_ROUTE_LINK_CLS_ADD_ACT_OWN_REFERENCE
 
+       /**
+        * Indicate that the local port is unspecified until the user accesses
+        * it (via nl_socket_get_local_port()) or until nl_connect(). More importantly,
+        * if the port is left unspecified, nl_connect() will retry generating another
+        * port when bind() fails with ADDRINUSE.
+        */
+       NL_CAPABILITY_NL_CONNECT_RETRY_GENERATE_PORT_ON_ADDRINUSE = 4,
+#define NL_CAPABILITY_NL_CONNECT_RETRY_GENERATE_PORT_ON_ADDRINUSE NL_CAPABILITY_NL_CONNECT_RETRY_GENERATE_PORT_ON_ADDRINUSE
+
        __NL_CAPABILITY_MAX
 #define NL_CAPABILITY_MAX                               (__NL_CAPABILITY_MAX - 1)
 };

diff --git a/lib/nl.c b/lib/nl.c
index 46924907f663830a9d1f60eb40daf4054ab214ee..25fd59cfd7f894270a73388a5676910cb884571e 100644 (file)
--- a/lib/nl.c
+++ b/lib/nl.c
@@ -26,6 +26,7 @@
  */
 
 #include <netlink-private/netlink.h>
+#include <netlink-private/socket.h>
 #include <netlink/netlink.h>
 #include <netlink/utils.h>
 #include <netlink/handlers.h>
@@ -75,6 +76,16 @@
  *       be closed automatically if any of the `exec` family functions succeed.
  *       This is essential for multi threaded programs.
  *
+ * @note The local port (`nl_socket_get_local_port()`) is unspecified after
+ *       creating a new socket. It only gets determined when accessing the
+ *       port the first time or during `nl_connect()`. When nl_connect()
+ *       fails during `bind()` due to `ADDRINUSE`, it will retry with
+ *       different ports if the port is unspecified. Unless you want to enforce
+ *       the use of a specific local port, don't access the local port (or
+ *       reset it to `unspecified` by calling `nl_socket_set_local_port(sk, 0)`).
+ *       This capability is indicated by
+ *       `%NL_CAPABILITY_NL_CONNECT_RETRY_GENERATE_PORT_ON_ADDRINUSE`.
+ *
  * @see nl_socket_alloc()
  * @see nl_close()
  *
@@ -85,6 +96,7 @@
 int nl_connect(struct nl_sock *sk, int protocol)
 {
        int err, flags = 0;
+       int errsv;
        socklen_t addrlen;
 
 #ifdef SOCK_CLOEXEC
@@ -96,7 +108,9 @@ int nl_connect(struct nl_sock *sk, int protocol)
 
        sk->s_fd = socket(AF_NETLINK, SOCK_RAW | flags, protocol);
        if (sk->s_fd < 0) {
-               err = -nl_syserr2nlerr(errno);
+               errsv = errno;
+               NL_DBG(4, "nl_connect(%p): socket() failed with %d\n", sk, errsv);
+               err = -nl_syserr2nlerr(errsv);
                goto errout;
        }
 
@@ -106,11 +120,45 @@ int nl_connect(struct nl_sock *sk, int protocol)
                        goto errout;
        }
 
-       err = bind(sk->s_fd, (struct sockaddr*) &sk->s_local,
-                  sizeof(sk->s_local));
-       if (err < 0) {
-               err = -nl_syserr2nlerr(errno);
-               goto errout;
+       if (_nl_socket_is_local_port_unspecified (sk)) {
+               uint32_t port;
+               uint32_t used_ports[32] = { 0 };
+
+               while (1) {
+                       port = _nl_socket_generate_local_port_no_release(sk);
+
+                       if (port == UINT32_MAX) {
+                               NL_DBG(4, "nl_connect(%p): no more unused local ports.\n", sk);
+                               _nl_socket_used_ports_release_all(used_ports);
+                               err = -NLE_EXIST;
+                               goto errout;
+                       }
+                       err = bind(sk->s_fd, (struct sockaddr*) &sk->s_local,
+                                  sizeof(sk->s_local));
+                       if (err == 0)
+                               break;
+
+                       errsv = errno;
+                       if (errsv == EADDRINUSE) {
+                               NL_DBG(4, "nl_connect(%p): local port %u already in use. Retry.\n", sk, (unsigned) port);
+                               _nl_socket_used_ports_set(used_ports, port);
+                       } else {
+                               NL_DBG(4, "nl_connect(%p): bind() for port %u failed with %d\n", sk, (unsigned) port, errsv);
+                               _nl_socket_used_ports_release_all(used_ports);
+                               err = -nl_syserr2nlerr(errsv);
+                               goto errout;
+                       }
+               }
+               _nl_socket_used_ports_release_all(used_ports);
+       } else {
+               err = bind(sk->s_fd, (struct sockaddr*) &sk->s_local,
+                          sizeof(sk->s_local));
+               if (err != 0) {
+                       errsv = errno;
+                       NL_DBG(4, "nl_connect(%p): bind() failed with %d\n", sk, errsv);
+                       err = -nl_syserr2nlerr(errsv);
+                       goto errout;
+               }
        }
 
        addrlen = sizeof(sk->s_local);
@@ -405,7 +453,7 @@ void nl_complete_msg(struct nl_sock *sk, struct nl_msg *msg)
 
        nlh = nlmsg_hdr(msg);
        if (nlh->nlmsg_pid == NL_AUTO_PORT)
-               nlh->nlmsg_pid = sk->s_local.nl_pid;
+               nlh->nlmsg_pid = nl_socket_get_local_port(sk);
 
        if (nlh->nlmsg_seq == NL_AUTO_SEQ)
                nlh->nlmsg_seq = sk->s_seq_next++;

diff --git a/lib/socket.c b/lib/socket.c
index eb4a9785fc698f168d32ba7743239ce9ca557e79..c08f053f256d51ca76fe9161a7eac4da6e969a39 100644 (file)
--- a/lib/socket.c
+++ b/lib/socket.c
@@ -30,6 +30,7 @@
 #include "defs.h"
 
 #include <netlink-private/netlink.h>
+#include <netlink-private/socket.h>
 #include <netlink/netlink.h>
 #include <netlink/utils.h>
 #include <netlink/handlers.h>
@@ -96,17 +97,59 @@ static uint32_t generate_local_port(void)
 static void release_local_port(uint32_t port)
 {
        int nr;
+       uint32_t mask;
 
        if (port == UINT32_MAX)
                return;
-       
+
+       BUG_ON(port == 0);
+
        nr = port >> 22;
+       mask = 1UL << (nr % 32);
+       nr /= 32;
 
        nl_write_lock(&port_map_lock);
-       used_ports_map[nr / 32] &= ~(1 << (nr % 32));
+       BUG_ON((used_ports_map[nr] & mask) != mask);
+       used_ports_map[nr] &= ~mask;
        nl_write_unlock(&port_map_lock);
 }
 
+/** \cond skip */
+void _nl_socket_used_ports_release_all(const uint32_t *used_ports)
+{
+       int i;
+
+       for (i = 0; i < 32; i++) {
+               if (used_ports[i] != 0) {
+                       nl_write_lock(&port_map_lock);
+                       for (; i < 32; i++) {
+                               BUG_ON((used_ports_map[i] & used_ports[i]) != used_ports[i]);
+                               used_ports_map[i] &= ~(used_ports[i]);
+                       }
+                       nl_write_unlock(&port_map_lock);
+                       return;
+               }
+       }
+}
+
+void _nl_socket_used_ports_set(uint32_t *used_ports, uint32_t port)
+{
+       int nr;
+       int32_t mask;
+
+       nr = port >> 22;
+       mask = 1UL << (nr % 32);
+       nr /= 32;
+
+       /*
+       BUG_ON(port == UINT32_MAX || port == 0 || (getpid() & 0x3FFFFF) != (port & 0x3FFFFF));
+       BUG_ON(used_ports[nr] & mask);
+       */
+
+       used_ports[nr] |= mask;
+}
+/** \endcond */
+
 /**
  * @name Allocation
  * @{
@@ -125,7 +168,9 @@ static struct nl_sock *__alloc_socket(struct nl_cb *cb)
        sk->s_local.nl_family = AF_NETLINK;
        sk->s_peer.nl_family = AF_NETLINK;
        sk->s_seq_expect = sk->s_seq_next = time(0);
-       sk->s_local.nl_pid = generate_local_port();
+
+       /* the port is 0 (unspecified), meaning NL_OWN_PORT */
+       sk->s_flags = NL_OWN_PORT;
 
        return sk;
 }
@@ -261,6 +306,26 @@ void nl_socket_enable_auto_ack(struct nl_sock *sk)
 
 /** @} */
 
+/** \cond skip */
+int _nl_socket_is_local_port_unspecified(struct nl_sock *sk)
+{
+       return (sk->s_local.nl_pid == 0);
+}
+
+uint32_t _nl_socket_generate_local_port_no_release(struct nl_sock *sk)
+{
+       uint32_t port;
+
+       /* reset the port to generate_local_port(), but do not release
+        * the previously generated port. */
+
+       port = generate_local_port();
+       sk->s_flags &= ~NL_OWN_PORT;
+       sk->s_local.nl_pid = port;
+       return port;
+}
+/** \endcond */
+
 /**
  * @name Source Idenficiation
  * @{
@@ -268,6 +333,18 @@ void nl_socket_enable_auto_ack(struct nl_sock *sk)
 
 uint32_t nl_socket_get_local_port(const struct nl_sock *sk)
 {
+       if (sk->s_local.nl_pid == 0) {
+               /* modify the const argument sk. This is justified, because
+                * nobody ever saw the local_port from externally. So, we
+                * initilize it on first use.
+                *
+                * Note that this also means that you cannot call this function
+                * from multiple threads without synchronization. But nl_sock
+                * is not automatically threadsafe anyway, so the user is not
+                * allowed to do that.
+                */
+               return _nl_socket_generate_local_port_no_release((struct nl_sock *) sk);
+       }
        return sk->s_local.nl_pid;
 }
 
@@ -276,27 +353,18 @@ uint32_t nl_socket_get_local_port(const struct nl_sock *sk)
  * @arg sk             Netlink socket.
  * @arg port           Local port identifier
  *
- * Assigns a local port identifier to the socket. If port is 0
- * a unique port identifier will be generated automatically.
+ * Assigns a local port identifier to the socket.
+ *
+ * If port is 0, the port is reset to 'unspecified' as it is after newly
+ * calling nl_socket_alloc().
+ * Unspecified means, that the port will be generated automatically later
+ * on first use (either on nl_socket_get_local_port() or nl_connect()).
  */
 void nl_socket_set_local_port(struct nl_sock *sk, uint32_t port)
 {
-       if (port == 0) {
-               port = generate_local_port(); 
-               /*
-                * Release local port after generation of a new one to be
-                * able to change local port using nl_socket_set_local_port(, 0)
-                */
-               if (!(sk->s_flags & NL_OWN_PORT))
-                       release_local_port(sk->s_local.nl_pid);
-               else
-                       sk->s_flags &= ~NL_OWN_PORT;
-       } else  {
-               if (!(sk->s_flags & NL_OWN_PORT))
-                       release_local_port(sk->s_local.nl_pid);
-               sk->s_flags |= NL_OWN_PORT;
-       }
-
+       if (!(sk->s_flags & NL_OWN_PORT))
+               release_local_port(sk->s_local.nl_pid);
+       sk->s_flags |= NL_OWN_PORT;
        sk->s_local.nl_pid = port;
 }
 

diff --git a/lib/utils.c b/lib/utils.c
index e2294e6324351f62e682b4fa7d11ae87c316620f..5cc9e94fa0e228a482e000dcb936c6dda5648919 100644 (file)
--- a/lib/utils.c
+++ b/lib/utils.c
@@ -1147,7 +1147,7 @@ int nl_has_capability (int capability)
                        NL_CAPABILITY_ROUTE_BUILD_MSG_SET_SCOPE,
                        NL_CAPABILITY_ROUTE_LINK_VETH_GET_PEER_OWN_REFERENCE,
                        NL_CAPABILITY_ROUTE_LINK_CLS_ADD_ACT_OWN_REFERENCE,
-                       0,
+                       NL_CAPABILITY_NL_CONNECT_RETRY_GENERATE_PORT_ON_ADDRINUSE,
                        0,
                        0,
                        0,

diff --git a/libnl.sym.in b/libnl.sym.in
index e8f6c5301fe8ee4f17aac1985193cf3ca393035a..df8888c699a862b30177688e1241d65b9c97d7dd 100644 (file)
--- a/libnl.sym.in
+++ b/libnl.sym.in
@@ -1,4 +1,9 @@
 libnl_@MAJ_VERSION@ {
 global:
        *;
+local:
+       _nl_socket_generate_local_port_no_release;
+       _nl_socket_is_local_port_unspecified;
+       _nl_socket_used_ports_release_all;
+       _nl_socket_used_ports_set;
 };