From 958b2bcf9600d1c631880be9688f3ac20161d3f0 Mon Sep 17 00:00:00 2001 From: Wouter Wijngaards Date: Mon, 27 Jan 2014 10:27:19 +0000 Subject: [PATCH] - reuseport is attempted, then fallback to without on failure. git-svn-id: file:///svn/unbound/trunk@3054 be551aaa-1e26-0410-a405-d3ace91eadb9 --- daemon/daemon.c | 52 ++++++++++++++++++++++++++------------ doc/Changelog | 3 +++ doc/unbound.conf.5.in | 5 +++- services/listen_dnsport.c | 52 +++++++++++++++++++++----------------- services/listen_dnsport.h | 16 ++++++++---- services/outside_network.c | 4 +-- testcode/fake_event.c | 3 ++- 7 files changed, 87 insertions(+), 48 deletions(-) diff --git a/daemon/daemon.c b/daemon/daemon.c index 44d0b87c9..790c2e7f1 100644 --- a/daemon/daemon.c +++ b/daemon/daemon.c @@ -256,32 +256,52 @@ daemon_open_shared_ports(struct daemon* daemon) log_assert(daemon); if(daemon->cfg->port != daemon->listening_port) { size_t i; -#if defined(__linux__) && defined(SO_REUSEPORT) - if(daemon->cfg->so_reuseport && daemon->cfg->num_threads > 0) - daemon->num_ports = daemon->cfg->num_threads; - else - daemon->num_ports = 1; -#else - daemon->num_ports = 1; -#endif + int reuseport = 0; + struct listen_port* p0; + /* free and close old ports */ if(daemon->ports != NULL) { for(i=0; inum_ports; i++) listening_ports_free(daemon->ports[i]); free(daemon->ports); daemon->ports = NULL; } + /* see if we want to reuseport */ +#if defined(__linux__) && defined(SO_REUSEPORT) + if(daemon->cfg->so_reuseport && daemon->cfg->num_threads > 0) + reuseport = 1; +#endif + /* try to use reuseport */ + p0 = listening_ports_open(daemon->cfg, &reuseport); + if(!p0) { + listening_ports_free(p0); + return 0; + } + if(reuseport) { + /* reuseport was successful, allocate for it */ + daemon->num_ports = daemon->cfg->num_threads; + } else { + /* do the normal, singleportslist thing, + * reuseport not enabled or did not work */ + daemon->num_ports = 1; + } if(!(daemon->ports = (struct listen_port**)calloc( daemon->num_ports, sizeof(*daemon->ports)))) { + listening_ports_free(p0); return 0; } - for(i=0; inum_ports; i++) { - if(!(daemon->ports[i]= - listening_ports_open(daemon->cfg))) { - for(i=0; inum_ports; i++) - listening_ports_free(daemon->ports[i]); - free(daemon->ports); - daemon->ports = NULL; - return 0; + daemon->ports[0] = p0; + if(reuseport) { + /* continue to use reuseport */ + for(i=1; inum_ports; i++) { + if(!(daemon->ports[i]= + listening_ports_open(daemon->cfg, + &reuseport)) || !reuseport ) { + for(i=0; inum_ports; i++) + listening_ports_free(daemon->ports[i]); + free(daemon->ports); + daemon->ports = NULL; + return 0; + } } } daemon->listening_port = daemon->cfg->port; diff --git a/doc/Changelog b/doc/Changelog index 5457b4110..fb42f0228 100644 --- a/doc/Changelog +++ b/doc/Changelog @@ -1,3 +1,6 @@ +27 January 2014: Wouter + - reuseport is attempted, then fallback to without on failure. + 24 January 2014: Wouter - Change unbound-event.h to use void* buffer, length idiom. - iana portlist updated. diff --git a/doc/unbound.conf.5.in b/doc/unbound.conf.5.in index d4ae222d5..93885fd7d 100644 --- a/doc/unbound.conf.5.in +++ b/doc/unbound.conf.5.in @@ -254,7 +254,10 @@ to so\-rcvbuf. If yes, then open dedicated listening sockets for incoming queries for each thread and try to set the SO_REUSEPORT socket option on each socket. May distribute incoming queries to threads more evenly. Default is no. Only -supported on Linux >= 3.9. +supported on Linux >= 3.9. You can enable it (on any platform and kernel), +it then attempts to open the port and passes the option if it was available +at compile time, if that works it is used, if it fails, it continues +silently (unless verbosity 3) without the option. .TP .B rrset\-cache\-size: \fI Number of bytes size of the RRset cache. Default is 4 megabytes. diff --git a/services/listen_dnsport.c b/services/listen_dnsport.c index 2c285bba1..90ba97db4 100644 --- a/services/listen_dnsport.c +++ b/services/listen_dnsport.c @@ -92,7 +92,7 @@ verbose_print_addr(struct addrinfo *addr) int create_udp_sock(int family, int socktype, struct sockaddr* addr, socklen_t addrlen, int v6only, int* inuse, int* noproto, - int rcv, int snd, int listen, int reuseport) + int rcv, int snd, int listen, int* reuseport) { int s; #if defined(SO_REUSEADDR) || defined(SO_REUSEPORT) || defined(IPV6_USE_MIN_MTU) @@ -154,15 +154,16 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr, * Each thread must have its own socket bound to the same port, * with SO_REUSEPORT set on each socket. */ - if (reuseport && + if (reuseport && *reuseport && setsockopt(s, SOL_SOCKET, SO_REUSEPORT, (void*)&on, (socklen_t)sizeof(on)) < 0) { - log_err("setsockopt(.. SO_REUSEPORT ..) failed: %s", - strerror(errno)); - close(s); - *noproto = 0; - *inuse = 0; - return -1; +#ifdef ENOPROTOOPT + if(errno != ENOPROTOOPT || verbosity >= 3) + log_warn("setsockopt(.. SO_REUSEPORT ..) failed: %s", + strerror(errno)); +#endif + /* this option is not essential, we can continue */ + *reuseport = 0; } #else (void)reuseport; @@ -431,7 +432,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr, int create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, - int reuseport) + int* reuseport) { int s; #if defined(SO_REUSEADDR) || defined(SO_REUSEPORT) || defined(IPV6_V6ONLY) @@ -478,12 +479,16 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, * Each thread must have its own socket bound to the same port, * with SO_REUSEPORT set on each socket. */ - if (reuseport && setsockopt(s, SOL_SOCKET, SO_REUSEPORT, (void*)&on, + if (reuseport && *reuseport && + setsockopt(s, SOL_SOCKET, SO_REUSEPORT, (void*)&on, (socklen_t)sizeof(on)) < 0) { - log_err("setsockopt(.. SO_REUSEPORT ..) failed: %s", - strerror(errno)); - close(s); - return -1; +#ifdef ENOPROTOOPT + if(errno != ENOPROTOOPT || verbosity >= 3) + log_warn("setsockopt(.. SO_REUSEPORT ..) failed: %s", + strerror(errno)); +#endif + /* this option is not essential, we can continue */ + *reuseport = 0; } #else (void)reuseport; @@ -556,7 +561,7 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, static int make_sock(int stype, const char* ifname, const char* port, struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd, - int reuseport) + int* reuseport) { struct addrinfo *res = NULL; int r, s, inuse, noproto; @@ -604,7 +609,7 @@ make_sock(int stype, const char* ifname, const char* port, static int make_sock_port(int stype, const char* ifname, const char* port, struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd, - int reuseport) + int* reuseport) { char* s = strchr(ifname, '@'); if(s) { @@ -721,13 +726,14 @@ set_recvpktinfo(int s, int family) * @param rcv: receive buffer size for UDP * @param snd: send buffer size for UDP * @param ssl_port: ssl service port number - * @param reuseport: try to set SO_REUSEPORT. + * @param reuseport: try to set SO_REUSEPORT if nonNULL and true. + * set to false on exit if reuseport failed due to no kernel support. * @return: returns false on error. */ static int ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp, struct addrinfo *hints, const char* port, struct listen_port** list, - size_t rcv, size_t snd, int ssl_port, int reuseport) + size_t rcv, size_t snd, int ssl_port, int* reuseport) { int s, noip6=0; if(!do_udp && !do_tcp) @@ -901,7 +907,7 @@ listen_delete(struct listen_dnsport* front) } struct listen_port* -listening_ports_open(struct config_file* cfg) +listening_ports_open(struct config_file* cfg, int* reuseport) { struct listen_port* list = NULL; struct addrinfo hints; @@ -937,7 +943,7 @@ listening_ports_open(struct config_file* cfg) do_auto, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, cfg->so_reuseport)) { + cfg->ssl_port, reuseport)) { listening_ports_free(list); return NULL; } @@ -948,7 +954,7 @@ listening_ports_open(struct config_file* cfg) do_auto, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, cfg->so_reuseport)) { + cfg->ssl_port, reuseport)) { listening_ports_free(list); return NULL; } @@ -961,7 +967,7 @@ listening_ports_open(struct config_file* cfg) if(!ports_create_if(cfg->ifs[i], 0, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, cfg->so_reuseport)) { + cfg->ssl_port, reuseport)) { listening_ports_free(list); return NULL; } @@ -972,7 +978,7 @@ listening_ports_open(struct config_file* cfg) if(!ports_create_if(cfg->ifs[i], 0, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, cfg->so_reuseport)) { + cfg->ssl_port, reuseport)) { listening_ports_free(list); return NULL; } diff --git a/services/listen_dnsport.h b/services/listen_dnsport.h index fee355401..7bcce6edb 100644 --- a/services/listen_dnsport.h +++ b/services/listen_dnsport.h @@ -107,9 +107,13 @@ struct listen_port { * interfaces for IP4 and/or IP6, for UDP and/or TCP. * On the given port number. It creates the sockets. * @param cfg: settings on what ports to open. + * @param reuseport: set to true if you want reuseport, or NULL to not have it, + * set to false on exit if reuseport failed to apply (because of no + * kernel support). * @return: linked list of ports or NULL on error. */ -struct listen_port* listening_ports_open(struct config_file* cfg); +struct listen_port* listening_ports_open(struct config_file* cfg, + int* reuseport); /** * Close and delete the (list of) listening ports. @@ -181,22 +185,24 @@ void listen_start_accept(struct listen_dnsport* listen); * @param snd: set size on sndbuf with socket option, if 0 it is not set. * @param listen: if true, this is a listening UDP port, eg port 53, and * set SO_REUSEADDR on it. - * @param reuseport: if true, try to set SO_REUSEPORT on listening UDP port. + * @param reuseport: if nonNULL and true, try to set SO_REUSEPORT on + * listening UDP port. Set to false on return if it failed to do so. * @return: the socket. -1 on error. */ int create_udp_sock(int family, int socktype, struct sockaddr* addr, socklen_t addrlen, int v6only, int* inuse, int* noproto, int rcv, - int snd, int listen, int reuseport); + int snd, int listen, int* reuseport); /** * Create and bind TCP listening socket * @param addr: address info ready to make socket. * @param v6only: enable ip6 only flag on ip6 sockets. * @param noproto: if error caused by lack of protocol support. - * @param reuseport: if true, try to set SO_REUSEPORT. + * @param reuseport: if nonNULL and true, try to set SO_REUSEPORT on + * listening UDP port. Set to false on return if it failed to do so. * @return: the socket. -1 on error. */ int create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, - int reuseport); + int* reuseport); #endif /* LISTEN_DNSPORT_H */ diff --git a/services/outside_network.c b/services/outside_network.c index 21a6effd0..cda02f1cb 100644 --- a/services/outside_network.c +++ b/services/outside_network.c @@ -849,13 +849,13 @@ udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int port, sa->sin6_port = (in_port_t)htons((uint16_t)port); fd = create_udp_sock(AF_INET6, SOCK_DGRAM, (struct sockaddr*)addr, addrlen, 1, inuse, &noproto, - 0, 0, 0, 0); + 0, 0, 0, NULL); } else { struct sockaddr_in* sa = (struct sockaddr_in*)addr; sa->sin_port = (in_port_t)htons((uint16_t)port); fd = create_udp_sock(AF_INET, SOCK_DGRAM, (struct sockaddr*)addr, addrlen, 1, inuse, &noproto, - 0, 0, 0, 0); + 0, 0, 0, NULL); } return fd; } diff --git a/testcode/fake_event.c b/testcode/fake_event.c index c52456bdf..b64ac3ad5 100644 --- a/testcode/fake_event.c +++ b/testcode/fake_event.c @@ -1142,7 +1142,8 @@ void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg) log_info("double delete of pending serviced query"); } -struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg)) +struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg), + int* ATTR_UNUSED(reuseport)) { return calloc(1, 1); } -- 2.47.2