fixes PowerDNS/pdns#666

SO_REUSEPORT is available on various bsd operating systems as standard, and also as a linux kernel patch from Google. It allows

1) Running 2 powerdns processes concurrently so that you can restart powerdns without loosing any packets
2) (the main purpose for my writing this patch) On linux with a patched kernel removes contention from many threads using a socket. In my tests this improves performance with a packet cache from 300kqps to 1mqps

If the SO_REUSEPORT call is available and the reuseport config option set to yes, the attached patch causes each receiver thread to open a new socket for connections which allows each thread (on linux) to operate at full speed rather than waiting on a slow socket. It should fail nicely ie if the call is not available at either compile time or run time it will just use the initially created socket.

Was merged in to linux 3.9 series - see https://lwn.net/Articles/542629/ for more information

diff --git a/pdns/common_startup.cc b/pdns/common_startup.cc
index eb75054918ef18c16aaf984742d2f391b9909750..f3d6ea156708f4085ec8fc582749d3844303fbd4 100644 (file)
--- a/pdns/common_startup.cc
+++ b/pdns/common_startup.cc
@@ -54,6 +54,7 @@ void declareArguments()
   ::arg().set("smtpredirector","Our smtpredir MX host")="a.misconfigured.powerdns.smtp.server";
   ::arg().set("local-address","Local IP addresses to which we bind")="0.0.0.0";
   ::arg().set("local-ipv6","Local IP address to which we bind")="";
+  ::arg().setSwitch("reuseport","Enable higher performance on compliant kernels by using SO_REUSEPORT allowing each receiver thread to open its own socket")="no";
   ::arg().set("query-local-address","Source IP address for sending queries")="0.0.0.0";
   ::arg().set("query-local-address6","Source IPv6 address for sending queries")="::";
   ::arg().set("overload-queue-length","Maximum queuelength moving to packetcache only")="0";
@@ -243,6 +244,15 @@ void *qthread(void *number)
   bool logDNSQueries = ::arg().mustDo("log-dns-queries");
   bool skipfirst=true;
   unsigned int maintcount = 0;
+  UDPNameserver *NS = N;
+
+  // If we have SO_REUSEPORT then create a new port for all receiver threads
+  // other than the first one.
+  if( number > 0 && NS->canReusePort() ) {
+    L<<Logger::Notice<<"Starting new listen thread on the same IPs/ports using SO_REUSEPORT"<<endl;
+    NS = new UDPNameserver( true );
+  }
+
   for(;;) {
     if (skipfirst)
       skipfirst=false;
@@ -258,7 +268,8 @@ void *qthread(void *number)
       }
     }
 
-    if(!(P=N->receive(&question))) { // receive a packet         inline
+
+    if(!(P=NS->receive(&question))) { // receive a packet         inline
       continue;                    // packet was broken, try again
     }
 
@@ -297,7 +308,7 @@ void *qthread(void *number)
       cached.d.id=P->d.id;
       cached.commitD(); // commit d to the packet                        inlined
 
-      N->send(&cached);   // answer it then                              inlined
+      NS->send(&cached);   // answer it then                              inlined
       diff=P->d_dt.udiff();                                                    
       avg_latency=(int)(0.999*avg_latency+0.001*diff); // 'EWMA'
       

diff --git a/pdns/docs/pdns.xml b/pdns/docs/pdns.xml
index 9f9bda4468278b95eb377fee0c85e54e5ad4fc72..27a2179be97bdef5abc2d9950173e133fe50f5db 100644 (file)
--- a/pdns/docs/pdns.xml
+++ b/pdns/docs/pdns.xml
@@ -16180,6 +16180,18 @@ Tell PowerDNS to log all incoming DNS queries. This will lead to a lot of loggin
            <listitem><para>
              If this many packets are waiting for database attention, answer any new questions strictly from the packet cache.
              </para></listitem></varlistentry>
+         <varlistentry><term>reuseport=[yes|no]</term>
+           <listitem><para>
+          On Linux 3.9 and some BSD kernels the SO_REUSEPORT option allows each
+          receiver-thread to open a new socket on the same port which allows
+          for much higher performance on multi-core boxes. Setting this option
+          will enable use of SO_REUSEPORT when available and seamlessly fall
+          back to a single socket when it is not available. A side-effect is
+          that you can start multiple servers on the same IP/port combination
+          which may or may not be a good idea. You could use this to enable
+          transparent restarts, but it may also mask configuration issues and
+          for this reason it is disabled by default.
+             </para></listitem></varlistentry>
          <varlistentry>
            <term>server-id</term>
            <listitem>

diff --git a/pdns/nameserver.cc b/pdns/nameserver.cc
index 9c9eddf3267142f56c06a0db3e7e99172f0bc93f..75ab1d511c900de87082126562264966edfeba08 100644 (file)
--- a/pdns/nameserver.cc
+++ b/pdns/nameserver.cc
@@ -94,6 +94,7 @@ void UDPNameserver::bindIPv4()
 {
   vector<string>locals;
   stringtok(locals,::arg()["local-address"]," ,");
+  int one = 1;
 
   if(locals.empty())
     throw PDNSException("No local address specified");
@@ -118,15 +119,22 @@ void UDPNameserver::bindIPv4()
     memset(&locala,0,sizeof(locala));
     locala.sin4.sin_family=AF_INET;
 
-    if(localname=="0.0.0.0") {
-      int val=1;
-      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &val, sizeof(val));
-    }
+    if(localname=="0.0.0.0")
+      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one));
+
+#ifdef SO_REUSEPORT
+    if( d_can_reuseport )
+        if( setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)) )
+          d_can_reuseport = false;
+#endif
+
     locala=ComboAddress(localname, ::arg().asNum("local-port"));
     if(locala.sin4.sin_family != AF_INET) 
       throw PDNSException("Attempting to bind IPv4 socket to IPv6 address");
 
-    g_localaddresses.push_back(locala);
+    if( !d_additional_socket )
+        g_localaddresses.push_back(locala);
+
     if(::bind(s, (sockaddr*)&locala, locala.getSocklen()) < 0) {
       L<<Logger::Error<<"binding UDP socket to '"+locala.toStringWithPort()+": "<<strerror(errno)<<endl;
       throw PDNSException("Unable to bind to UDP socket");
@@ -190,6 +198,7 @@ void UDPNameserver::bindIPv6()
 #if HAVE_IPV6
   vector<string> locals;
   stringtok(locals,::arg()["local-ipv6"]," ,");
+  int one=1;
 
   if(locals.empty())
     return;
@@ -212,12 +221,19 @@ void UDPNameserver::bindIPv6()
     ComboAddress locala(localname, ::arg().asNum("local-port"));
     
     if(IsAnyAddress(locala)) {
-      int val=1;
-      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &val, sizeof(val));     // linux supports this, so why not - might fail on other systems
-      setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, &val, sizeof(val)); 
-      setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &val, sizeof(val));      // if this fails, we report an error in tcpreceiver too
+      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one));     // linux supports this, so why not - might fail on other systems
+      setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one)); 
+      setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &one, sizeof(one));      // if this fails, we report an error in tcpreceiver too
     }
-    g_localaddresses.push_back(locala);
+
+#ifdef SO_REUSEPORT
+    if( d_can_reuseport )
+        if( setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)) )
+          d_can_reuseport = false;
+#endif
+
+    if( !d_additional_socket )
+        g_localaddresses.push_back(locala);
     if(::bind(s, (sockaddr*)&locala, sizeof(locala))<0) {
       L<<Logger::Error<<"binding to UDP ipv6 socket: "<<strerror(errno)<<endl;
       throw PDNSException("Unable to bind to UDP ipv6 socket");
@@ -234,8 +250,14 @@ void UDPNameserver::bindIPv6()
 #endif
 }
 
-UDPNameserver::UDPNameserver()
+UDPNameserver::UDPNameserver( bool additional_socket )
 {
+#ifdef SO_REUSEPORT
+  d_can_reuseport = ::arg().mustDo("reuseport");
+#endif
+  // Are we the main socket (false) or a rebinding using SO_REUSEPORT ?
+  d_additional_socket = additional_socket;
+
   if(!::arg()["local-address"].empty())
     bindIPv4();
   if(!::arg()["local-ipv6"].empty())

diff --git a/pdns/nameserver.hh b/pdns/nameserver.hh
index 9a9ab22b4ec6e1c6afb12d9c27aa5cfa72be06c0..d33628296f8428fe9850b17ebe0e78890aeb3396 100644 (file)
--- a/pdns/nameserver.hh
+++ b/pdns/nameserver.hh
@@ -71,14 +71,31 @@
 
 */
 
+#ifdef __linux__
+#ifndef SO_REUSEPORT
+#define SO_REUSEPORT 15
+#endif
+#endif
+
 class UDPNameserver
 {
 public:
-  UDPNameserver();  //!< Opens the socket
+  UDPNameserver( bool additional_socket = false );  //!< Opens the socket
   DNSPacket *receive(DNSPacket *prefilled=0); //!< call this in a while or for(;;) loop to get packets
-  static void send(DNSPacket *); //!< send a DNSPacket. Will call DNSPacket::truncate() if over 512 bytes
+  void send(DNSPacket *); //!< send a DNSPacket. Will call DNSPacket::truncate() if over 512 bytes
+  inline bool canReusePort() {
+#ifdef SO_REUSEPORT
+    return d_can_reuseport;
+#else
+    return false;
+#endif
+  };
   
 private:
+  bool d_additional_socket;
+#ifdef SO_REUSEPORT
+  bool d_can_reuseport;
+#endif
   vector<int> d_sockets;
   void bindIPv4();
   void bindIPv6();