fixes PowerDNS/pdns#666

SO_REUSEPORT is available on various bsd operating systems as standard, and also as a linux kernel patch from Google. It allows

1) Running 2 powerdns processes concurrently so that you can restart powerdns without loosing any packets
2) (the main purpose for my writing this patch) On linux with a patched kernel removes contention from many threads using a socket. In my tests this improves performance with a packet cache from 300kqps to 1mqps

If the SO_REUSEPORT call is available, the attached patch causes each receiver thread to open a new socket for connections which allows each thread (on linux) to operate at full speed rather than waiting on a slow socket. It should fail nicely ie if the call is not available at either compile time or run time it will just use the initially created socket.

Was merged in to linux 3.9 series - see https://lwn.net/Articles/542629/ for more information

diff --git a/pdns/common_startup.cc b/pdns/common_startup.cc
index eb75054918ef18c16aaf984742d2f391b9909750..af0bd6b521549f9db64dd46fb3c891bad1248f93 100644 (file)
--- a/pdns/common_startup.cc
+++ b/pdns/common_startup.cc
@@ -243,6 +243,13 @@ void *qthread(void *number)
   bool logDNSQueries = ::arg().mustDo("log-dns-queries");
   bool skipfirst=true;
   unsigned int maintcount = 0;
+  UDPNameserver *NS = N;
+
+  // If we have SO_REUSEPORT then create a new port for all receiver threads
+  // other than the first one.
+  if( number > 0 && NS->canReusePort() )
+    NS = new UDPNameserver();
+
   for(;;) {
     if (skipfirst)
       skipfirst=false;
@@ -258,7 +265,8 @@ void *qthread(void *number)
       }
     }
 
-    if(!(P=N->receive(&question))) { // receive a packet         inline
+
+    if(!(P=NS->receive(&question))) { // receive a packet         inline
       continue;                    // packet was broken, try again
     }
 
@@ -297,7 +305,7 @@ void *qthread(void *number)
       cached.d.id=P->d.id;
       cached.commitD(); // commit d to the packet                        inlined
 
-      N->send(&cached);   // answer it then                              inlined
+      NS->send(&cached);   // answer it then                              inlined
       diff=P->d_dt.udiff();                                                    
       avg_latency=(int)(0.999*avg_latency+0.001*diff); // 'EWMA'
       

diff --git a/pdns/nameserver.cc b/pdns/nameserver.cc
index c5b0bb036591996d48753b03e2c5f8a20057e490..0e773fa30029c610d935fadb5feb715284885cf7 100644 (file)
--- a/pdns/nameserver.cc
+++ b/pdns/nameserver.cc
@@ -94,6 +94,7 @@ void UDPNameserver::bindIPv4()
 {
   vector<string>locals;
   stringtok(locals,::arg()["local-address"]," ,");
+  int one = 1;
 
   if(locals.empty())
     throw PDNSException("No local address specified");
@@ -116,10 +117,14 @@ void UDPNameserver::bindIPv4()
     memset(&locala,0,sizeof(locala));
     locala.sin4.sin_family=AF_INET;
 
-    if(localname=="0.0.0.0") {
-      int val=1;
-      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &val, sizeof(val));
-    }
+    if(localname=="0.0.0.0")
+      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one));
+
+#ifdef SO_REUSEPORT
+    if( setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)) )
+      d_can_reuseport = false;
+#endif
+
     locala=ComboAddress(localname, ::arg().asNum("local-port"));
     if(locala.sin4.sin_family != AF_INET) 
       throw PDNSException("Attempting to bind IPv4 socket to IPv6 address");
@@ -188,6 +193,7 @@ void UDPNameserver::bindIPv6()
 #if HAVE_IPV6
   vector<string> locals;
   stringtok(locals,::arg()["local-ipv6"]," ,");
+  int one=1;
 
   if(locals.empty())
     return;
@@ -208,11 +214,16 @@ void UDPNameserver::bindIPv6()
     ComboAddress locala(localname, ::arg().asNum("local-port"));
     
     if(IsAnyAddress(locala)) {
-      int val=1;
-      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &val, sizeof(val));     // linux supports this, so why not - might fail on other systems
-      setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, &val, sizeof(val)); 
-      setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &val, sizeof(val));      // if this fails, we report an error in tcpreceiver too
+      setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one));     // linux supports this, so why not - might fail on other systems
+      setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one)); 
+      setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &one, sizeof(one));      // if this fails, we report an error in tcpreceiver too
     }
+
+#ifdef SO_REUSEPORT
+    if( setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)) )
+      d_can_reuseport = false;
+#endif
+
     g_localaddresses.push_back(locala);
     if(::bind(s, (sockaddr*)&locala, sizeof(locala))<0) {
       L<<Logger::Error<<"binding to UDP ipv6 socket: "<<strerror(errno)<<endl;
@@ -232,6 +243,10 @@ void UDPNameserver::bindIPv6()
 
 UDPNameserver::UDPNameserver()
 {
+#ifdef SO_REUSEPORT
+  d_can_reuseport = true;
+#endif
+
   if(!::arg()["local-address"].empty())
     bindIPv4();
   if(!::arg()["local-ipv6"].empty())

diff --git a/pdns/nameserver.hh b/pdns/nameserver.hh
index 9a9ab22b4ec6e1c6afb12d9c27aa5cfa72be06c0..642d8193450e25766ee1025bbd475f3f407567ec 100644 (file)
--- a/pdns/nameserver.hh
+++ b/pdns/nameserver.hh
@@ -71,14 +71,30 @@
 
 */
 
+#ifdef __linux__
+#ifndef SO_REUSEPORT
+#define SO_REUSEPORT 15
+#endif
+#endif
+
 class UDPNameserver
 {
 public:
   UDPNameserver();  //!< Opens the socket
   DNSPacket *receive(DNSPacket *prefilled=0); //!< call this in a while or for(;;) loop to get packets
-  static void send(DNSPacket *); //!< send a DNSPacket. Will call DNSPacket::truncate() if over 512 bytes
+  void send(DNSPacket *); //!< send a DNSPacket. Will call DNSPacket::truncate() if over 512 bytes
+  inline bool canReusePort() {
+#ifdef SO_REUSEPORT
+    return d_can_reuseport;
+#else
+    return false;
+#endif
+  };
   
 private:
+#ifdef SO_REUSEPORT
+  bool d_can_reuseport;
+#endif
   vector<int> d_sockets;
   void bindIPv4();
   void bindIPv6();