fix distributor overload behaviour, closes #4311. Also adds & documents metric overlo...

diff --git a/docs/markdown/authoritative/performance.md b/docs/markdown/authoritative/performance.md
index 43645c7a5d6c437ef7ce1432aa46adacd53ca911..4197ee31cbde4c3f5f15642b98d3506a62ab342a 100644 (file)
--- a/docs/markdown/authoritative/performance.md
+++ b/docs/markdown/authoritative/performance.md
@@ -57,6 +57,7 @@ daemon.
 * `key-cache-size`: Number of entries in the key cache
 * `latency`: Average number of microseconds a packet spends within PowerDNS
 * `meta-cache-size`: Number of entries in the metadata cache
+* `overload-drops`: Number of questions dropped because backends overloaded 
 * `packetcache-hit`: Number of packets which were answered out of the cache
 * `packetcache-miss`: Number of times a packet could not be answered out of the cache
 * `packetcache-size`: Amount of packets in the packetcache

diff --git a/pdns/common_startup.cc b/pdns/common_startup.cc
index 46f3d50159f2c48a57329aefda59e25fb0721a6b..83111347da16013e3b83337b1bc968d458b1bfd1 100644 (file)
--- a/pdns/common_startup.cc
+++ b/pdns/common_startup.cc
@@ -250,6 +250,7 @@ void declareStats(void)
   S.declare("udp4-queries","Number of IPv4 UDP queries received");
   S.declare("udp6-answers","Number of IPv6 answers sent out over UDP");
   S.declare("udp6-queries","Number of IPv6 UDP queries received");
+  S.declare("overload-drops","Queries dropped because backends overloaded");
 
   S.declare("rd-queries", "Number of recursion desired questions");
   S.declare("recursion-unanswered", "Number of packets unanswered by configured recursor");
@@ -351,6 +352,7 @@ void *qthread(void *number)
   AtomicCounter &numreceived4=*S.getPointer("udp4-queries");
 
   AtomicCounter &numreceived6=*S.getPointer("udp6-queries");
+  AtomicCounter &overloadDrops=*S.getPointer("overload-drops");
 
   int diff;
   bool logDNSQueries = ::arg().mustDo("log-dns-queries");
@@ -437,7 +439,8 @@ void *qthread(void *number)
     
     if(distributor->isOverloaded()) {
       if(logDNSQueries) 
-        L<<"Dropped query, db is overloaded"<<endl;
+        L<<"Dropped query, backends are overloaded"<<endl;
+      overloadDrops++;
       continue;
     }
         

diff --git a/pdns/distributor.hh b/pdns/distributor.hh
index b5be8503a6bcfc9236e37219d5703ea6af5ecfeb..c0a769195de334366a5671980e9f47c406fae150 100644 (file)
--- a/pdns/distributor.hh
+++ b/pdns/distributor.hh
@@ -102,13 +102,13 @@ public:
 
   bool isOverloaded() override
   {
-    return d_overloaded;
+    return d_overloadQueueLength && (d_queued > d_overloadQueueLength);
   }
   
 private:
-  bool d_overloaded;
   int nextid;
   time_t d_last_started;
+  unsigned int d_overloadQueueLength, d_maxQueueLength;
   int d_num_threads;
   std::atomic<unsigned int> d_queued{0}, d_running{0};
   std::vector<std::pair<int,int>> d_pipes;
@@ -142,8 +142,8 @@ template<class Answer, class Question, class Backend>SingleThreadDistributor<Ans
 template<class Answer, class Question, class Backend>MultiThreadDistributor<Answer,Question,Backend>::MultiThreadDistributor(int n)
 {
   d_num_threads=n;
-  d_overloaded = false;
-
+  d_overloadQueueLength=::arg().asNum("overload-queue-length");
+  d_maxQueueLength=::arg().asNum("max-queue-length");
   nextid=0;
   d_last_started=time(0);
 
@@ -322,14 +322,10 @@ template<class Answer, class Question, class Backend>int MultiThreadDistributor<
     unixDie("write");
 
   d_queued++;
-  
-  static unsigned int overloadQueueLength=::arg().asNum("overload-queue-length");
-  static unsigned int maxQueueLength=::arg().asNum("max-queue-length");
 
-  if(overloadQueueLength) 
-    d_overloaded= d_queued > overloadQueueLength;
 
-  if(d_queued > maxQueueLength) {
+
+  if(d_queued > d_maxQueueLength) {
     L<<Logger::Error<< d_queued <<" questions waiting for database/backend attention. Limit is "<<::arg().asNum("max-queue-length")<<", respawning"<<endl;
     // this will leak the entire contents of all pipes, nothing will be freed. Respawn when this happens!
     throw DistributorFatal();