rec: Make blacklisted statistics configurable per component

This commit adds configuration parameters so it's possible to enable
or disable specific statistic metrics depending on the way they are
retrieved (API, Carbon, rec-control or SNMP).

diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc
index 80d30e5d117ef43558d325069cac5a25cba2c108..02ec7ecfdfb0d6b3c5f15263a8a485bd1da86757 100644 (file)
--- a/pdns/pdns_recursor.cc
+++ b/pdns/pdns_recursor.cc
@@ -3857,8 +3857,13 @@ static int serviceMain(int argc, char*argv[])
   g_tcpMaxQueriesPerConn=::arg().asNum("max-tcp-queries-per-connection");
   s_maxUDPQueriesPerRound=::arg().asNum("max-udp-queries-per-round");
 
+  blacklistStats(StatComponent::API, ::arg()["blacklisted-stats-api"]);
+  blacklistStats(StatComponent::Carbon, ::arg()["blacklisted-stats-carbon"]);
+  blacklistStats(StatComponent::RecControl, ::arg()["blacklisted-stats-rec-control"]);
+  blacklistStats(StatComponent::SNMP, ::arg()["blacklisted-stats-snmp"]);
+
   if (::arg().mustDo("snmp-agent")) {
-    g_snmpAgent = std::make_shared<RecursorSNMPAgent>("recursor", ::arg()["snmp-master-socket"], ::arg().mustDo("snmp-enable-expensive-stats"));
+    g_snmpAgent = std::make_shared<RecursorSNMPAgent>("recursor", ::arg()["snmp-master-socket"]);
     g_snmpAgent->run();
   }
 
@@ -4279,7 +4284,18 @@ int main(int argc, char **argv)
 
     ::arg().setSwitch("snmp-agent", "If set, register as an SNMP agent")="no";
     ::arg().set("snmp-master-socket", "If set and snmp-agent is set, the socket to use to register to the SNMP master")="";
-    ::arg().setSwitch("snmp-enable-expensive-stats", "If set and snmp-agent is set, even statistics whose reporting can have an impact on production will be enabled")="no";
+
+    std::string defaultBlacklistedStats = "cache-bytes, packetcache-bytes, special-memory-usage";
+    for (size_t idx = 0; idx < 32; idx++) {
+      defaultBlacklistedStats += ", ecs-v4-response-bits-" + std::to_string(idx + 1);
+    }
+    for (size_t idx = 0; idx < 128; idx++) {
+      defaultBlacklistedStats += ", ecs-v6-response-bits-" + std::to_string(idx + 1);
+    }
+    ::arg().set("blacklisted-stats-api", "List of statistics that are disabled when retrieving the complete list of statistics via the API")=defaultBlacklistedStats;
+    ::arg().set("blacklisted-stats-carbon", "List of statistics that are prevented from being exported via Carbon")=defaultBlacklistedStats;
+    ::arg().set("blacklisted-stats-rec-control", "List of statistics that are prevented from being exported via rec_control get-all")=defaultBlacklistedStats;
+    ::arg().set("blacklisted-stats-snmp", "List of statistics that are prevented from being exported via SNMP")=defaultBlacklistedStats;
 
     ::arg().set("tcp-fast-open", "Enable TCP Fast Open support on the listening sockets, using the supplied numerical value as the queue size")="0";
     ::arg().set("nsec3-max-iterations", "Maximum number of iterations allowed for an NSEC3 record")="2500";

diff --git a/pdns/rec-carbon.cc b/pdns/rec-carbon.cc
index b8f4e258c6355c6130e1412a0397d036aa03f2df..5f2b92fa08e42db31a8d5f65c2488bb9bf65556c 100644 (file)
--- a/pdns/rec-carbon.cc
+++ b/pdns/rec-carbon.cc
@@ -57,7 +57,7 @@ try
  
     if(msg.empty()) {
       typedef map<string,string> all_t;
-      all_t all=getAllStatsMap();
+      all_t all=getAllStatsMap(StatComponent::Carbon);
       
       ostringstream str;
       time_t now=time(0);

diff --git a/pdns/rec-snmp.cc b/pdns/rec-snmp.cc
index 41ae6b8caf8ee91da4e07b55576eeeab7af49fcc..415a36ff30de362efb4bac4ee8fda4da7f5452ae 100644 (file)
--- a/pdns/rec-snmp.cc
+++ b/pdns/rec-snmp.cc
@@ -164,7 +164,7 @@ static int handleDisabledCounter64Stats(netsnmp_mib_handler* handler,
   return RecursorSNMPAgent::setCounter64Value(requests, 0);
 }
 
-static void registerCounter64Stat(const std::string& name, const oid statOID[], size_t statOIDLength, bool expensiveStats)
+static void registerCounter64Stat(const std::string& name, const oid statOID[], size_t statOIDLength)
 {
   if (statOIDLength != OID_LENGTH(questionsOID)) {
     g_log<<Logger::Error<<"Invalid OID for SNMP Counter64 statistic "<<name<<endl;
@@ -178,7 +178,7 @@ static void registerCounter64Stat(const std::string& name, const oid statOID[],
 
   s_statsMap[statOID[statOIDLength - 1]] = name.c_str();
   netsnmp_register_scalar(netsnmp_create_handler_registration(name.c_str(),
-                                                              (expensiveStats || !isStatExpensive(name)) ? handleCounter64Stats : handleDisabledCounter64Stats,
+                                                              isStatBlacklisted(StatComponent::SNMP, name) ? handleCounter64Stats : handleDisabledCounter64Stats,
                                                               statOID,
                                                               statOIDLength,
                                                               HANDLER_CAN_RONLY));
@@ -213,112 +213,112 @@ bool RecursorSNMPAgent::sendCustomTrap(const std::string& reason)
 }
 
 
-RecursorSNMPAgent::RecursorSNMPAgent(const std::string& name, const std::string& masterSocket, bool enableExpensiveStatistics): SNMPAgent(name, masterSocket)
+RecursorSNMPAgent::RecursorSNMPAgent(const std::string& name, const std::string& masterSocket): SNMPAgent(name, masterSocket)
 {
 #ifdef HAVE_NET_SNMP
   /* This is done so that the statistics maps are
      initialized. */
   registerAllStats();
 
-  registerCounter64Stat("questions", questionsOID, OID_LENGTH(questionsOID), enableExpensiveStatistics);
-  registerCounter64Stat("ipv6-questions", ipv6QuestionsOID, OID_LENGTH(ipv6QuestionsOID), enableExpensiveStatistics);
-  registerCounter64Stat("tcp-questions", tcpQuestionsOID, OID_LENGTH(tcpQuestionsOID), enableExpensiveStatistics);
-  registerCounter64Stat("cache-hits", cacheHitsOID, OID_LENGTH(cacheHitsOID), enableExpensiveStatistics);
-  registerCounter64Stat("cache-misses", cacheMissesOID, OID_LENGTH(cacheMissesOID), enableExpensiveStatistics);
-  registerCounter64Stat("cache-entries", cacheEntriesOID, OID_LENGTH(cacheEntriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("cache-bytes", cacheBytesOID, OID_LENGTH(cacheBytesOID), enableExpensiveStatistics);
-  registerCounter64Stat("packetcache-hits", packetcacheHitsOID, OID_LENGTH(packetcacheHitsOID), enableExpensiveStatistics);
-  registerCounter64Stat("packetcache-misses", packetcacheMissesOID, OID_LENGTH(packetcacheMissesOID), enableExpensiveStatistics);
-  registerCounter64Stat("packetcache-entries", packetcacheEntriesOID, OID_LENGTH(packetcacheEntriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("packetcache-bytes", packetcacheBytesOID, OID_LENGTH(packetcacheBytesOID), enableExpensiveStatistics);
-  registerCounter64Stat("malloc-bytes", mallocBytesOID, OID_LENGTH(mallocBytesOID), enableExpensiveStatistics);
-  registerCounter64Stat("servfail-answers", servfailAnswersOID, OID_LENGTH(servfailAnswersOID), enableExpensiveStatistics);
-  registerCounter64Stat("nxdomain-answers", nxdomainAnswersOID, OID_LENGTH(nxdomainAnswersOID), enableExpensiveStatistics);
-  registerCounter64Stat("noerror-answers", noerrorAnswersOID, OID_LENGTH(noerrorAnswersOID), enableExpensiveStatistics);
-  registerCounter64Stat("unauthorized-udp", unauthorizedUdpOID, OID_LENGTH(unauthorizedUdpOID), enableExpensiveStatistics);
-  registerCounter64Stat("unauthorized-tcp", unauthorizedTcpOID, OID_LENGTH(unauthorizedTcpOID), enableExpensiveStatistics);
-  registerCounter64Stat("tcp-client-overflow", tcpClientOverflowOID, OID_LENGTH(tcpClientOverflowOID), enableExpensiveStatistics);
-  registerCounter64Stat("client-parse-errors", clientParseErrorsOID, OID_LENGTH(clientParseErrorsOID), enableExpensiveStatistics);
-  registerCounter64Stat("server-parse-errors", serverParseErrorsOID, OID_LENGTH(serverParseErrorsOID), enableExpensiveStatistics);
-  registerCounter64Stat("too-old-drops", tooOldDropsOID, OID_LENGTH(tooOldDropsOID), enableExpensiveStatistics);
-  registerCounter64Stat("query-pipe-full-drops", queryPipeFullDropsOID, OID_LENGTH(queryPipeFullDropsOID), enableExpensiveStatistics);
-  registerCounter64Stat("truncated-drops", truncatedDropsOID, OID_LENGTH(truncatedDropsOID), enableExpensiveStatistics);
-  registerCounter64Stat("empty-queries", emptyQueriesOID, OID_LENGTH(emptyQueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("variable-responses", variableResponsesOID, OID_LENGTH(variableResponsesOID), enableExpensiveStatistics);
-  registerCounter64Stat("answers0-1", answers01OID, OID_LENGTH(answers01OID), enableExpensiveStatistics);
-  registerCounter64Stat("answers1-10", answers110OID, OID_LENGTH(answers110OID), enableExpensiveStatistics);
-  registerCounter64Stat("answers10-100", answers10100OID, OID_LENGTH(answers10100OID), enableExpensiveStatistics);
-  registerCounter64Stat("answers100-1000", answers1001000OID, OID_LENGTH(answers1001000OID), enableExpensiveStatistics);
-  registerCounter64Stat("answers-slow", answersSlowOID, OID_LENGTH(answersSlowOID), enableExpensiveStatistics);
-  registerCounter64Stat("auth4-answers0-1", auth4Answers01OID, OID_LENGTH(auth4Answers01OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth4-answers1-10", auth4Answers110OID, OID_LENGTH(auth4Answers110OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth4-answers10-100", auth4Answers10100OID, OID_LENGTH(auth4Answers10100OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth4-answers100-1000", auth4Answers1001000OID, OID_LENGTH(auth4Answers1001000OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth4-answers-slow", auth4AnswersslowOID, OID_LENGTH(auth4AnswersslowOID), enableExpensiveStatistics);
-  registerCounter64Stat("auth6-answers0-1", auth6Answers01OID, OID_LENGTH(auth6Answers01OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth6-answers1-10", auth6Answers110OID, OID_LENGTH(auth6Answers110OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth6-answers10-100", auth6Answers10100OID, OID_LENGTH(auth6Answers10100OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth6-answers100-1000", auth6Answers1001000OID, OID_LENGTH(auth6Answers1001000OID), enableExpensiveStatistics);
-  registerCounter64Stat("auth6-answers-slow", auth6AnswersSlowOID, OID_LENGTH(auth6AnswersSlowOID), enableExpensiveStatistics);
-  registerCounter64Stat("qa-latency", qaLatencyOID, OID_LENGTH(qaLatencyOID), enableExpensiveStatistics);
-  registerCounter64Stat("unexpected-packets", unexpectedPacketsOID, OID_LENGTH(unexpectedPacketsOID), enableExpensiveStatistics);
-  registerCounter64Stat("case-mismatches", caseMismatchesOID, OID_LENGTH(caseMismatchesOID), enableExpensiveStatistics);
-  registerCounter64Stat("spoof-prevents", spoofPreventsOID, OID_LENGTH(spoofPreventsOID), enableExpensiveStatistics);
-  registerCounter64Stat("nsset-invalidations", nssetInvalidationsOID, OID_LENGTH(nssetInvalidationsOID), enableExpensiveStatistics);
-  registerCounter64Stat("resource-limits", resourceLimitsOID, OID_LENGTH(resourceLimitsOID), enableExpensiveStatistics);
-  registerCounter64Stat("over-capacity-drops", overCapacityDropsOID, OID_LENGTH(overCapacityDropsOID), enableExpensiveStatistics);
-  registerCounter64Stat("policy-drops", policyDropsOID, OID_LENGTH(policyDropsOID), enableExpensiveStatistics);
-  registerCounter64Stat("no-packet-error", noPacketErrorOID, OID_LENGTH(noPacketErrorOID), enableExpensiveStatistics);
-  registerCounter64Stat("dlg-only-drops", dlgOnlyDropsOID, OID_LENGTH(dlgOnlyDropsOID), enableExpensiveStatistics);
-  registerCounter64Stat("ignored-packets", ignoredPacketsOID, OID_LENGTH(ignoredPacketsOID), enableExpensiveStatistics);
-  registerCounter64Stat("max-mthread-stack", maxMthreadStackOID, OID_LENGTH(maxMthreadStackOID), enableExpensiveStatistics);
-  registerCounter64Stat("negcache-entries", negcacheEntriesOID, OID_LENGTH(negcacheEntriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("throttle-entries", throttleEntriesOID, OID_LENGTH(throttleEntriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("nsspeeds-entries", nsspeedsEntriesOID, OID_LENGTH(nsspeedsEntriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("failed-host-entries", failedHostEntriesOID, OID_LENGTH(failedHostEntriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("concurrent-queries", concurrentQueriesOID, OID_LENGTH(concurrentQueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("security-status", securityStatusOID, OID_LENGTH(securityStatusOID), enableExpensiveStatistics);
-  registerCounter64Stat("outgoing-timeouts", outgoingTimeoutsOID, OID_LENGTH(outgoingTimeoutsOID), enableExpensiveStatistics);
-  registerCounter64Stat("outgoing4-timeouts", outgoing4TimeoutsOID, OID_LENGTH(outgoing4TimeoutsOID), enableExpensiveStatistics);
-  registerCounter64Stat("outgoing6-timeouts", outgoing6TimeoutsOID, OID_LENGTH(outgoing6TimeoutsOID), enableExpensiveStatistics);
-  registerCounter64Stat("tcp-outqueries", tcpOutqueriesOID, OID_LENGTH(tcpOutqueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("all-outqueries", allOutqueriesOID, OID_LENGTH(allOutqueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("ipv6-outqueries", ipv6OutqueriesOID, OID_LENGTH(ipv6OutqueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("throttled-outqueries", throttledOutqueriesOID, OID_LENGTH(throttledOutqueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("dont-outqueries", dontOutqueriesOID, OID_LENGTH(dontOutqueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("unreachables", unreachablesOID, OID_LENGTH(unreachablesOID), enableExpensiveStatistics);
-  registerCounter64Stat("chain-resends", chainResendsOID, OID_LENGTH(chainResendsOID), enableExpensiveStatistics);
-  registerCounter64Stat("tcp-clients", tcpClientsOID, OID_LENGTH(tcpClientsOID), enableExpensiveStatistics);
+  registerCounter64Stat("questions", questionsOID, OID_LENGTH(questionsOID));
+  registerCounter64Stat("ipv6-questions", ipv6QuestionsOID, OID_LENGTH(ipv6QuestionsOID));
+  registerCounter64Stat("tcp-questions", tcpQuestionsOID, OID_LENGTH(tcpQuestionsOID));
+  registerCounter64Stat("cache-hits", cacheHitsOID, OID_LENGTH(cacheHitsOID));
+  registerCounter64Stat("cache-misses", cacheMissesOID, OID_LENGTH(cacheMissesOID));
+  registerCounter64Stat("cache-entries", cacheEntriesOID, OID_LENGTH(cacheEntriesOID));
+  registerCounter64Stat("cache-bytes", cacheBytesOID, OID_LENGTH(cacheBytesOID));
+  registerCounter64Stat("packetcache-hits", packetcacheHitsOID, OID_LENGTH(packetcacheHitsOID));
+  registerCounter64Stat("packetcache-misses", packetcacheMissesOID, OID_LENGTH(packetcacheMissesOID));
+  registerCounter64Stat("packetcache-entries", packetcacheEntriesOID, OID_LENGTH(packetcacheEntriesOID));
+  registerCounter64Stat("packetcache-bytes", packetcacheBytesOID, OID_LENGTH(packetcacheBytesOID));
+  registerCounter64Stat("malloc-bytes", mallocBytesOID, OID_LENGTH(mallocBytesOID));
+  registerCounter64Stat("servfail-answers", servfailAnswersOID, OID_LENGTH(servfailAnswersOID));
+  registerCounter64Stat("nxdomain-answers", nxdomainAnswersOID, OID_LENGTH(nxdomainAnswersOID));
+  registerCounter64Stat("noerror-answers", noerrorAnswersOID, OID_LENGTH(noerrorAnswersOID));
+  registerCounter64Stat("unauthorized-udp", unauthorizedUdpOID, OID_LENGTH(unauthorizedUdpOID));
+  registerCounter64Stat("unauthorized-tcp", unauthorizedTcpOID, OID_LENGTH(unauthorizedTcpOID));
+  registerCounter64Stat("tcp-client-overflow", tcpClientOverflowOID, OID_LENGTH(tcpClientOverflowOID));
+  registerCounter64Stat("client-parse-errors", clientParseErrorsOID, OID_LENGTH(clientParseErrorsOID));
+  registerCounter64Stat("server-parse-errors", serverParseErrorsOID, OID_LENGTH(serverParseErrorsOID));
+  registerCounter64Stat("too-old-drops", tooOldDropsOID, OID_LENGTH(tooOldDropsOID));
+  registerCounter64Stat("query-pipe-full-drops", queryPipeFullDropsOID, OID_LENGTH(queryPipeFullDropsOID));
+  registerCounter64Stat("truncated-drops", truncatedDropsOID, OID_LENGTH(truncatedDropsOID));
+  registerCounter64Stat("empty-queries", emptyQueriesOID, OID_LENGTH(emptyQueriesOID));
+  registerCounter64Stat("variable-responses", variableResponsesOID, OID_LENGTH(variableResponsesOID));
+  registerCounter64Stat("answers0-1", answers01OID, OID_LENGTH(answers01OID));
+  registerCounter64Stat("answers1-10", answers110OID, OID_LENGTH(answers110OID));
+  registerCounter64Stat("answers10-100", answers10100OID, OID_LENGTH(answers10100OID));
+  registerCounter64Stat("answers100-1000", answers1001000OID, OID_LENGTH(answers1001000OID));
+  registerCounter64Stat("answers-slow", answersSlowOID, OID_LENGTH(answersSlowOID));
+  registerCounter64Stat("auth4-answers0-1", auth4Answers01OID, OID_LENGTH(auth4Answers01OID));
+  registerCounter64Stat("auth4-answers1-10", auth4Answers110OID, OID_LENGTH(auth4Answers110OID));
+  registerCounter64Stat("auth4-answers10-100", auth4Answers10100OID, OID_LENGTH(auth4Answers10100OID));
+  registerCounter64Stat("auth4-answers100-1000", auth4Answers1001000OID, OID_LENGTH(auth4Answers1001000OID));
+  registerCounter64Stat("auth4-answers-slow", auth4AnswersslowOID, OID_LENGTH(auth4AnswersslowOID));
+  registerCounter64Stat("auth6-answers0-1", auth6Answers01OID, OID_LENGTH(auth6Answers01OID));
+  registerCounter64Stat("auth6-answers1-10", auth6Answers110OID, OID_LENGTH(auth6Answers110OID));
+  registerCounter64Stat("auth6-answers10-100", auth6Answers10100OID, OID_LENGTH(auth6Answers10100OID));
+  registerCounter64Stat("auth6-answers100-1000", auth6Answers1001000OID, OID_LENGTH(auth6Answers1001000OID));
+  registerCounter64Stat("auth6-answers-slow", auth6AnswersSlowOID, OID_LENGTH(auth6AnswersSlowOID));
+  registerCounter64Stat("qa-latency", qaLatencyOID, OID_LENGTH(qaLatencyOID));
+  registerCounter64Stat("unexpected-packets", unexpectedPacketsOID, OID_LENGTH(unexpectedPacketsOID));
+  registerCounter64Stat("case-mismatches", caseMismatchesOID, OID_LENGTH(caseMismatchesOID));
+  registerCounter64Stat("spoof-prevents", spoofPreventsOID, OID_LENGTH(spoofPreventsOID));
+  registerCounter64Stat("nsset-invalidations", nssetInvalidationsOID, OID_LENGTH(nssetInvalidationsOID));
+  registerCounter64Stat("resource-limits", resourceLimitsOID, OID_LENGTH(resourceLimitsOID));
+  registerCounter64Stat("over-capacity-drops", overCapacityDropsOID, OID_LENGTH(overCapacityDropsOID));
+  registerCounter64Stat("policy-drops", policyDropsOID, OID_LENGTH(policyDropsOID));
+  registerCounter64Stat("no-packet-error", noPacketErrorOID, OID_LENGTH(noPacketErrorOID));
+  registerCounter64Stat("dlg-only-drops", dlgOnlyDropsOID, OID_LENGTH(dlgOnlyDropsOID));
+  registerCounter64Stat("ignored-packets", ignoredPacketsOID, OID_LENGTH(ignoredPacketsOID));
+  registerCounter64Stat("max-mthread-stack", maxMthreadStackOID, OID_LENGTH(maxMthreadStackOID));
+  registerCounter64Stat("negcache-entries", negcacheEntriesOID, OID_LENGTH(negcacheEntriesOID));
+  registerCounter64Stat("throttle-entries", throttleEntriesOID, OID_LENGTH(throttleEntriesOID));
+  registerCounter64Stat("nsspeeds-entries", nsspeedsEntriesOID, OID_LENGTH(nsspeedsEntriesOID));
+  registerCounter64Stat("failed-host-entries", failedHostEntriesOID, OID_LENGTH(failedHostEntriesOID));
+  registerCounter64Stat("concurrent-queries", concurrentQueriesOID, OID_LENGTH(concurrentQueriesOID));
+  registerCounter64Stat("security-status", securityStatusOID, OID_LENGTH(securityStatusOID));
+  registerCounter64Stat("outgoing-timeouts", outgoingTimeoutsOID, OID_LENGTH(outgoingTimeoutsOID));
+  registerCounter64Stat("outgoing4-timeouts", outgoing4TimeoutsOID, OID_LENGTH(outgoing4TimeoutsOID));
+  registerCounter64Stat("outgoing6-timeouts", outgoing6TimeoutsOID, OID_LENGTH(outgoing6TimeoutsOID));
+  registerCounter64Stat("tcp-outqueries", tcpOutqueriesOID, OID_LENGTH(tcpOutqueriesOID));
+  registerCounter64Stat("all-outqueries", allOutqueriesOID, OID_LENGTH(allOutqueriesOID));
+  registerCounter64Stat("ipv6-outqueries", ipv6OutqueriesOID, OID_LENGTH(ipv6OutqueriesOID));
+  registerCounter64Stat("throttled-outqueries", throttledOutqueriesOID, OID_LENGTH(throttledOutqueriesOID));
+  registerCounter64Stat("dont-outqueries", dontOutqueriesOID, OID_LENGTH(dontOutqueriesOID));
+  registerCounter64Stat("unreachables", unreachablesOID, OID_LENGTH(unreachablesOID));
+  registerCounter64Stat("chain-resends", chainResendsOID, OID_LENGTH(chainResendsOID));
+  registerCounter64Stat("tcp-clients", tcpClientsOID, OID_LENGTH(tcpClientsOID));
 #ifdef __linux__
-  registerCounter64Stat("udp-recvbuf-errors", udpRecvbufErrorsOID, OID_LENGTH(udpRecvbufErrorsOID), enableExpensiveStatistics);
-  registerCounter64Stat("udp-sndbuf-errors", udpSndbufErrorsOID, OID_LENGTH(udpSndbufErrorsOID), enableExpensiveStatistics);
-  registerCounter64Stat("udp-noport-errors", udpNoportErrorsOID, OID_LENGTH(udpNoportErrorsOID), enableExpensiveStatistics);
-  registerCounter64Stat("udp-in-errors", udpinErrorsOID, OID_LENGTH(udpinErrorsOID), enableExpensiveStatistics);
+  registerCounter64Stat("udp-recvbuf-errors", udpRecvbufErrorsOID, OID_LENGTH(udpRecvbufErrorsOID));
+  registerCounter64Stat("udp-sndbuf-errors", udpSndbufErrorsOID, OID_LENGTH(udpSndbufErrorsOID));
+  registerCounter64Stat("udp-noport-errors", udpNoportErrorsOID, OID_LENGTH(udpNoportErrorsOID));
+  registerCounter64Stat("udp-in-errors", udpinErrorsOID, OID_LENGTH(udpinErrorsOID));
 #endif /* __linux__ */
-  registerCounter64Stat("edns-ping-matches", ednsPingMatchesOID, OID_LENGTH(ednsPingMatchesOID), enableExpensiveStatistics);
-  registerCounter64Stat("edns-ping-mismatches", ednsPingMismatchesOID, OID_LENGTH(ednsPingMismatchesOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-queries", dnssecQueriesOID, OID_LENGTH(dnssecQueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-authentic-data-queries", dnssecAuthenticDataQueriesOID, OID_LENGTH(dnssecAuthenticDataQueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-check-disabled-queries", dnssecCheckDisabledQueriesOID, OID_LENGTH(dnssecCheckDisabledQueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("noping-outqueries", nopingOutqueriesOID, OID_LENGTH(nopingOutqueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("noedns-outqueries", noednsOutqueriesOID, OID_LENGTH(noednsOutqueriesOID), enableExpensiveStatistics);
-  registerCounter64Stat("uptime", uptimeOID, OID_LENGTH(uptimeOID), enableExpensiveStatistics);
-  registerCounter64Stat("real-memory-usage", realMemoryUsageOID, OID_LENGTH(realMemoryUsageOID), enableExpensiveStatistics);
-  registerCounter64Stat("fd-usage", fdUsageOID, OID_LENGTH(fdUsageOID), enableExpensiveStatistics);
-  registerCounter64Stat("user-msec", userMsecOID, OID_LENGTH(userMsecOID), enableExpensiveStatistics);
-  registerCounter64Stat("sys-msec", sysMsecOID, OID_LENGTH(sysMsecOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-validations", dnssecValidationsOID, OID_LENGTH(dnssecValidationsOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-result-insecure", dnssecResultInsecureOID, OID_LENGTH(dnssecResultInsecureOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-result-secure", dnssecResultSecureOID, OID_LENGTH(dnssecResultSecureOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-result-bogus", dnssecResultBogusOID, OID_LENGTH(dnssecResultBogusOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-result-indeterminate", dnssecResultIndeterminateOID, OID_LENGTH(dnssecResultIndeterminateOID), enableExpensiveStatistics);
-  registerCounter64Stat("dnssec-result-nta", dnssecResultNtaOID, OID_LENGTH(dnssecResultNtaOID), enableExpensiveStatistics);
-  registerCounter64Stat("policy-result-noaction", policyResultNoactionOID, OID_LENGTH(policyResultNoactionOID), enableExpensiveStatistics);
-  registerCounter64Stat("policy-result-drop", policyResultDropOID, OID_LENGTH(policyResultDropOID), enableExpensiveStatistics);
-  registerCounter64Stat("policy-result-nxdomain", policyResultNxdomainOID, OID_LENGTH(policyResultNxdomainOID), enableExpensiveStatistics);
-  registerCounter64Stat("policy-result-nodata", policyResultNodataOID, OID_LENGTH(policyResultNodataOID), enableExpensiveStatistics);
-  registerCounter64Stat("policy-result-truncate", policyResultTruncateOID, OID_LENGTH(policyResultTruncateOID), enableExpensiveStatistics);
-  registerCounter64Stat("policy-result-custom", policyResultCustomOID, OID_LENGTH(policyResultCustomOID), enableExpensiveStatistics);
-  registerCounter64Stat("special-memory-usage", specialMemoryUsageOID, OID_LENGTH(specialMemoryUsageOID), enableExpensiveStatistics);
+  registerCounter64Stat("edns-ping-matches", ednsPingMatchesOID, OID_LENGTH(ednsPingMatchesOID));
+  registerCounter64Stat("edns-ping-mismatches", ednsPingMismatchesOID, OID_LENGTH(ednsPingMismatchesOID));
+  registerCounter64Stat("dnssec-queries", dnssecQueriesOID, OID_LENGTH(dnssecQueriesOID));
+  registerCounter64Stat("dnssec-authentic-data-queries", dnssecAuthenticDataQueriesOID, OID_LENGTH(dnssecAuthenticDataQueriesOID));
+  registerCounter64Stat("dnssec-check-disabled-queries", dnssecCheckDisabledQueriesOID, OID_LENGTH(dnssecCheckDisabledQueriesOID));
+  registerCounter64Stat("noping-outqueries", nopingOutqueriesOID, OID_LENGTH(nopingOutqueriesOID));
+  registerCounter64Stat("noedns-outqueries", noednsOutqueriesOID, OID_LENGTH(noednsOutqueriesOID));
+  registerCounter64Stat("uptime", uptimeOID, OID_LENGTH(uptimeOID));
+  registerCounter64Stat("real-memory-usage", realMemoryUsageOID, OID_LENGTH(realMemoryUsageOID));
+  registerCounter64Stat("fd-usage", fdUsageOID, OID_LENGTH(fdUsageOID));
+  registerCounter64Stat("user-msec", userMsecOID, OID_LENGTH(userMsecOID));
+  registerCounter64Stat("sys-msec", sysMsecOID, OID_LENGTH(sysMsecOID));
+  registerCounter64Stat("dnssec-validations", dnssecValidationsOID, OID_LENGTH(dnssecValidationsOID));
+  registerCounter64Stat("dnssec-result-insecure", dnssecResultInsecureOID, OID_LENGTH(dnssecResultInsecureOID));
+  registerCounter64Stat("dnssec-result-secure", dnssecResultSecureOID, OID_LENGTH(dnssecResultSecureOID));
+  registerCounter64Stat("dnssec-result-bogus", dnssecResultBogusOID, OID_LENGTH(dnssecResultBogusOID));
+  registerCounter64Stat("dnssec-result-indeterminate", dnssecResultIndeterminateOID, OID_LENGTH(dnssecResultIndeterminateOID));
+  registerCounter64Stat("dnssec-result-nta", dnssecResultNtaOID, OID_LENGTH(dnssecResultNtaOID));
+  registerCounter64Stat("policy-result-noaction", policyResultNoactionOID, OID_LENGTH(policyResultNoactionOID));
+  registerCounter64Stat("policy-result-drop", policyResultDropOID, OID_LENGTH(policyResultDropOID));
+  registerCounter64Stat("policy-result-nxdomain", policyResultNxdomainOID, OID_LENGTH(policyResultNxdomainOID));
+  registerCounter64Stat("policy-result-nodata", policyResultNodataOID, OID_LENGTH(policyResultNodataOID));
+  registerCounter64Stat("policy-result-truncate", policyResultTruncateOID, OID_LENGTH(policyResultTruncateOID));
+  registerCounter64Stat("policy-result-custom", policyResultCustomOID, OID_LENGTH(policyResultCustomOID));
+  registerCounter64Stat("special-memory-usage", specialMemoryUsageOID, OID_LENGTH(specialMemoryUsageOID));
 #endif /* HAVE_NET_SNMP */
 }

diff --git a/pdns/rec-snmp.hh b/pdns/rec-snmp.hh
index 745d7fc89f94c677e8f93cbbaac1cac3b2b3d6ec..3ccc73fb1380508eda86422767ea067c11f5329d 100644 (file)
--- a/pdns/rec-snmp.hh
+++ b/pdns/rec-snmp.hh
@@ -10,7 +10,7 @@ class RecursorSNMPAgent;
 class RecursorSNMPAgent: public SNMPAgent
 {
 public:
-  RecursorSNMPAgent(const std::string& name, const std::string& masterSocket, bool enableExpensiveStatistics);
+  RecursorSNMPAgent(const std::string& name, const std::string& masterSocket);
   bool sendCustomTrap(const std::string& reason);
 };
 

diff --git a/pdns/rec_channel.hh b/pdns/rec_channel.hh
index 9f40f9d46c23fed14e908ef055f64c99541c7345..2eb6fb8fd6cd5bf0cbf84dd6aa2ea8521ab236d7 100644 (file)
--- a/pdns/rec_channel.hh
+++ b/pdns/rec_channel.hh
@@ -64,7 +64,9 @@ public:
   std::string getAnswer(const std::string& question, func_t** func);
 };
 
-std::map<std::string, std::string> getAllStatsMap();
+enum class StatComponent { API, Carbon, RecControl, SNMP };
+
+std::map<std::string, std::string> getAllStatsMap(StatComponent component);
 extern pthread_mutex_t g_carbon_config_lock;
 std::vector<std::pair<DNSName, uint16_t> >* pleaseGetQueryRing();
 std::vector<std::pair<DNSName, uint16_t> >* pleaseGetServfailQueryRing();
@@ -77,6 +79,9 @@ std::vector<ComboAddress>* pleaseGetTimeouts();
 DNSName getRegisteredName(const DNSName& dom);
 std::atomic<unsigned long>* getDynMetric(const std::string& str);
 optional<uint64_t> getStatByName(const std::string& name);
-bool isStatExpensive(const std::string& name);
+bool isStatBlacklisted(StatComponent component, const std::string& name);
+void blacklistStat(StatComponent component, const string& name);
+void blacklistStats(StatComponent component, const string& stats);
+
 void registerAllStats();
 

diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc
index 488697b19f0cc7838df4ad3e2b90756db9b2138c..4b101b638a3f14e5a74d8ce8866163a2300d5ad9 100644 (file)
--- a/pdns/rec_channel_rec.cc
+++ b/pdns/rec_channel_rec.cc
@@ -45,16 +45,26 @@ static map<string, function< uint64_t() > >  d_get64bitmembers;
 static pthread_mutex_t d_dynmetricslock = PTHREAD_MUTEX_INITIALIZER;
 static map<string, std::atomic<unsigned long>* > d_dynmetrics;
 
-static std::set<std::string> s_expensiveStats = { "cache-bytes", "packetcache-bytes", "special-memory-usage" };
+static std::map<StatComponent, std::set<std::string>> s_blacklistedStats;
 
-bool isStatExpensive(const string& name)
+bool isStatBlacklisted(StatComponent component, const string& name)
 {
-  return s_expensiveStats.count(name) != 0;
+  return s_blacklistedStats[component].count(name) != 0;
 }
 
-void markStatAsExpensive(const string& name)
+void blacklistStat(StatComponent component, const string& name)
 {
-  s_expensiveStats.insert(name);
+  s_blacklistedStats[component].insert(name);
+}
+
+void blacklistStats(StatComponent component, const string& stats)
+{
+  std::vector<std::string> blacklistedStats;
+  stringtok(blacklistedStats, stats, ", ");
+  auto& map = s_blacklistedStats[component];
+  for (const auto &st : blacklistedStats) {
+    map.insert(st);
+  }
 }
 
 static void addGetStat(const string& name, const uint32_t* place)
@@ -108,37 +118,44 @@ optional<uint64_t> getStatByName(const std::string& name)
   return get(name);
 }
 
-map<string,string> getAllStatsMap()
+map<string,string> getAllStatsMap(StatComponent component)
 {
   map<string,string> ret;
-  
+  const auto& blacklistMap = s_blacklistedStats.at(component);
+
   for(const auto& the32bits :  d_get32bitpointers) {
-    if (!isStatExpensive(the32bits.first)) {
+    if (blacklistMap.count(the32bits.first) == 0) {
       ret.insert(make_pair(the32bits.first, std::to_string(*the32bits.second)));
     }
   }
   for(const auto& atomic :  d_getatomics) {
-    if (!isStatExpensive(atomic.first)) {
+    if (blacklistMap.count(atomic.first) == 0) {
       ret.insert(make_pair(atomic.first, std::to_string(atomic.second->load())));
     }
   }
 
-  for(const auto& the64bitmembers :  d_get64bitmembers) { 
-    if (!isStatExpensive(the64bitmembers.first)) {
+  for(const auto& the64bitmembers :  d_get64bitmembers) {
+    if (blacklistMap.count(the64bitmembers.first) == 0) {
       ret.insert(make_pair(the64bitmembers.first, std::to_string(the64bitmembers.second())));
     }
   }
 
-  Lock l(&d_dynmetricslock);
-  for(const auto& a : d_dynmetrics)
-    ret.insert({a.first, std::to_string(*a.second)});
+  {
+    Lock l(&d_dynmetricslock);
+    for(const auto& a : d_dynmetrics) {
+      if (blacklistMap.count(a.first) == 0) {
+        ret.insert({a.first, std::to_string(*a.second)});
+      }
+    }
+  }
+
   return ret;
 }
 
-string getAllStats()
+static string getAllStats()
 {
   typedef map<string, string> varmap_t;
-  varmap_t varmap = getAllStatsMap();
+  varmap_t varmap = getAllStatsMap(StatComponent::RecControl);
   string ret;
   for(varmap_t::value_type& tup :  varmap) {
     ret += tup.first + "\t" + tup.second +"\n";
@@ -1061,12 +1078,10 @@ void registerAllStats()
   for (size_t idx = 0; idx < SyncRes::s_ecsResponsesBySubnetSize4.size(); idx++) {
     const std::string name = "ecs-v4-response-bits-" + std::to_string(idx + 1);
     addGetStat(name, &(SyncRes::s_ecsResponsesBySubnetSize4.at(idx)));
-    markStatAsExpensive(name);
   }
   for (size_t idx = 0; idx < SyncRes::s_ecsResponsesBySubnetSize6.size(); idx++) {
     const std::string name = "ecs-v6-response-bits-" + std::to_string(idx + 1);
     addGetStat(name, &(SyncRes::s_ecsResponsesBySubnetSize6.at(idx)));
-    markStatAsExpensive(name);
   }
 }
 

diff --git a/pdns/recursordist/docs/settings.rst b/pdns/recursordist/docs/settings.rst
index 803c6fedea3a948e5e0a796cc0b2b3189af6f5ab..646a28970c6db44b6a63469650e06cd95b6fecfc 100644 (file)
--- a/pdns/recursordist/docs/settings.rst
+++ b/pdns/recursordist/docs/settings.rst
@@ -126,6 +126,46 @@ DNSSEC is not supported. Example:
 
     auth-zones=example.org=/var/zones/example.org, powerdns.com=/var/zones/powerdns.com
 
+.. _setting-blacklisted-stats-api:
+
+``blacklisted-stats-api``
+-------------------------
+.. versionadded:: 4.2.0
+
+-  String
+-  Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*"
+
+A list of comma-separated statistic names, that are disabled when retrieving the complete list of statistics via the API for performance reasons.
+These statistics can still be retrieved individually by specifically asking for it.
+
+``blacklisted-stats-carbon``
+---------------------------
+.. versionadded:: 4.2.0
+
+-  String
+-  Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*"
+
+A list of comma-separated statistic names, that are prevented from being exported via carbon for performance reasons.
+
+``blacklisted-stats-rec-control``
+---------------------------------
+.. versionadded:: 4.2.0
+
+-  String
+-  Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*"
+
+A list of comma-separated statistic names, that are disabled when retrieving the complete list of statistics via `rec_control get-all`, for performance reasons.
+These statistics can still be retrieved individually.
+
+``blacklisted-stats-snmp``
+--------------------------
+.. versionadded:: 4.2.0
+
+-  String
+-  Default: "cache-bytes, packetcache-bytes, ecs-v4-response-bits-*, ecs-v6-response-bits-*"
+
+A list of comma-separated statistic names, that are prevented from being exported via SNMP, for performance reasons.
+
 .. _setting-carbon-interval:
 
 ``carbon-interval``
@@ -1225,17 +1265,6 @@ Use only a single socket for outgoing queries.
 
 If set to true and PowerDNS has been compiled with SNMP support, it will register as an SNMP agent to provide statistics and be able to send traps.
 
-.. _setting-snmp-enable-expensive-stats:
-
-``snmp-enable-expensive-stats``
-----------------------
-.. versionadded:: 4.2.0
-
--  Boolean
--  Default: no
-
-If set and snmp-agent is set, even statistics whose reporting can have an impact on production will be enabled
-
 .. _setting-snmp-master-socket:
 
 ``snmp-master-socket``

diff --git a/pdns/ws-recursor.cc b/pdns/ws-recursor.cc
index a1889a3a676449076234204072a0786da5cb19cf..dfb2368c4f55056728a3dba7fda2c7b7bb92a8be 100644 (file)
--- a/pdns/ws-recursor.cc
+++ b/pdns/ws-recursor.cc
@@ -48,7 +48,7 @@ using json11::Json;
 
 void productServerStatisticsFetch(map<string,string>& out)
 {
-  map<string,string> stats = getAllStatsMap();
+  map<string,string> stats = getAllStatsMap(StatComponent::API);
   out.swap(stats);
 }