From: Reinier Schoof Date: Tue, 23 Aug 2016 12:44:59 +0000 (+0200) Subject: dnsdist: implemented query counting X-Git-Tag: rec-4.0.2~13^2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=786e4d8c20c2048f5473a514a15d699830d253e6;p=pdns dnsdist: implemented query counting --- diff --git a/pdns/README-dnsdist.md b/pdns/README-dnsdist.md index 007d91b2f..6b74c42a3 100644 --- a/pdns/README-dnsdist.md +++ b/pdns/README-dnsdist.md @@ -978,6 +978,48 @@ latest version of [PowerDNS Metronome](https://github.com/ahupowerdns/metronome) comes with attractive graphs for `dnsdist` by default. +Query counters +------------- +When using `carbonServer`, it is also possible to send per-records statistics of +the amount of queries by using `setQueryCount(true)`. With query counting enabled, +`dnsdist` will increase a counter for every unique record or the behaviour you define +in a custom Lua function by setting `setQueryCountFilter(func)`. This filter can decide +whether to keep count on a query at all or rewrite for which query the counter will be +increased. +An example of a `QueryCountFilter` would be: + +``` +function filter(dq) + qname = dq.qname:toString() + + -- don't count PTRs at all + if(qname:match('in%-addr.arpa$')) then + return false, "" + end + + -- count these queries as if they were queried without leading www. + if(qname:match('^www.')) then + qname = qname:gsub('^www.', '') + end + + -- count queries by default + return true, qname +end + +setQueryCountFilter(filter) +``` + +Valid return values for `QueryCountFilter` functions are: + +* `true`: count the specified query +* `false`: don't count the query + +Note that the query counters are buffered and flushed each time statistics are +sent to the carbon server. The current content of the buffer can be inspected +with `getQueryCounters()`. If you decide to enable query counting without +`carbonServer`, make sure you implement clearing the log from `maintenance()` +by issuing `clearQueryCounters()`. + DNSCrypt -------- `dnsdist`, when compiled with --enable-dnscrypt, can be used as a DNSCrypt server, @@ -1176,6 +1218,11 @@ Here are all functions: * `addDomainBlock(domain)`: block queries within this domain * Carbon/Graphite/Metronome statistics related: * `carbonServer(serverIP, [ourname], [interval])`: report statistics to serverIP using our hostname, or 'ourname' if provided, every 'interval' seconds + * Query counting related: + * `clearQueryCounters()`: clears the query counter buffer. + * `getQueryCounters([max])`: show current buffer of query counters, limited by `max` if provided. + * `setQueryCount(bool)`: set whether queries should be counted. + * `setQueryCountFilter(func)`: filter queries that would be counted, where `func` is a function with parameter `dq` which decides whether a query should and how it should be counted. * Control socket related: * `makeKey()`: generate a new server access key, emit configuration line ready for pasting * `setKey(key)`: set access key to that key. diff --git a/pdns/dnsdist-carbon.cc b/pdns/dnsdist-carbon.cc index 9cab108bd..92262cd52 100644 --- a/pdns/dnsdist-carbon.cc +++ b/pdns/dnsdist-carbon.cc @@ -104,6 +104,18 @@ try str<getTTLTooShorts() << " " << now << "\r\n"; } } + + { + WriteLock wl(&g_qcount.queryLock); + std::string qname; + for(auto &record: g_qcount.records) { + qname = record.first; + boost::replace_all(qname, ".", "_"); + str<<"dnsdist.querycount." << qname << ".queries " << record.second << " " << now << "\r\n"; + } + g_qcount.records.clear(); + } + const string msg = str.str(); int ret = waitForRWData(s.getHandle(), false, 1 , 0); diff --git a/pdns/dnsdist-console.cc b/pdns/dnsdist-console.cc index fb64b5783..79d132152 100644 --- a/pdns/dnsdist-console.cc +++ b/pdns/dnsdist-console.cc @@ -245,6 +245,7 @@ const std::vector g_consoleKeywords{ { "carbonServer", true, "serverIP, [ourname], [interval]", "report statistics to serverIP using our hostname, or 'ourname' if provided, every 'interval' seconds" }, { "controlSocket", true, "addr", "open a control socket on this address / connect to this address in client mode" }, { "clearDynBlocks", true, "", "clear all dynamic blocks" }, + { "clearQueryCounters", true, "", "clears the query counter buffer" }, { "clearRules", true, "", "remove all current rules" }, { "DelayAction", true, "milliseconds", "delay the response by the specified amount of milliseconds (UDP-only)" }, { "DelayResponseAction", true, "milliseconds", "delay the response by the specified amount of milliseconds (UDP-only)" }, @@ -263,6 +264,7 @@ const std::vector g_consoleKeywords{ { "generateDNSCryptCertificate", true, "\"/path/to/providerPrivate.key\", \"/path/to/resolver.cert\", \"/path/to/resolver.key\", serial, validFrom, validUntil", "generate a new resolver private key and related certificate, valid from the `validFrom` timestamp until the `validUntil` one, signed with the provider private key" }, { "generateDNSCryptProviderKeys", true, "\"/path/to/providerPublic.key\", \"/path/to/providerPrivate.key\"", "generate a new provider keypair"}, { "getPoolServers", true, "pool", "return servers part of this pool" }, + { "getQueryCounters", true, "[max=10]", "show current buffer of query counters, limited by 'max' if provided" }, { "getResponseRing", true, "", "return the current content of the response ring" }, { "getServer", true, "n", "returns server with index n" }, { "getServers", true, "", "returns a table with all defined servers" }, @@ -306,6 +308,8 @@ const std::vector g_consoleKeywords{ { "setMaxTCPClientThreads", true, "n", "set the maximum of TCP client threads, handling TCP connections" }, { "setMaxTCPQueuedConnections", true, "n", "set the maximum number of TCP connections queued (waiting to be picked up by a client thread)" }, { "setMaxUDPOutstanding", true, "n", "set the maximum number of outstanding UDP queries to a given backend server. This can only be set at configuration time and defaults to 10240" }, + { "setQueryCount", true, "bool", "set whether queries should be counted" }, + { "setQueryCountFilter", true, "func", "filter queries that would be counted, where `func` is a function with parameter `dq` which decides whether a query should and how it should be counted" }, { "setRules", true, "list of rules", "replace the current rules with the supplied list of pairs of DNS Rules and DNS Actions (see `newRuleAction()`)" }, { "setServerPolicy", true, "policy", "set server selection policy to that policy" }, { "setServerPolicyLua", true, "name, function", "set server selection policy to one named 'name' and provided by 'function'" }, diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index 2b82fb97d..1db28624b 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -1246,10 +1246,41 @@ vector> setupLua(bool client, const std::string& confi return ret; }); - + g_lua.executeCode(R"(function topQueries(top, labels) top = top or 10; for k,v in ipairs(getTopQueries(top,labels)) do show(string.format("%4d %-40s %4d %4.1f%%",k,v[1],v[2], v[3])) end end)"); + g_lua.writeFunction("clearQueryCounters", []() { + unsigned int size{0}; + { + WriteLock wl(&g_qcount.queryLock); + size = g_qcount.records.size(); + g_qcount.records.clear(); + } + + boost::format fmt("%d records cleared from query counter buffer\n"); + g_outputBuffer = (fmt % size).str(); + }); + g_lua.writeFunction("getQueryCounters", [](boost::optional optMax) { + setLuaNoSideEffect(); + ReadLock rl(&g_qcount.queryLock); + g_outputBuffer = "query counting is currently: "; + g_outputBuffer+= g_qcount.enabled ? "enabled" : "disabled"; + g_outputBuffer+= (boost::format(" (%d records in buffer)\n") % g_qcount.records.size()).str(); + + boost::format fmt("%-3d %s: %d request(s)\n"); + QueryCountRecords::iterator it; + unsigned int max = optMax ? *optMax : 10; + unsigned int index{1}; + for(it = g_qcount.records.begin(); it != g_qcount.records.end() && index <= max; ++it, ++index) { + g_outputBuffer += (fmt % index % it->first % it->second).str(); + } + }); + + g_lua.writeFunction("setQueryCount", [](bool enabled) { g_qcount.enabled=enabled; }); + g_lua.writeFunction("setQueryCountFilter", [](QueryCountFilter func) { + g_qcount.filter = func; + }); g_lua.writeFunction("getResponseRing", []() { setLuaNoSideEffect(); diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc index 382675a0d..7977460ab 100644 --- a/pdns/dnsdist.cc +++ b/pdns/dnsdist.cc @@ -126,6 +126,7 @@ GlobalStateHolder g_pools; GlobalStateHolder, std::shared_ptr > > > g_rulactions; GlobalStateHolder, std::shared_ptr > > > g_resprulactions; Rings g_rings; +QueryCount g_qcount; GlobalStateHolder g_dstates; GlobalStateHolder> g_dynblockNMG; @@ -749,6 +750,23 @@ bool processQuery(LocalStateHolder >& localDynNMGBlock, g_rings.queryRing.push_back({now,*dq.remote,*dq.qname,dq.len,dq.qtype,*dq.dh}); } + if(g_qcount.enabled) { + string qname = (*dq.qname).toString("."); + bool countQuery{true}; + if(g_qcount.filter) { + std::lock_guard lock(g_luamutex); + std::tie (countQuery, qname) = g_qcount.filter(dq); + } + + if(countQuery) { + WriteLock wl(&g_qcount.queryLock); + if(!g_qcount.records.count(qname)) { + g_qcount.records[qname] = 0; + } + g_qcount.records[qname]++; + } + } + if(auto got=localDynNMGBlock->lookup(*dq.remote)) { if(now < got->second.until) { vinfolog("Query from %s dropped because of dynamic block", dq.remote->toStringWithPort()); diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index 5c95753d2..2b1945301 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -274,6 +274,21 @@ struct Rings { extern Rings g_rings; +typedef std::unordered_map QueryCountRecords; +typedef std::function(DNSQuestion dq)> QueryCountFilter; +struct QueryCount { + QueryCount() + { + pthread_rwlock_init(&queryLock, 0); + } + QueryCountRecords records; + QueryCountFilter filter; + pthread_rwlock_t queryLock; + bool enabled{false}; +}; + +extern QueryCount g_qcount; + struct ClientState { ComboAddress local;