#endif /* HAVE_EBPF */
/* PacketCache */
- g_lua.writeFunction("newPacketCache", [](size_t maxEntries, boost::optional<uint32_t> maxTTL, boost::optional<uint32_t> minTTL, boost::optional<uint32_t> tempFailTTL, boost::optional<uint32_t> staleTTL, boost::optional<bool> dontAge, boost::optional<size_t> numberOfShards, boost::optional<bool> deferrableInsertLock, boost::optional<uint32_t> maxNegativeTTL, boost::optional<bool> ecsParsing) {
- return std::make_shared<DNSDistPacketCache>(maxEntries, maxTTL ? *maxTTL : 86400, minTTL ? *minTTL : 0, tempFailTTL ? *tempFailTTL : 60, maxNegativeTTL ? *maxNegativeTTL : 3600, staleTTL ? *staleTTL : 60, dontAge ? *dontAge : false, numberOfShards ? *numberOfShards : 1, deferrableInsertLock ? *deferrableInsertLock : true, ecsParsing ? *ecsParsing : false);
+ g_lua.writeFunction("newPacketCache", [](size_t maxEntries, boost::optional<uint32_t> maxTTL, boost::optional<uint32_t> minTTL, boost::optional<uint32_t> tempFailTTL, boost::optional<uint32_t> staleTTL, boost::optional<bool> dontAge, boost::optional<size_t> numberOfShards, boost::optional<bool> deferrableInsertLock, boost::optional<uint32_t> maxNegativeTTL, boost::optional<bool> ecsParsing, boost::optional<std::unordered_map<std::string, boost::variant<bool, size_t>>> vars) {
+
+ bool keepStaleData = false;
+
+ if (vars) {
+
+ if (vars->count("deferrableInsertLock")) {
+ deferrableInsertLock = boost::get<bool>((*vars)["deferrableInsertLock"]);
+ }
+
+ if (vars->count("dontAge")) {
+ dontAge = boost::get<bool>((*vars)["dontAge"]);
+ }
+
+ if (vars->count("keepStaleData")) {
+ keepStaleData = boost::get<bool>((*vars)["keepStaleData"]);
+ }
+
+ if (vars->count("maxEntries")) {
+ maxEntries = boost::get<size_t>((*vars)["maxEntries"]);
+ }
+
+ if (vars->count("maxNegativeTTL")) {
+ maxNegativeTTL = boost::get<size_t>((*vars)["maxNegativeTTL"]);
+ }
+
+ if (vars->count("maxTTL")) {
+ maxTTL = boost::get<size_t>((*vars)["maxTTL"]);
+ }
+
+ if (vars->count("minTTL")) {
+ minTTL = boost::get<size_t>((*vars)["minTTL"]);
+ }
+
+ if (vars->count("numberOfShards")) {
+ numberOfShards = boost::get<size_t>((*vars)["numberOfShards"]);
+ }
+
+ if (vars->count("parseECS")) {
+ ecsParsing = boost::get<bool>((*vars)["parseECS"]);
+ }
+
+ if (vars->count("staleTTL")) {
+ staleTTL = boost::get<size_t>((*vars)["staleTTL"]);
+ }
+
+ if (vars->count("temporaryFailureTTL")) {
+ tempFailTTL = boost::get<size_t>((*vars)["temporaryFailureTTL"]);
+ }
+
+ }
+
+ auto res = std::make_shared<DNSDistPacketCache>(maxEntries, maxTTL ? *maxTTL : 86400, minTTL ? *minTTL : 0, tempFailTTL ? *tempFailTTL : 60, maxNegativeTTL ? *maxNegativeTTL : 3600, staleTTL ? *staleTTL : 60, dontAge ? *dontAge : false, numberOfShards ? *numberOfShards : 1, deferrableInsertLock ? *deferrableInsertLock : true, ecsParsing ? *ecsParsing : false);
+
+ res->setKeepStaleData(keepStaleData);
+
+ return res;
});
g_lua.registerFunction("toString", &DNSDistPacketCache::toString);
g_lua.registerFunction("isFull", &DNSDistPacketCache::isFull);
g_outputBuffer+="TTL Too Shorts: " + std::to_string(cache->getTTLTooShorts()) + "\n";
}
});
+ g_lua.registerFunction<std::unordered_map<std::string, uint64_t>(std::shared_ptr<DNSDistPacketCache>::*)()>("getStats", [](const std::shared_ptr<DNSDistPacketCache> cache) {
+ std::unordered_map<std::string, uint64_t> stats;
+ if (cache) {
+ stats["entries"] = cache->getEntriesCount();
+ stats["maxEntries"] = cache->getMaxEntries();
+ stats["hits"] = cache->getHits();
+ stats["misses"] = cache->getMisses();
+ stats["deferredInserts"] = cache->getDeferredInserts();
+ stats["deferredLookups"] = cache->getDeferredLookups();
+ stats["lookupCollisions"] = cache->getLookupCollisions();
+ stats["insertCollisions"] = cache->getInsertCollisions();
+ stats["ttlTooShorts"] = cache->getTTLTooShorts();
+ }
+ return stats;
+ });
g_lua.registerFunction<void(std::shared_ptr<DNSDistPacketCache>::*)(const std::string& fname)>("dump", [](const std::shared_ptr<DNSDistPacketCache> cache, const std::string& fname) {
if (cache) {
A Pool can have a packet cache to answer queries directly in stead of going to the backend.
See :doc:`../guides/cache` for a how to.
-.. function:: newPacketCache(maxEntries[, maxTTL=86400[, minTTL=0[, temporaryFailureTTL=60[, staleTTL=60[, dontAge=false[, numberOfShards=1[, deferrableInsertLock=true[, maxNegativeTTL=3600[, parseECS=false]]]]]]]) -> PacketCache
+.. function:: newPacketCache(maxEntries[, maxTTL=86400[, minTTL=0[, temporaryFailureTTL=60[, staleTTL=60[, dontAge=false[, numberOfShards=1[, deferrableInsertLock=true[, maxNegativeTTL=3600[, parseECS=false [,options]]]]]]]]) -> PacketCache
.. versionchanged:: 1.3.0
``numberOfShards`` and ``deferrableInsertLock`` parameters added.
.. versionchanged:: 1.3.1
``maxNegativeTTL`` and ``parseECS`` parameters added.
+ .. versionchanged:: 1.3.4
+ ``options`` parameter added.
+
Creates a new :class:`PacketCache` with the settings specified.
+ Starting with 1.3.4, all parameters can be specified in the ``options`` table, overriding the value from the existing parameters if any.
:param int maxEntries: The maximum number of entries in this cache
:param int maxTTL: Cap the TTL for records to his number
:param bool deferrableInsertLock: Whether the cache should give up insertion if the lock is held by another thread, or simply wait to get the lock
:param int maxNegativeTTL: Cache a NXDomain or NoData answer from the backend for at most this amount of seconds, even if the TTL of the SOA record is higher
:param bool parseECS: Whether any EDNS Client Subnet option present in the query should be extracted and stored to be able to detect hash collisions involving queries with the same qname, qtype and qclass but a different incoming ECS value. Enabling this option adds a parsing cost and only makes sense if at least one backend might send different responses based on the ECS value, so it's disabled by default
+ :param table options: A table with key: value pairs with the options listed below:
+
+ Options:
+
+ * ``deferrableInsertLock=true``: bool - Whether the cache should give up insertion if the lock is held by another thread, or simply wait to get the lock.
+ * ``dontAge=false``: bool - Don't reduce TTLs when serving from the cache. Use this when :program:`dnsdist` fronts a cluster of authoritative servers.
+ * ``keepStaleData=false``: bool - Whether to suspend the removal of expired entries from the cache when there is no backend available in at least one of the pools using this cache.
+ * ``maxEntries``: int - The maximum number of entries in this cache.
+ * ``maxNegativeTTL=3600``: int - Cache a NXDomain or NoData answer from the backend for at most this amount of seconds, even if the TTL of the SOA record is higher.
+ * ``maxTTL=86400``: int - Cap the TTL for records to his number.
+ * ``minTTL=0``: int - Don't cache entries with a TTL lower than this.
+ * ``numberOfShards=1``: int - Number of shards to divide the cache into, to reduce lock contention.
+ * ``parseECS=false``: bool - Whether any EDNS Client Subnet option present in the query should be extracted and stored to be able to detect hash collisions involving queries with the same qname, qtype and qclass but a different incoming ECS value. Enabling this option adds a parsing cost and only makes sense if at least one backend might send different responses based on the ECS value, so it's disabled by default.
+ * ``staleTTL=60``: int - When the backend servers are not reachable, and global configuration ``setStaleCacheEntriesTTL`` is set appropriately, TTL that will be used when a stale cache entry is returned.
+ * ``temporaryFailureTTL=60``: int - On a SERVFAIL or REFUSED from the backend, cache for this amount of seconds..
.. class:: PacketCache
:param int qtype: The type to expunge
:param bool suffixMatch: When set to true, remove al entries under ``name``
+ .. method:: PacketCache:getStats()
+
+ .. versionadded:: 1.3.4
+
+ Return the cache stats (number of entries, hits, misses, deferred lookups, deferred inserts, lookup collisions, insert collisons and TTL too shorts) as a Lua table.
+
.. method:: PacketCache:isFull() -> bool
Return true if the cache has reached the maximum number of entries.
.. method:: PacketCache:printStats()
- Print the cache stats (hits, misses, deferred lookups and deferred inserts).
+ Print the cache stats (number of entries, hits, misses, deferred lookups, deferred inserts, lookup collisions, insert collisons and TTL too shorts).
.. method:: PacketCache:purgeExpired(n)
class TestCaching(DNSDistTest):
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(100, 86400, 1)
getPool(""):setCache(pc)
addAction(makeRule("nocache.cache.tests.powerdns.com."), SkipCacheAction())
return DNSAction.None, ""
end
addAction("nocachevialua.cache.tests.powerdns.com.", LuaAction(skipViaLua))
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCached(self):
class TestTempFailureCacheTTLAction(DNSDistTest):
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(100, 86400, 1)
getPool(""):setCache(pc)
addAction("servfail.cache.tests.powerdns.com.", TempFailureCacheTTLAction(1))
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testTempFailureCacheTTLAction(self):
class TestCachingWithExistingEDNS(DNSDistTest):
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(5, 86400, 1)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheWithEDNS(self):
"""
class TestCachingCacheFull(DNSDistTest):
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(1, 86400, 1)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheFull(self):
"""
_consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
_config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(100, 86400, 1)
getPool(""):setCache(pc)
setKey("%s")
- controlSocket("127.0.0.1:%s")
- newServer{address="127.0.0.1:%s"}
+ controlSocket("127.0.0.1:%d")
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheNoStale(self):
"""
_staleCacheTTL = 60
_config_params = ['_staleCacheTTL', '_consoleKeyB64', '_consolePort', '_testServerPort']
_config_template = """
- pc = newPacketCache(100, 86400, 1, %s)
+ -- maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=XX
+ pc = newPacketCache(100, 86400, 1, 0, %d)
getPool(""):setCache(pc)
setStaleCacheEntriesTTL(600)
setKey("%s")
- controlSocket("127.0.0.1:%s")
- newServer{address="127.0.0.1:%s"}
+ controlSocket("127.0.0.1:%d")
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheStale(self):
"""
self.assertEquals(total, misses)
+class TestCachingStaleExpunged(DNSDistTest):
+
+ _consoleKey = DNSDistTest.generateConsoleKey()
+ _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
+ _staleCacheTTL = 60
+ _config_params = ['_staleCacheTTL', '_consoleKeyB64', '_consolePort', '_testServerPort']
+ _config_template = """
+ -- maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=XX
+ pc = newPacketCache(100, 86400, 1, 0, %d)
+ getPool(""):setCache(pc)
+ setStaleCacheEntriesTTL(600)
+ -- try to remove all expired entries
+ setCacheCleaningPercentage(100)
+ -- clean the cache every second
+ setCacheCleaningDelay(1)
+ setKey("%s")
+ controlSocket("127.0.0.1:%d")
+ newServer{address="127.0.0.1:%d"}
+ """
+ def testCacheStale(self):
+ """
+ Cache: Cache entry, set backend down, wait for the cache cleaning to run and remove the entry, get no entry
+ """
+ misses = 0
+ drops = 0
+ ttl = 1
+ name = 'stale-but-expunged.cache.tests.powerdns.com.'
+ query = dns.message.make_query(name, 'A', 'IN')
+ response = dns.message.make_response(query)
+ rrset = dns.rrset.from_text(name,
+ ttl,
+ dns.rdataclass.IN,
+ dns.rdatatype.A,
+ '127.0.0.1')
+ response.answer.append(rrset)
+
+ # Miss
+ (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+ self.assertTrue(receivedQuery)
+ self.assertTrue(receivedResponse)
+ receivedQuery.id = query.id
+ self.assertEquals(query, receivedQuery)
+ self.assertEquals(response, receivedResponse)
+ misses += 1
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), misses + drops)
+
+ # next queries should hit the cache
+ (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+ self.assertEquals(receivedResponse, response)
+ # the cache should have one entry
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
+
+ # ok, we mark the backend as down
+ self.sendConsoleCommand("getServer(0):setDown()")
+ # and we wait for the entry to expire
+ time.sleep(ttl + 1)
+ # wait a bit more to be sure that the cache cleaning algo has been run
+ time.sleep(1)
+ # the cache should be empty now
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 0)
+
+ # we should get a DROP (backend is down, nothing in the cache anymore)
+ (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+ self.assertEquals(receivedResponse, None)
+ drops += 1
+
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), misses + drops)
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
+
+ total = 0
+ for key in self._responsesCounter:
+ total += self._responsesCounter[key]
+
+ self.assertEquals(total, misses)
+
+class TestCachingStaleExpungePrevented(DNSDistTest):
+
+ _consoleKey = DNSDistTest.generateConsoleKey()
+ _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
+ _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
+ _config_template = """
+ -- maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, ecsParsing=false, keepStaleData=true
+ pc = newPacketCache(100, 86400, 1, 0, 60, false, 1, true, 3600, false, { keepStaleData=true})
+ getPool(""):setCache(pc)
+ setStaleCacheEntriesTTL(600)
+ -- try to remove all expired entries
+ setCacheCleaningPercentage(100)
+ -- clean the cache every second
+ setCacheCleaningDelay(1)
+ setKey("%s")
+ controlSocket("127.0.0.1:%d")
+ newServer{address="127.0.0.1:%d"}
+ """
+ def testCacheStale(self):
+ """
+ Cache: Cache entry, set backend down, wait for the cache cleaning to run and remove the entry, still get a cache HIT because the stale entry was not removed
+ """
+ misses = 0
+ ttl = 1
+ name = 'stale-not-expunged.cache.tests.powerdns.com.'
+ query = dns.message.make_query(name, 'A', 'IN')
+ response = dns.message.make_response(query)
+ rrset = dns.rrset.from_text(name,
+ ttl,
+ dns.rdataclass.IN,
+ dns.rdatatype.A,
+ '127.0.0.1')
+ response.answer.append(rrset)
+
+ # Miss
+ (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+ self.assertTrue(receivedQuery)
+ self.assertTrue(receivedResponse)
+ receivedQuery.id = query.id
+ self.assertEquals(query, receivedQuery)
+ self.assertEquals(response, receivedResponse)
+ misses += 1
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), 1)
+
+ # next queries should hit the cache
+ (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+ self.assertEquals(receivedResponse, response)
+ # the cache should have one entry
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
+
+ # ok, we mark the backend as down
+ self.sendConsoleCommand("getServer(0):setDown()")
+ # and we wait for the entry to expire
+ time.sleep(ttl + 1)
+ # wait a bit more to be sure that the cache cleaning algo has been run
+ time.sleep(1)
+ # the cache should NOT be empty because the removal of the expired entry should have been prevented
+ # since all backends for this pool are down
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
+
+ # we should get a HIT
+ (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+ self.assertEquals(receivedResponse, response)
+
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), 1)
+ self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 2)
+
+ total = 0
+ for key in self._responsesCounter:
+ total += self._responsesCounter[key]
+
+ self.assertEquals(total, misses)
+
class TestCacheManagement(DNSDistTest):
_consoleKey = DNSDistTest.generateConsoleKey()
_consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
_config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(100, 86400, 1)
getPool(""):setCache(pc)
setKey("%s")
- controlSocket("127.0.0.1:%s")
- newServer{address="127.0.0.1:%s"}
+ controlSocket("127.0.0.1:%d")
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheExpunge(self):
"""
_minCacheTTL = 600
_config_params = ['_maxCacheTTL', '_minCacheTTL', '_testServerPort']
_config_template = """
- pc = newPacketCache(1000, %s, %s)
+ -- maxTTL=XX, minTTL=XX
+ pc = newPacketCache(1000, %d, %d)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheShortTTL(self):
"""
_maxCacheTTL = 2
_config_params = ['_maxCacheTTL', '_testServerPort']
_config_template = """
- pc = newPacketCache(1000, %s)
+ -- maxTTL=XX
+ pc = newPacketCache(1000, %d)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheLongTTL(self):
"""
_failureCacheTTL = 2
_config_params = ['_failureCacheTTL', '_testServerPort']
_config_template = """
+ -- maxTTL=86400, minTTL=0, temporaryFailureTTL=XX, staleTTL=60
pc = newPacketCache(1000, 86400, 0, %d, 60)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheServFailTTL(self):
"""
_negCacheTTL = 1
_config_params = ['_negCacheTTL', '_testServerPort']
_config_template = """
+ -- maxTTL=86400, minTTL=0, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=XX
pc = newPacketCache(1000, 86400, 0, 60, 60, false, 1, true, %d)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheNegativeTTLNXDomain(self):
class TestCachingDontAge(DNSDistTest):
_config_template = """
+ -- maxTTL=86400, minTTL=0, temporaryFailureTTL=60, staleTTL=60, dontAge=true
pc = newPacketCache(100, 86400, 0, 60, 60, true)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheDoesntDecreaseTTL(self):
"""
_consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
_config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(100, 86400, 1)
getPool(""):setCache(pc)
setKey("%s")
_consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
_config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(100, 86400, 1)
getPool(""):setCache(pc)
getPool(""):setECS(true)
class TestCachingCollisionNoECSParsing(DNSDistTest):
_config_template = """
+ -- maxTTL=86400, minTTL=1
pc = newPacketCache(100, 86400, 1)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheCollisionNoECSParsing(self):
class TestCachingCollisionWithECSParsing(DNSDistTest):
_config_template = """
+ -- maxTTL=86400, minTTL=1, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, parseECS=true
pc = newPacketCache(100, 86400, 1, 60, 60, false, 1, true, 3600, true)
getPool(""):setCache(pc)
- newServer{address="127.0.0.1:%s"}
+ newServer{address="127.0.0.1:%d"}
"""
def testCacheCollisionWithECSParsing(self):
projects / pdns / commitdiff