From 9bc0d00afe0c604d56df69ac81ad3b401358339c Mon Sep 17 00:00:00 2001 From: Peter van Dijk Date: Tue, 17 Nov 2015 11:30:54 +0100 Subject: [PATCH] remove LMDB backend completely --- .travis.yml | 6 +- build-scripts/debian-authoritative/control | 15 +- build-scripts/test-auth | 3 - configure.ac | 7 - m4/pdns_with_lmdb.m4 | 8 - modules/Makefile.am | 1 - modules/lmdbbackend/Makefile.am | 13 - modules/lmdbbackend/OBJECTFILES | 1 - modules/lmdbbackend/OBJECTLIBS | 1 - modules/lmdbbackend/dumpdb.pl | 36 -- modules/lmdbbackend/lmdb-example.pl | 63 --- modules/lmdbbackend/lmdbbackend.cc | 574 --------------------- modules/lmdbbackend/lmdbbackend.hh | 68 --- pdns/.gitignore | 1 - pdns/Makefile.am | 37 +- pdns/zone2lmdb.cc | 315 ----------- regression-tests/.gitignore | 3 - regression-tests/backends/common | 4 - regression-tests/backends/lmdb-master | 107 ---- 19 files changed, 4 insertions(+), 1259 deletions(-) delete mode 100644 m4/pdns_with_lmdb.m4 delete mode 100644 modules/lmdbbackend/Makefile.am delete mode 100644 modules/lmdbbackend/OBJECTFILES delete mode 100644 modules/lmdbbackend/OBJECTLIBS delete mode 100644 modules/lmdbbackend/dumpdb.pl delete mode 100644 modules/lmdbbackend/lmdb-example.pl delete mode 100644 modules/lmdbbackend/lmdbbackend.cc delete mode 100644 modules/lmdbbackend/lmdbbackend.hh delete mode 100644 pdns/zone2lmdb.cc delete mode 100644 regression-tests/backends/lmdb-master diff --git a/.travis.yml b/.travis.yml index daad369f9..0c3407ddf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,7 +53,7 @@ before_script: xmlto - sudo sh -c 'sed s/precise/trusty/g /etc/apt/sources.list > /etc/apt/sources.list.d/trusty.list' - sudo apt-get update --quiet --quiet - - sudo apt-get install --quiet --quiet liblmdb0 liblmdb-dev lmdb-utils libyaml-cpp-dev libzmq3-dev pandoc libgeoip-dev + - sudo apt-get install --quiet --quiet libyaml-cpp-dev libzmq3-dev pandoc libgeoip-dev - sudo pip install pandocfilters==1.2.3 mkdocs==0.14 linkchecker==9.3 click==5.1 - sudo update-alternatives --set ruby /usr/bin/ruby1.9.1 - sudo touch /etc/authbind/byport/53 @@ -82,7 +82,6 @@ before_script: - p11-kit -l # ensure it's ok script: - ./bootstrap -#DNSName --with-dynmodules='bind gmysql gpgsql gsqlite3 mydns tinydns remote random opendbx ldap lmdb lua' #Build without --enable-botan1.10 option, Botan/SoftHSM conflict #2496 - CFLAGS='-O1' CXXFLAGS='-O1' ./configure --with-dynmodules='bind gmysql geoip gpgsql gsqlite3 mydns tinydns pipe remote random opendbx ldap lua' @@ -105,7 +104,7 @@ script: - test -f pdns/test-suite.log && cat pdns/test-suite.log || true - test -f modules/remotebackend/test-suite.log && cat modules/remotebackend/test-suite.log || true # DNSName: - make -k -j 2 -C pdns $(grep '(EXEEXT):' pdns/Makefile | cut -f1 -d\$ | grep -E -v 'dnsdist|calidns') - - make -k -j 2 -C pdns $(grep '(EXEEXT):' pdns/Makefile | cut -f1 -d\$ | grep -E -v 'dnsdist|calidns|zone2lmdb|speedtest') + - make -k -j 2 -C pdns $(grep '(EXEEXT):' pdns/Makefile | cut -f1 -d\$ | grep -E -v 'dnsdist|calidns|speedtest') - cd pdns - make -k -j 2 pdns_recursor - rm -f pdns_recursor @@ -170,7 +169,6 @@ script: - ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-both # - ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-optout-both # - ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-narrow -#DNSName - ./timestamp ./start-test-stop 5300 lmdb-nodnssec - ./timestamp ./start-test-stop 5300 mydns - ./timestamp ./start-test-stop 5300 opendbx-sqlite3 - travis_retry ./timestamp timeout 120s ./start-test-stop 5300 remotebackend-pipe diff --git a/build-scripts/debian-authoritative/control b/build-scripts/debian-authoritative/control index f8912e7ef..6af8203df 100644 --- a/build-scripts/debian-authoritative/control +++ b/build-scripts/debian-authoritative/control @@ -4,7 +4,7 @@ Priority: extra Standards-Version: 3.9.6 Maintainer: PowerDNS Autobuilder Origin: PowerDNS -Build-Depends: debhelper (>= 9~), dh-autoreconf, dh-systemd, po-debconf, libtool, flex, bison, libmysqlclient-dev, libpq-dev, libssl-dev, libpolarssl-dev, libgdbm-dev, libldap2-dev, libsqlite3-dev, dpkg-dev (>= 1.17.0~), libboost-dev, libboost-serialization-dev, libboost-program-options-dev, libboost-test-dev, autotools-dev, automake, autoconf, liblua5.2-dev, pkg-config, libcrypto++-dev, ragel, libgmp-dev, libbotan1.10-dev, libcurl4-openssl-dev, libzmq-dev, liblmdb-dev, libyaml-cpp-dev (>= 0.5), libgeoip-dev, libopendbx1-dev, libcdb-dev +Build-Depends: debhelper (>= 9~), dh-autoreconf, dh-systemd, po-debconf, libtool, flex, bison, libmysqlclient-dev, libpq-dev, libssl-dev, libpolarssl-dev, libgdbm-dev, libldap2-dev, libsqlite3-dev, dpkg-dev (>= 1.17.0~), libboost-dev, libboost-serialization-dev, libboost-program-options-dev, libboost-test-dev, autotools-dev, automake, autoconf, liblua5.2-dev, pkg-config, libcrypto++-dev, ragel, libgmp-dev, libbotan1.10-dev, libcurl4-openssl-dev, libzmq-dev, libyaml-cpp-dev (>= 0.5), libgeoip-dev, libopendbx1-dev, libcdb-dev Homepage: http://www.powerdns.com/ Package: pdns-server @@ -157,19 +157,6 @@ Description: Lua backend for PowerDNS . This package contains the Lua backend for PowerDNS. -Package: pdns-backend-lmdb -Architecture: any -Depends: pdns-server (>= ${source:Version}), ${shlibs:Depends}, ${misc:Depends} -Provides: pdns-backend -Description: lmdb backend for PowerDNS - PowerDNS is a versatile nameserver which supports a large number - of different backends ranging from simple zonefiles to relational - databases and load balancing/failover algorithms. - PowerDNS tries to emphasize speed and security. - . - This package contains the Symas Lightning Memory-Mapped Database - backend for PowerDNS. - Package: pdns-backend-remote Architecture: any Depends: pdns-server (>= ${source:Version}), ${shlibs:Depends}, ${misc:Depends} diff --git a/build-scripts/test-auth b/build-scripts/test-auth index cc2438e47..4f64bbba4 100755 --- a/build-scripts/test-auth +++ b/build-scripts/test-auth @@ -71,9 +71,6 @@ export geoipregion=oc geoipregionip=1.2.3.4 ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-optout-both || EXITCODE=1 ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-narrow || EXITCODE=1 -# LMDB is broken pending DNSName -# ./timestamp ./start-test-stop 5300 lmdb-nodnssec || EXITCODE=1 - ./timestamp ./start-test-stop 5300 mydns || EXITCODE=1 ./timestamp ./start-test-stop 5300 opendbx-sqlite3 || EXITCODE=1 diff --git a/configure.ac b/configure.ac index e325bfae1..06e9c8dd1 100644 --- a/configure.ac +++ b/configure.ac @@ -242,10 +242,6 @@ for a in $modules $dynmodules; do PDNS_WITH_ORACLE needoracle=yes ;; - lmdb) - PDNS_WITH_LMDB - needlmdb=yes - ;; mydns|gmysql|pdns) PDNS_WITH_MYSQL ;; @@ -292,8 +288,6 @@ AM_CONDITIONAL([ORACLE],[test "x$needoracle" = "xyes"]) AM_CONDITIONAL([LDAP],[test "x$needldap" = "xyes"]) -AM_CONDITIONAL([LMDB],[test "x$needlmdb" = "xyes"]) - PDNS_CHECK_SQLITE3 AM_CONDITIONAL([SQLITE3], [test "x$needsqlite3" = "xyes"]) @@ -369,7 +363,6 @@ AC_CONFIG_FILES([ modules/randombackend/Makefile modules/remotebackend/Makefile modules/tinydnsbackend/Makefile - modules/lmdbbackend/Makefile ]) AC_OUTPUT diff --git a/m4/pdns_with_lmdb.m4 b/m4/pdns_with_lmdb.m4 deleted file mode 100644 index 30a054057..000000000 --- a/m4/pdns_with_lmdb.m4 +++ /dev/null @@ -1,8 +0,0 @@ -AC_DEFUN([PDNS_WITH_LMDB],[ - AC_CHECK_HEADERS([lmdb.h], , [AC_MSG_ERROR([lmdb header (lmdb.h) not found])]) - AC_SUBST([LMDB_LIBS]) - AC_CHECK_LIB( - [lmdb], [mdb_env_create], - [AC_DEFINE([HAVE_LIBLMDB], 1, [Have -llmdb]) LMDB_LIBS="-llmdb"] - ) -]) diff --git a/modules/Makefile.am b/modules/Makefile.am index 40d2a867e..7dcb11543 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -8,7 +8,6 @@ DIST_SUBDIRS = \ gpgsqlbackend \ gsqlite3backend \ ldapbackend \ - lmdbbackend \ luabackend \ mydnsbackend \ opendbxbackend \ diff --git a/modules/lmdbbackend/Makefile.am b/modules/lmdbbackend/Makefile.am deleted file mode 100644 index 2c7cb9a66..000000000 --- a/modules/lmdbbackend/Makefile.am +++ /dev/null @@ -1,13 +0,0 @@ -pkglib_LTLIBRARIES = liblmdbbackend.la - -EXTRA_DIST = \ - lmdb-example.pl \ - OBJECTFILES \ - OBJECTLIBS - -liblmdbbackend_la_SOURCES = \ - lmdbbackend.cc lmdbbackend.hh - -liblmdbbackend_la_LDFLAGS = -module -avoid-version -liblmdbbackend_la_LIBADD = $(LMDB_LIBS) - diff --git a/modules/lmdbbackend/OBJECTFILES b/modules/lmdbbackend/OBJECTFILES deleted file mode 100644 index ac1bd2e77..000000000 --- a/modules/lmdbbackend/OBJECTFILES +++ /dev/null @@ -1 +0,0 @@ -lmdbbackend.lo diff --git a/modules/lmdbbackend/OBJECTLIBS b/modules/lmdbbackend/OBJECTLIBS deleted file mode 100644 index ad4006b9f..000000000 --- a/modules/lmdbbackend/OBJECTLIBS +++ /dev/null @@ -1 +0,0 @@ -$(LMDB_LIBS) diff --git a/modules/lmdbbackend/dumpdb.pl b/modules/lmdbbackend/dumpdb.pl deleted file mode 100644 index a682b6c03..000000000 --- a/modules/lmdbbackend/dumpdb.pl +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/perl -use strict; -use warnings; - -use LMDB_File qw( :dbflags :envflags :cursor_op :writeflags ); - -my ($path, $dbname, $searchkey) = @ARGV; -die unless -d $path; - -my $env = LMDB::Env->new( $path, { - mapsize => 1024*1024*1024, - maxdbs => 3, - flags => MDB_RDONLY, -}); -my $txn = LMDB::Txn->new( $env, MDB_RDONLY ); -my $db = $txn->OpenDB( $dbname, MDB_DUPSORT ); -my $c = $db->Cursor; -my ($k, $v); -if( $searchkey ) { - $c->get( $k = $searchkey, $v, MDB_SET_RANGE ); -} else { - $c->get( $k, $v, MDB_FIRST ); -} - -print "key: $k; value: $v\n"; - -while(1) { - eval { - $c->get( $k, $v, MDB_NEXT ); - }; - if( $@ =~ /MDB_NOTFOUND/ ) { - exit; - } - die $@ if $@; - print "key: $k; value: $v\n"; -} diff --git a/modules/lmdbbackend/lmdb-example.pl b/modules/lmdbbackend/lmdb-example.pl deleted file mode 100644 index 2f96cd004..000000000 --- a/modules/lmdbbackend/lmdb-example.pl +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/perl -# An example script to generate files for the PowerDNS LMDB high performance -# backend - -use LMDB_File 0.04 qw( :dbflags :envflags :cursor_op :writeflags ); - -use strict; -use warnings; - -my $HOME = "/var/tmp/lmdb"; - -mkdir $HOME unless -d $HOME; -my $env = LMDB::Env->new( $HOME, { - mapsize => 100*1024*1024*1024, - maxdbs => 3, -}); -my $txn = LMDB::Txn->new( $env, 0 ); -my $dns_zone = $txn->OpenDB( 'zone', MDB_CREATE ); - -my $zone = 'example.com'; -my $zone_id = 1; -my $zone_ttl = 300; -my $soa_entry = "ns.$zone. hostmaster.$zone. 2012021101 86400 7200 604800 86400"; -# XXX $zone length MUST be less than 500 bytes -$dns_zone->put( scalar reverse(lc $zone), join("\t", $zone_id, $zone_ttl, $soa_entry) ); - -my $dns_data = $txn->OpenDB( 'data', MDB_CREATE | MDB_DUPSORT ); -my $dns_extended_data = $txn->OpenDB( 'extended_data', MDB_CREATE ); -my @entries = ( - # host type data - [ $zone, 'NS', "ns.$zone" ], - # MX/SRV put priority data - [ $zone, 'MX', "10 mail.example.com" ], - # No SOA records - [ "test.$zone", 'A', '192.0.2.66' ], - [ "text.$zone", 'TXT', "test\n123" ], - [ "longtext.$zone", 'TXT', "A" x 550 ], - -); - -my $extended_ref = 0; -for my $row (@entries) { - my ($host, $type, $data) = @$row; - - # Don't ever allow these characters as they break powerdns - $data =~ tr/"\\//d; - - if( $type eq 'TXT' ) { - $data =~ s/([^ -~])/sprintf '\\%03d', ord $1/eg; - } - - my $key = join( "\t", scalar reverse(lc $host), $type ); # XXX must be less than 500 bytes - my $val = join( "\t", $zone_id, $zone_ttl, $data); - if( length $val > 500 ) { - $dns_data->put( $key, "REF\t" . ++$extended_ref ); - $dns_extended_data->put( $extended_ref, $val ); - # Extended data record storage as DUPSORT can only store up to 500 bytes of data unfortunately - } else { - $dns_data->put( $key, $val ); - } -} - -$txn->commit; diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc deleted file mode 100644 index b21407452..000000000 --- a/modules/lmdbbackend/lmdbbackend.cc +++ /dev/null @@ -1,574 +0,0 @@ -/* - * LMDBBackend - a high performance LMDB based backend for PowerDNS written by - * Mark Zealey, 2013 - * - * This was originally going to be a backend using BerkeleyDB 5 for high - * performance DNS over massive (millions of zones) databases. However, - * BerkeleyDB had a number of issues to do with locking, contention and - * corruption which made it unsuitable for use. Instead, we use LMDB to perform - * very fast lookups. - * - * See the documentation for more details, and lmdb-example.pl for an example - * script which generates a simple zone. - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif -#include "pdns/utility.hh" -#include "pdns/dnsbackend.hh" -#include "pdns/dns.hh" -#include "pdns/dnspacket.hh" -#include "pdns/pdnsexception.hh" -#include "pdns/logger.hh" -#include -#include "lmdbbackend.hh" -#include "pdns/arguments.hh" -#include "pdns/base32.hh" -#include "pdns/lock.hh" - -#if 0 -#define DEBUGLOG(msg) L< d_lastreload) { - WriteLock wl(&s_initlock); - d_lastreload = s_reloadcount; - close_db(); - open_db(); - } -} - -bool LMDBBackend::getDomainMetadata(const string& name, const std::string& kind, std::vector& meta) -{ - if (!d_doDnssec) - return false; - - needReload(); - ReadLock rl(&s_initlock); - - if (kind == "PRESIGNED" || kind == "NSEC3PARAM") { - int rc; - MDB_val key, data; - string key_str, cur_value; - vector valparts; - - key_str=bitFlip(labelReverse(toLower(name)))+"\xff"; - key.mv_data = (char *)key_str.c_str(); - key.mv_size = key_str.length(); - - if ((rc = mdb_cursor_get(zone_cursor, &key, &data, MDB_SET_KEY)) == 0) { - cur_value.assign((const char *)data.mv_data, data.mv_size); - stringtok(valparts,cur_value,"\t"); - - if (valparts.size() == 4) { - if (kind == "PRESIGNED") - meta.push_back("1"); - else if (valparts[3] != "1") - meta.push_back(valparts[3]); - } - } - - if (rc == MDB_NOTFOUND) - DEBUGLOG("Metadata records for zone: '"< keyparts, valparts; - - if (qtype == QType::NSEC) - key_str=itoa(id)+"\t"+bitFlip(hashed)+"\xff"; - else - key_str=itoa(id)+"\t"+toBase32Hex(bitFlip(hashed)); - key.mv_data = (char *)key_str.c_str(); - key.mv_size = key_str.length(); - - before.clear(); - if(!mdb_cursor_get(nsecx_cursor, &key, &data, MDB_SET_RANGE)) { - cur_key.assign((const char *)key.mv_data, key.mv_size); - cur_value.assign((const char *)data.mv_data, data.mv_size); - stringtok(keyparts,cur_key,"\t"); - stringtok(valparts,cur_value,"\t"); - - if( keyparts.size() != 2 || valparts.size() != 4 ) { - throw PDNSException("Invalid record in nsecx table: key: '" + cur_key + "'; value: "+ cur_value); - } - - // is the key a full match or does the id part match our zone? - // if it does we have a valid answer. - if (!key_str.compare(cur_key) || atoi(keyparts[0].c_str()) == (int) id) // FIXME we need atoui - goto hasnsecx; - } - // no match, now we look for the last record in the NSECx chain. - key_str=itoa(id)+"\t"; - key.mv_data = (char *)key_str.c_str(); - key.mv_size = key_str.length(); - - if(!mdb_cursor_get(nsecx_cursor, &key, &data, MDB_NEXT_NODUP )) { - cur_key.assign((const char *)key.mv_data, key.mv_size); - cur_value.assign((const char *)data.mv_data, data.mv_size); - stringtok(keyparts,cur_key,"\t"); - stringtok(valparts,cur_value,"\t"); - - if( keyparts.size() != 2 || valparts.size() != 4 ) { - throw PDNSException("Invalid record in nsecx table: key: '" + cur_key + "'; value: "+ cur_value); - } - - if (!key_str.compare(cur_key) || atoi(keyparts[0].c_str()) == (int) id) // FIXME we need atoui - goto hasnsecx; - } - - DEBUGLOG("NSECx record for '"< &rrsigs) -{ - if (!d_doDnssec) - return false; - - needReload(); - ReadLock rl(&s_initlock); - - int rc; - MDB_val key, data; - string key_str, cur_value; - vector valparts; - - key_str=signer+"\t"+makeRelative(qname, signer)+"\t"+qtype.getName(); - key.mv_data = (char *)key_str.c_str(); - key.mv_size = key_str.length(); - - if ((rc = mdb_cursor_get(rrsig_cursor, &key, &data, MDB_SET_KEY)) == 0) { - DNSResourceRecord rr; - rr.qname=qname; - rr.qtype=QType::RRSIG; - //rr.d_place = (DNSResourceRecord::Place) signPlace; - rr.auth=false; - - do { - cur_value.assign((const char *)data.mv_data, data.mv_size); - stringtok(valparts,cur_value,"\t"); - - if( valparts.size() != 2 ) { - throw PDNSException("Invalid record in rrsig table: qname: '" + qname + "'; value: "+ cur_value); - } - - rr.ttl=atoi(valparts[0].c_str()); - rr.content = valparts[1]; - rrsigs.push_back(rr); - - } while (mdb_cursor_get(rrsig_cursor, &key, &data, MDB_NEXT_DUP) == 0); - } - - if (rc == MDB_NOTFOUND) - DEBUGLOG("RRSIG records for qname: '"<parts; - stringtok(parts,data,"\t"); - - if(parts.size() < 3) - throw PDNSException("Invalid record in zone table: " + data ); - - fillSOAData( parts[2], soa ); - - soa.domain_id = atoi( parts[0].c_str() ); - soa.ttl = atoi( parts[1].c_str() ); - - soa.scopeMask = 0; - soa.db = this; - - return true; -} - -// Called to start an AXFR then ->get() is called. Return true if the domain exists -bool LMDBBackend::list(const string &target, int zoneId, bool include_disabled) { - DEBUGLOG("list() requested for " < keyparts, valparts; - - stringtok(keyparts,cur_key,"\t"); - stringtok(valparts,cur_value,"\t"); - - if( valparts.size() == 2 && valparts[0] == "REF" ) { - MDB_val extended_key, extended_val; - - // XXX parse into an int and have extended table as MDB_INTEGER to have - // a bit better performance/smaller space? - extended_key.mv_data = (char *)valparts[1].c_str(); - extended_key.mv_size = valparts[1].length(); - - if( int rc = mdb_cursor_get( data_extended_cursor, &extended_key, &extended_val, MDB_SET_KEY ) ) - throw PDNSException("Record " + cur_key + " references extended record " + cur_value + " but this doesn't exist: " + mdb_strerror( rc )); - - cur_value.assign((const char *)extended_val.mv_data, extended_val.mv_size); - valparts.clear(); - stringtok(valparts, cur_value, "\t"); - } - - if (valparts.size() != 3) // FIXME - valparts.push_back("."); - - if( keyparts.size() != 2 || valparts.size() != 3 ) - throw PDNSException("Invalid record in record table: key: '" + cur_key + "'; value: '"+ cur_value+"'"); - - string compare_string = cur_key.substr(0, d_searchkey.length()); - DEBUGLOG( "searchkey: " << d_searchkey << "; compare: " << compare_string << ";" << endl); - - // If we're onto records not beginning with this search prefix, then we - // must be past the end - if( compare_string.compare( d_searchkey ) ) - return get_finished(); - - int domain_id = atoi( valparts[0].c_str() ); - - // If we are doing an AXFR and the record fetched has been outside of our domain then end the transfer - if( is_axfr ) { - // Check it's not a subdomain ie belongs to this record - if( domain_id != d_domain_id ) - goto next_record; - - // If it's under the main domain then append the . to the comparison to - // ensure items outside our zone don't enter - if( keyparts[0].length() > d_querykey.length() ) { - string test = d_querykey; - test.append("."); - - compare_string = cur_key.substr(0, d_querykey.length() + 1); - - DEBUGLOG("test: " << test << "; compare: " << compare_string << ";" << endl); - - if( test.compare( compare_string ) ) - goto next_record; - } - - // We need to maintain query casing so strip off domain (less dot) and append originial query - string sub = keyparts[0].substr( d_origdomain.length(), string::npos ); - rr.qname = string( sub.rbegin(), sub.rend() ) + d_origdomain; - } else - rr.qname = d_origdomain; // use cached and original casing - - DEBUGLOG("Found record: " < -#include -#include "pdns/dnsbackend.hh" - -class LMDBBackend : public DNSReversedBackend -{ -private: - - MDB_env *env; - MDB_dbi data_db, zone_db, data_extended_db, rrsig_db, nsecx_db; - MDB_txn *txn; - MDB_cursor *data_cursor, *zone_cursor, *data_extended_cursor, *rrsig_cursor, *nsecx_cursor; - - // Domain that we are querying for in list()/lookup()/get(). In original case and direction. - string d_origdomain; - - // Current QType being queried for - QType d_curqtype; - - // Is this the first call to ::get() ? - bool d_first; - - // Is dnssec enabled ? - bool d_doDnssec; - - // Current domain ID being queried for - int d_domain_id; - - // The reversed and lowercase key that we are querying in the database. Set after the first ::get() call. - string d_querykey; - - // d_querykey with some additional bits potentially tacked on to make searching faster - string d_searchkey; - - // d_lastreload last time the db was reloaded - int d_lastreload; - - void open_db(); - void close_db(); - void needReload(); - inline bool get_finished(); - static int s_reloadcount; - static pthread_rwlock_t s_initlock; - -public: - LMDBBackend(const string &suffix=""); - ~LMDBBackend(); - bool list(const string &target, int id, bool include_disabled=false); - void lookup(const QType &type, const string &qdomain, DNSPacket *p, int zoneId); - void reload(); - bool get(DNSResourceRecord &rr); - - bool getDomainMetadata(const string& name, const std::string& kind, std::vector& meta); - bool getDirectNSECx(uint32_t id, const string &hashed, const QType &qtype, string &before, DNSResourceRecord &rr); - bool getDirectRRSIGs(const string &signer, const string &qname, const QType &qtype, vector &rrsigs); - - bool getAuthZone( string &rev_zone ); - bool getAuthData( SOAData &, DNSPacket *); -}; - -#endif /* PDNS_LMDBBACKEND_HH */ diff --git a/pdns/.gitignore b/pdns/.gitignore index f929d6025..6d3ce8105 100644 --- a/pdns/.gitignore +++ b/pdns/.gitignore @@ -39,7 +39,6 @@ version_generated.h /zone2ldap /zone2sql /zone2json -/zone2lmdb /bindlexer.c /bindparser.cc /bindparser.h diff --git a/pdns/Makefile.am b/pdns/Makefile.am index f93f34b55..386da5f8c 100644 --- a/pdns/Makefile.am +++ b/pdns/Makefile.am @@ -123,8 +123,7 @@ EXTRA_PROGRAMS = \ testrunner \ toysdig \ tsig-tests \ - zone2ldap \ - zone2lmdb + zone2ldap pdns_server_SOURCES = \ arguments.cc arguments.hh \ @@ -445,40 +444,6 @@ zone2ldap_SOURCES = \ zone2ldap_LDADD = $(MBEDTLS_LIBS) -if LMDB -bin_PROGRAMS += zone2lmdb - -zone2lmdb_SOURCES = \ - arguments.cc \ - base32.cc \ - base64.cc \ - bind-dnssec.schema.sqlite3.sql.h \ - bindlexer.l \ - bindparser.yy \ - bindparserclasses.hh \ - dns.cc \ - dns_random.cc \ - dnslabeltext.cc \ - dnsname.cc dnsname.hh \ - dnsparser.cc \ - dnsrecords.cc \ - dnswriter.cc \ - logger.cc \ - misc.cc \ - nsecrecords.cc \ - qtype.cc \ - rcpgenerator.cc \ - sillyrecords.cc \ - statbag.cc \ - unix_utility.cc \ - zone2lmdb.cc \ - zoneparser-tng.cc - -zone2lmdb_LDADD = \ - $(MBEDTLS_LIBS) \ - $(LMDB_LIBS) -endif - sdig_SOURCES = \ base32.cc \ base64.cc base64.hh \ diff --git a/pdns/zone2lmdb.cc b/pdns/zone2lmdb.cc deleted file mode 100644 index a193d2ab1..000000000 --- a/pdns/zone2lmdb.cc +++ /dev/null @@ -1,315 +0,0 @@ -/* - PowerDNS Versatile Database Driven Nameserver - Copyright (C) 2002 - 2014 PowerDNS.COM BV - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 - as published by the Free Software Foundation - - Additionally, the license of this program contains a special - exception which allows to distribute the program in binary form when - it is linked against OpenSSL. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -*/ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include -#include - -#include -#include -#include "namespaces.hh" - -#include "dns.hh" -#include "arguments.hh" -#include "bindparserclasses.hh" -#include "statbag.hh" -#include "misc.hh" -#include "dnspacket.hh" -#include "zoneparser-tng.hh" -#include "dnsrecords.hh" -#include -#include -#include -#include -#include -#include -#include "base32.hh" - -StatBag S; -int g_numZones=0; -int g_numRecords=0; -int g_numRefs=0; - -MDB_env *env; -MDB_dbi data_db, zone_db, data_extended_db, rrsig_db, nsecx_db; -MDB_txn *txn, *txn_zone; - -void openDB(){ - mdb_env_create(&env); - mdb_env_set_mapsize(env, 1*1024*1024*1024); - mdb_env_set_maxdbs(env, 5); - mdb_env_open(env, "./", 0, 0644); - - mdb_txn_begin(env, NULL, 0, &txn); - - mdb_dbi_open(txn, "zone", MDB_CREATE, &zone_db); - mdb_dbi_open(txn, "data", MDB_CREATE | MDB_DUPSORT, &data_db); - mdb_dbi_open(txn, "extended_data", MDB_CREATE, &data_extended_db); - mdb_dbi_open(txn, "rrsig", MDB_CREATE | MDB_DUPSORT, &rrsig_db); - mdb_dbi_open(txn, "nsecx", MDB_CREATE, &nsecx_db); -} - -void closeDB(){ - mdb_txn_commit(txn); - mdb_dbi_close(env, data_db); - mdb_dbi_close(env, zone_db); - mdb_dbi_close(env, data_extended_db); - mdb_dbi_close(env, rrsig_db); - mdb_dbi_close(env, nsecx_db); - mdb_env_close(env); -} - -string reverse(const string &name) { - return string(name.rbegin(), name.rend()); -} - -void emitData(string zone, ZoneParserTNG &zpt){ - - bool hasSOA=false, isPresigned=false; - int numRefs=g_numRefs; - int numRecords=g_numRecords; - string metaData="1", qname; - SOAData sd; - DNSResourceRecord rr; - MDB_val key, data, keyExt, dataExt; - - zone=toLower(zone); - mdb_txn_begin(env, txn, 0, &txn_zone); - while(zpt.get(rr)) { - numRecords++; - qname=toLower(stripDot(rr.qname)); - if (rr.qtype == QType::SOA) { - hasSOA=true; - fillSOAData(rr.content, sd); - sd.ttl=rr.ttl; - continue; - } - if (rr.qtype == QType::NSEC3PARAM) { - metaData=rr.content; - continue; - } - - string keyStr, dataStr; - - if (rr.qtype == QType::RRSIG) { - isPresigned=true; - RRSIGRecordContent rrc(rr.content); - keyStr=zone+"\t"+makeRelative(qname, zone)+"\t"+DNSRecordContent::NumberToType(rrc.d_type); - dataStr=itoa(rr.ttl)+"\t"+rr.content; - - key.mv_data = (char*)keyStr.c_str(); - key.mv_size = keyStr.length(); - data.mv_data = (char*)dataStr.c_str(); - data.mv_size = dataStr.length(); - - mdb_put(txn_zone, rrsig_db, &key, &data, 0); - continue; - } - - if (rr.qtype == QType::NSEC || rr.qtype == QType::NSEC3) { - if (rr.qtype == QType::NSEC) - keyStr=itoa(g_numZones+1)+"\t"+bitFlip(labelReverse(makeRelative(qname,zone)))+"\xff"; - else - keyStr=itoa(g_numZones+1)+"\t"+toBase32Hex(bitFlip(fromBase32Hex(makeRelative(qname, zone)))); - dataStr=qname+"\t"+itoa(rr.ttl)+"\t"+rr.qtype.getName()+"\t"+rr.content; - - key.mv_data = (char*)keyStr.c_str(); - key.mv_size = keyStr.length(); - data.mv_data = (char*)dataStr.c_str(); - data.mv_size = dataStr.length(); - - mdb_put(txn_zone, nsecx_db, &key, &data, 0); - continue; - } - - keyStr=reverse(qname)+"\t"+rr.qtype.getName(); - dataStr=itoa(g_numZones+1)+"\t"+itoa(rr.ttl)+"\t"+rr.content; - - key.mv_data = (char*)keyStr.c_str(); - key.mv_size = keyStr.length(); - data.mv_data = (char*)dataStr.c_str(); - data.mv_size = dataStr.length(); - - if (dataStr.length() > 500) { - string keyStrExt=itoa(++numRefs); - string dataStrExt="REF\t"+itoa(numRefs); - - keyExt.mv_data = (char*)keyStrExt.c_str(); - keyExt.mv_size = keyStrExt.length(); - dataExt.mv_data = (char*)dataStrExt.c_str(); - dataExt.mv_size = dataStrExt.length(); - - mdb_put(txn_zone, data_extended_db, &keyExt, &data, 0); - mdb_put(txn_zone, data_db, &key, &dataExt, 0); - } else - mdb_put(txn_zone, data_db, &key, &data, 0); - } - if (hasSOA) { - string keyStr=bitFlip(labelReverse(zone))+"\xff"; - string dataStr=itoa(g_numZones+1)+"\t"+itoa(sd.ttl)+"\t"+serializeSOAData(sd); - - if (isPresigned) - dataStr.append("\t"+metaData); - - key.mv_data = (char*)keyStr.c_str(); - key.mv_size = keyStr.length(); - data.mv_data = (char*)dataStr.c_str(); - data.mv_size = dataStr.length(); - - mdb_put(txn_zone, zone_db, &key, &data, 0); - } else { - mdb_txn_abort(txn_zone); - throw PDNSException("Zone'"+zone+"' has no SOA record"); - } - mdb_txn_commit(txn_zone); - g_numZones++; - g_numRecords=numRecords; - g_numRefs=numRefs; -} - -ArgvMap &arg() -{ - static ArgvMap theArg; - return theArg; -} - - -int main(int argc, char **argv) -try -{ - reportAllTypes(); -#if __GNUC__ >= 3 - std::ios_base::sync_with_stdio(false); -#endif - - ::arg().setSwitch("verbose","Verbose comments on operation")="no"; - ::arg().setSwitch("on-error-resume-next","Continue after errors")="no"; - ::arg().set("zone","Zonefile to parse")=""; - ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")=""; - ::arg().set("named-conf","Bind 8/9 named.conf to parse")=""; - - ::arg().set("soa-minimum-ttl","Do not change")="0"; - ::arg().set("soa-refresh-default","Do not change")="0"; - ::arg().set("soa-retry-default","Do not change")="0"; - ::arg().set("soa-expire-default","Do not change")="0"; - - ::arg().setCmd("help","Provide a helpful message"); - - S.declare("logmessages"); - - string namedfile=""; - string zonefile=""; - - ::arg().parse(argc, argv); - - if(::arg().mustDo("help")) { - cout<<"syntax:"< domains=BP.getDomains(); - struct stat st; - for(vector::iterator i=domains.begin(); i!=domains.end(); ++i) { - if(stat(i->filename.c_str(), &st) == 0) { - i->d_dev = st.st_dev; - i->d_ino = st.st_ino; - } - } - - sort(domains.begin(), domains.end()); // put stuff in inode order - - int numdomains=domains.size(); - int tick=numdomains/100; - - cout <<"["; - for(vector::const_iterator i=domains.begin(); i!=domains.end(); ++i) { - if(i->type!="master" && i->type!="slave") { - cerr<<" Warning! Skipping '"<type<<"' zone '"<name<<"'"<filename, i->name, BP.getDirectory()); - emitData(i->name, zpt); - } - catch(std::exception &ae) { - if(!::arg().mustDo("on-error-resume-next")) - throw; - else - cerr<filename<<")\033\133\113"; - } - cout << "]\n"; - cerr<<"\r100% done\033\133\113"< zones/$zone.signed - - echo "" >> named-lmdb.conf - echo "zone \"${zone}\" {" >> named-lmdb.conf - echo " type master;" >> named-lmdb.conf - echo " file \"zones/${zone}.signed\";" >> named-lmdb.conf - echo "};" >> named-lmdb.conf - done - - pids=$(cat pdns*.pid) - - if [ -n "$pids" ] - then - kill $pids - set +e - loopcount=0 - done=0 - while [ $loopcount -lt 10 ] && [ $done -eq 0 ] - do - done=1 - for pid in $pids - do - kill -0 $pid > /dev/null 2>&1 - if [ $? -eq 0 ]; - then - done=0 - fi - done - let loopcount=loopcount+1 - sleep 1 - done - - kill -9 $pids - set -e - fi - rm pdns*.pid - - context=${orgcontext%-zone} - fi - - ${MAKE} -C ../pdns zone2lmdb > /dev/null - rm -f data.mdb lock.mdb - - if [ $context = lmdb-nodnssec ] - then - ../pdns/zone2lmdb --named-conf=./named.conf - else - ../pdns/zone2lmdb --named-conf=./named-lmdb.conf - fi - - $RUNWRAPPER $PDNS --daemon=no --local-port=$port --config-name=lmdb --socket-dir=./ \ - --no-shuffle --launch=lmdb \ - --send-root-referral \ - --cache-ttl=$cachettl --experimental-dname-processing --no-config \ - --lmdb-datapath=./ --lmdb-experimental-dnssec --module-dir=./modules & - - skipreasons="noent nodyndns nometa lmdb" - - if [ $context = lmdb-nsec3 ] - then - extracontexts="dnssec nsec3" - skipreasons="$skipreasons nsec3" - elif [ $context = lmdb-nsec3-optout ] - then - extracontexts="dnssec nsec3 nsec3-optout" - skipreasons="$skipreasons optout" - elif [ $context = lmdb-nsec3-narrow ] - then - extracontexts="dnssec narrow" - skipreasons="$skipreasons narrow" - elif [ $context = lmdb-nodnssec ] - then - skipreasons="$skipreasons nodnssec" - else - extracontexts="dnssec" - skipreasons="$skipreasons" - fi - ;; - - *) - nocontext=yes -esac -- 2.40.0