From: Kees Monshouwer Date: Wed, 28 May 2014 22:02:18 +0000 (+0200) Subject: zone2lmdb and lmdb-backend regression tests X-Git-Tag: auth-3.4.0-rc1~106^2~8 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=cec11b417894a99bc17b6c861c702cb19c89664c;p=thirdparty%2Fpdns.git zone2lmdb and lmdb-backend regression tests --- diff --git a/.travis.yml b/.travis.yml index eee49816ee..2f338e8fbe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,6 +79,7 @@ script: - ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-both - ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-optout-both - ./timestamp ./start-test-stop 5300 gsqlite3-nsec3-narrow + - ./timestamp ./start-test-stop 5300 lmdb-nodnssec - ./timestamp ./start-test-stop 5300 mydns - ./timestamp ./start-test-stop 5300 opendbx-sqlite3 - travis_retry ./timestamp timeout 120s ./start-test-stop 5300 remotebackend-pipe # Workaround for remotebackend failures on travis-ci diff --git a/configure.ac b/configure.ac index 84dec81523..ad7f7ee447 100644 --- a/configure.ac +++ b/configure.ac @@ -247,6 +247,7 @@ for a in $modules $dynmodules; do ;; lmdb) PDNS_WITH_LMDB + needlmdb=yes ;; mydns|gmysql|pdns) PDNS_WITH_MYSQL @@ -271,6 +272,8 @@ done AM_CONDITIONAL([ORACLE],[test "x$needoracle" = "xyes"]) +AM_CONDITIONAL([LMDB],[test "x$needlmdb" = "xyes"]) + AM_CONDITIONAL([SQLITE3], [test "x$needsqlite3" = "xyes"]) if test "$needsqlite3"; then PKG_CHECK_MODULES([SQLITE3], $SQLITE3PC, diff --git a/pdns/.gitignore b/pdns/.gitignore index 2c4f780929..96f8b4054a 100644 --- a/pdns/.gitignore +++ b/pdns/.gitignore @@ -37,6 +37,7 @@ version_generated.h /zone2ldap /zone2sql /zone2json +/zone2lmdb /bindlexer.c /bindparser.cc /bindparser.h diff --git a/pdns/Makefile.am b/pdns/Makefile.am index 177ccb8b6f..17b3f53f29 100644 --- a/pdns/Makefile.am +++ b/pdns/Makefile.am @@ -145,6 +145,20 @@ zone2ldap_SOURCES=bindparser.yy bindlexer.l bind-dnssec.schema.sqlite3.sql.h \ zone2ldap_LDFLAGS=@THREADFLAGS@ zone2ldap_LDADD= $(POLARSSL_LIBS) +if LMDB +bin_PROGRAMS += zone2lmdb + +zone2lmdb_SOURCES=bindparser.yy bindlexer.l bind-dnssec.schema.sqlite3.sql.h \ + arguments.cc logger.cc zone2lmdb.cc statbag.cc misc.cc \ + unix_utility.cc qtype.cc zoneparser-tng.cc dnsrecords.cc \ + dnswriter.cc dnslabeltext.cc rcpgenerator.cc dnsparser.cc base64.cc sillyrecords.cc \ + nsecrecords.cc base32.cc bindparserclasses.hh \ + dns_random.cc dns.cc + +zone2lmdb_LDFLAGS=@THREADFLAGS@ +zone2lmdb_LDADD= $(POLARSSL_LIBS) -llmdb +endif + if BOTAN110 pdnssec_SOURCES += botan110signers.cc botansigners.cc pdnssec_LDADD += $(BOTAN110_LIBS) -lgmp -lrt diff --git a/pdns/zone2lmdb.cc b/pdns/zone2lmdb.cc new file mode 100644 index 0000000000..e7d7b64341 --- /dev/null +++ b/pdns/zone2lmdb.cc @@ -0,0 +1,263 @@ +/* + PowerDNS Versatile Database Driven Nameserver + Copyright (C) 2002 - 2014 PowerDNS.COM BV + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License version 2 + as published by the Free Software Foundation + + Additionally, the license of this program contains a special + exception which allows to distribute the program in binary form when + it is linked against OpenSSL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include +#include +#include + +#include +#include +#include "namespaces.hh" + +#include "dns.hh" +#include "arguments.hh" +#include "bindparserclasses.hh" +#include "statbag.hh" +#include "misc.hh" +#include "dnspacket.hh" +#include "zoneparser-tng.hh" +#include "dnsrecords.hh" +#include +#include +#include +#include +#include +#include + +StatBag S; +int g_numZones=0; +int g_numRecords=0; +int g_numRefs=0; + +MDB_env *env; +MDB_dbi data_db, zone_db, data_extended_db; +MDB_txn *txn, *txn_zone; + +void openDB(){ + mdb_env_create(&env); + mdb_env_set_mapsize(env, 1*1024*1024*1024); + mdb_env_set_maxdbs(env, 3); + mdb_env_open(env, "./", 0, 0644); + + mdb_txn_begin(env, NULL, 0, &txn); + + mdb_dbi_open(txn, "zone", MDB_CREATE, &zone_db); + mdb_dbi_open(txn, "data", MDB_CREATE | MDB_DUPSORT, &data_db); + mdb_dbi_open(txn, "extended_data", MDB_CREATE, &data_extended_db); +} + +void closeDB(){ + mdb_txn_commit(txn); + mdb_dbi_close(env, data_db); + mdb_dbi_close(env, zone_db); + mdb_dbi_close(env, data_extended_db); + mdb_env_close(env); +} + +string reverse(const string &name) { + return toLower(string(name.rbegin(), name.rend())); +} + +void emitData(string zone, ZoneParserTNG &zpt){ + + bool hasSOA = false; + int numRefs=g_numRefs; + int numRecords=g_numRecords; + SOAData sd; + DNSResourceRecord rr; + MDB_val key, data, keyExt, dataExt; + + mdb_txn_begin(env, txn, 0, &txn_zone); + while(zpt.get(rr)) { + numRecords++; + if (rr.qtype == QType::SOA) { + hasSOA = true; + fillSOAData(rr.content, sd); + sd.ttl=rr.ttl; + continue; + } + string keyStr=reverse(stripDot(rr.qname))+"\t"+rr.qtype.getName(); + string dataStr=itoa(g_numZones+1)+"\t"+itoa(rr.ttl)+"\t"+rr.content; + + key.mv_data = (char*)keyStr.c_str(); + key.mv_size = keyStr.length(); + data.mv_data = (char*)dataStr.c_str(); + data.mv_size = dataStr.length(); + + if (dataStr.length() > 500) { + string keyStrExt=itoa(++numRefs); + string dataStrExt="REF\t"+itoa(numRefs); + + keyExt.mv_data = (char*)keyStrExt.c_str(); + keyExt.mv_size = keyStrExt.length(); + dataExt.mv_data = (char*)dataStrExt.c_str(); + dataExt.mv_size = dataStrExt.length(); + + mdb_put(txn_zone, data_extended_db, &keyExt, &data, 0); + mdb_put(txn_zone, data_db, &key, &dataExt, 0); + } else + mdb_put(txn_zone, data_db, &key, &data, 0); + } + if (hasSOA) { + string keyStr=(reverse(stripDot(zone))); + string dataStr=itoa(g_numZones+1)+"\t"+itoa(sd.ttl)+"\t"+serializeSOAData(sd); + + key.mv_data = (char*)keyStr.c_str(); + key.mv_size = keyStr.length(); + data.mv_data = (char*)dataStr.c_str(); + data.mv_size = dataStr.length(); + + mdb_put(txn_zone, zone_db, &key, &data, 0); + } else { + mdb_txn_abort(txn_zone); + throw PDNSException("Zone'"+zone+"' has no SOA record"); + } + mdb_txn_commit(txn_zone); + g_numZones++; + g_numRecords=numRecords; + g_numRefs=numRefs; +} + +ArgvMap &arg() +{ + static ArgvMap theArg; + return theArg; +} + + +int main(int argc, char **argv) +try +{ + reportAllTypes(); + reportFancyTypes(); +#if __GNUC__ >= 3 + std::ios_base::sync_with_stdio(false); +#endif + + ::arg().setSwitch("verbose","Verbose comments on operation")="no"; + ::arg().setSwitch("on-error-resume-next","Continue after errors")="no"; + ::arg().set("zone","Zonefile to parse")=""; + ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")=""; + ::arg().set("named-conf","Bind 8/9 named.conf to parse")=""; + + ::arg().set("soa-minimum-ttl","Do not change")="0"; + ::arg().set("soa-refresh-default","Do not change")="0"; + ::arg().set("soa-retry-default","Do not change")="0"; + ::arg().set("soa-expire-default","Do not change")="0"; + + ::arg().setCmd("help","Provide a helpful message"); + + S.declare("logmessages"); + + string namedfile=""; + string zonefile=""; + + ::arg().parse(argc, argv); + + if(::arg().mustDo("help")) { + cout<<"syntax:"< domains=BP.getDomains(); + struct stat st; + for(vector::iterator i=domains.begin(); i!=domains.end(); ++i) { + if(stat(i->filename.c_str(), &st) == 0) { + i->d_dev = st.st_dev; + i->d_ino = st.st_ino; + } + } + + sort(domains.begin(), domains.end()); // put stuff in inode order + + int numdomains=domains.size(); + int tick=numdomains/100; + + cout <<"["; + openDB(); + for(vector::const_iterator i=domains.begin(); i!=domains.end(); ++i) { + if(i->type!="master" && i->type!="slave") { + cerr<<" Warning! Skipping '"<type<<"' zone '"<name<<"'"<filename, i->name, BP.getDirectory()); + emitData(i->name, zpt); + } + catch(std::exception &ae) { + if(!::arg().mustDo("on-error-resume-next")) + throw; + else + cerr<filename<<")\033\133\113"; + } + cout << "]\n"; + cerr<<"\r100% done\033\133\113"< zones/$zone.signed + done + kill $(cat pdns*.pid) + sleep 2 + context=${orgcontext%-zone} + fi + + ${MAKE} -C ../pdns zone2sql > /dev/null + rm -f data.mdb lock.mdb + ../pdns/zone2lmdb --named-conf=./named.conf + + $RUNWRAPPER $PDNS --daemon=no --local-port=$port --socket-dir=./ \ + --no-shuffle --launch=lmdb \ + --send-root-referral \ + --cache-ttl=$cachettl --experimental-dname-processing --no-config \ + --lmdb-datapath=./ & + + skipreasons="noent nodyndns nometa lmdb" + + if [ $context = lmdb-nsec3 ] + then + extracontexts="dnssec nsec3" + skipreasons="$skipreasons nsec3" + elif [ $context = lmdb-nsec3-optout ] + then + extracontexts="dnssec nsec3 nsec3-optout" + skipreasons="$skipreasons optout" + elif [ $context = lmdb-nsec3-narrow ] + then + extracontexts="dnssec narrow" + skipreasons="$skipreasons narrow" + elif [ $context = lmdb-nodnssec ] + then + skipreasons="$skipreasons nodnssec" + else + extracontexts="dnssec" + skipreasons="$skipreasons" + fi + ;; + + *) + nocontext=yes +esac