From f9cf6d92ed18974133580595bbdef45bfcd73048 Mon Sep 17 00:00:00 2001 From: Kees Monshouwer Date: Thu, 9 May 2013 21:10:32 +0200 Subject: [PATCH] cut the number of database queries in half for AXFR-in --- modules/gmysqlbackend/gmysqlbackend.cc | 9 +- modules/gpgsqlbackend/gpgsqlbackend.cc | 7 +- modules/gsqlite3backend/gsqlite3backend.cc | 9 +- pdns/backends/bind/bindbackend2.cc | 2 +- pdns/backends/bind/bindbackend2.hh | 2 +- pdns/backends/gsql/gsqlbackend.cc | 55 +++++- pdns/backends/gsql/gsqlbackend.hh | 7 +- pdns/dnsbackend.hh | 11 +- pdns/slavecommunicator.cc | 212 ++++++++++----------- 9 files changed, 188 insertions(+), 126 deletions(-) diff --git a/modules/gmysqlbackend/gmysqlbackend.cc b/modules/gmysqlbackend/gmysqlbackend.cc index ca398b2214..ffa73c535b 100644 --- a/modules/gmysqlbackend/gmysqlbackend.cc +++ b/modules/gmysqlbackend/gmysqlbackend.cc @@ -87,9 +87,14 @@ public: declare(suffix,"info-all-slaves-query","","select id,name,master,last_check,type from domains where type='SLAVE'"); declare(suffix,"supermaster-query","", "select account from supermasters where ip='%s' and nameserver='%s'"); declare(suffix,"insert-slave-query","", "insert into domains (type,name,master,account) values('SLAVE','%s','%s','%s')"); + declare(suffix,"insert-record-query","", "insert into records (content,ttl,prio,type,domain_id,name) values ('%s',%d,%d,'%s',%d,'%s')"); - declare(suffix,"insert-record-query-auth","", "insert into records (content,ttl,prio,type,domain_id,name,auth) values ('%s',%d,%d,'%s',%d,'%s', '%d')"); - + declare(suffix,"insert-record-query-auth","", "insert into records (content,ttl,prio,type,domain_id,name,auth) values ('%s',%d,%d,'%s',%d,'%s','%d')"); + declare(suffix,"insert-record-order-query-auth","", "insert into records (content,ttl,prio,type,domain_id,name,ordername,auth) values ('%s',%d,%d,'%s',%d,'%s','%s','%d')"); + declare(suffix,"insert-ent-query", "insert empty non-terminal in zone", "insert into records (type,domain_id,name) values (null,'%d','%s')"); + declare(suffix,"insert-ent-query-auth", "insert empty non-terminal in zone", "insert into records (type,domain_id,name,auth) values (null,'%d','%s','1')"); + declare(suffix,"insert-ent-order-query-auth", "insert empty non-terminal in zone", "insert into records (type,domain_id,name,ordername,auth) values (null,'%d','%s','%s','1')"); + declare(suffix,"get-order-first-query","DNSSEC Ordering Query, first", "select ordername, name from records where domain_id=%d and ordername is not null order by 1 asc limit 1"); declare(suffix,"get-order-before-query","DNSSEC Ordering Query, before", "select ordername, name from records where ordername <= '%s' and domain_id=%d and ordername is not null order by 1 desc limit 1"); declare(suffix,"get-order-after-query","DNSSEC Ordering Query, after", "select min(ordername) from records where ordername > '%s' and domain_id=%d and ordername is not null"); diff --git a/modules/gpgsqlbackend/gpgsqlbackend.cc b/modules/gpgsqlbackend/gpgsqlbackend.cc index 08c522e985..2c27ca56e0 100644 --- a/modules/gpgsqlbackend/gpgsqlbackend.cc +++ b/modules/gpgsqlbackend/gpgsqlbackend.cc @@ -87,8 +87,13 @@ public: declare(suffix,"info-all-slaves-query","","select id,name,master,last_check,type from domains where type='SLAVE'"); declare(suffix,"supermaster-query","", "select account from supermasters where ip='%s' and nameserver=E'%s'"); declare(suffix,"insert-slave-query","", "insert into domains (type,name,master,account) values('SLAVE',E'%s',E'%s',E'%s')"); + declare(suffix,"insert-record-query","", "insert into records (content,ttl,prio,type,domain_id,name) values (E'%s',%d,%d,'%s',%d,E'%s')"); - declare(suffix,"insert-record-query-auth","", "insert into records (content,ttl,prio,type,domain_id,name,auth) values (E'%s',%d,%d,'%s',%d,E'%s', '%d')"); + declare(suffix,"insert-record-query-auth","", "insert into records (content,ttl,prio,type,domain_id,name,auth) values (E'%s',%d,%d,'%s',%d,E'%s','%d')"); + declare(suffix,"insert-record-order-query-auth","", "insert into records (content,ttl,prio,type,domain_id,name,ordername,auth) values (E'%s',%d,%d,'%s',%d,E'%s',E'%s','%d')"); + declare(suffix,"insert-ent-query", "insert empty non-terminal in zone", "insert into records (type,domain_id,name) values (null,'%d',E'%s')"); + declare(suffix,"insert-ent-query-auth", "insert empty non-terminal in zone", "insert into records (type,domain_id,name,auth) values (null,'%d',E'%s',true)"); + declare(suffix,"insert-ent-order-query-auth", "insert empty non-terminal in zone", "insert into records (type,domain_id,name,ordername,auth) values (null,'%d',E'%s',E'%s',true)"); declare(suffix,"get-order-first-query","DNSSEC Ordering Query, last", "select ordername, name from records where domain_id=%d and ordername is not null order by 1 using ~<~ limit 1"); declare(suffix,"get-order-before-query","DNSSEC Ordering Query, before", "select ordername, name from records where ordername ~<=~ E'%s' and domain_id=%d and ordername is not null order by 1 using ~>~ limit 1"); diff --git a/modules/gsqlite3backend/gsqlite3backend.cc b/modules/gsqlite3backend/gsqlite3backend.cc index b9c1d7ed25..02dc4fee10 100644 --- a/modules/gsqlite3backend/gsqlite3backend.cc +++ b/modules/gsqlite3backend/gsqlite3backend.cc @@ -106,9 +106,14 @@ public: declare( suffix, "info-all-slaves-query", "","select id,name,master,last_check,type from domains where type='SLAVE'"); declare( suffix, "supermaster-query", "", "select account from supermasters where ip='%s' and nameserver='%s'"); declare( suffix, "insert-slave-query", "", "insert into domains (type,name,master,account) values('SLAVE','%s','%s','%s')"); + declare( suffix, "insert-record-query", "", "insert into records (content,ttl,prio,type,domain_id,name) values ('%s',%d,%d,'%s',%d,'%s')"); - declare( suffix, "insert-record-query-auth", "", "insert into records (content,ttl,prio,type,domain_id,name,auth) values ('%s',%d,%d,'%s',%d,'%s', %d)"); - + declare( suffix, "insert-record-query-auth", "", "insert into records (content,ttl,prio,type,domain_id,name,auth) values ('%s',%d,%d,'%s',%d,'%s',%d)"); + declare( suffix, "insert-record-order-query-auth","", "insert into records (content,ttl,prio,type,domain_id,name,ordername,auth) values ('%s',%d,%d,'%s',%d,'%s','%s','%d')"); + declare( suffix, "insert-ent-query", "insert empty non-terminal in zone", "insert into records (type,domain_id,name) values (null,'%d','%s')"); + declare( suffix, "insert-ent-query-auth", "insert empty non-terminal in zone", "insert into records (type,domain_id,name,auth) values (null,'%d','%s','1')"); + declare( suffix, "insert-ent-order-query-auth", "insert empty non-terminal in zone", "insert into records (type,domain_id,name,ordername,auth) values (null,'%d','%s','%s','1')"); + declare( suffix, "update-serial-query", "", "update domains set notified_serial=%d where id=%d"); declare( suffix, "update-lastcheck-query", "", "update domains set last_check=%d where id=%d"); declare (suffix, "zone-lastchange-query", "", "select max(change_date) from records where domain_id=%d"); diff --git a/pdns/backends/bind/bindbackend2.cc b/pdns/backends/bind/bindbackend2.cc index 6e65847a6f..fd4e83991c 100644 --- a/pdns/backends/bind/bindbackend2.cc +++ b/pdns/backends/bind/bindbackend2.cc @@ -233,7 +233,7 @@ bool Bind2Backend::updateDNSSECOrderAndAuthAbsolute(uint32_t domain_id, const st return false; } -bool Bind2Backend::feedRecord(const DNSResourceRecord &r) +bool Bind2Backend::feedRecord(const DNSResourceRecord &r, string *ordername) { string qname=r.qname; diff --git a/pdns/backends/bind/bindbackend2.hh b/pdns/backends/bind/bindbackend2.hh index 49e6db4155..87559bf835 100644 --- a/pdns/backends/bind/bindbackend2.hh +++ b/pdns/backends/bind/bindbackend2.hh @@ -141,7 +141,7 @@ public: void setNotified(uint32_t id, uint32_t serial); bool startTransaction(const string &qname, int id); // bool Bind2Backend::stopTransaction(const string &qname, int id); - bool feedRecord(const DNSResourceRecord &r); + bool feedRecord(const DNSResourceRecord &r, string *ordername=0); bool commitTransaction(); bool abortTransaction(); bool updateDNSSECOrderAndAuthAbsolute(uint32_t domain_id, const std::string& qname, const std::string& ordername, bool auth); diff --git a/pdns/backends/gsql/gsqlbackend.cc b/pdns/backends/gsql/gsqlbackend.cc index 14af9c64a2..918b872c65 100644 --- a/pdns/backends/gsql/gsqlbackend.cc +++ b/pdns/backends/gsql/gsqlbackend.cc @@ -34,6 +34,8 @@ #include "pdns/ahuexception.hh" #include "pdns/logger.hh" #include "pdns/arguments.hh" +#include "pdns/base32.hh" +#include "pdns/dnssecinfra.hh" #include #include #include @@ -273,6 +275,7 @@ GSQLBackend::GSQLBackend(const string &mode, const string &suffix) d_SuperMasterInfoQuery=getArg("supermaster-query"); d_InsertSlaveZoneQuery=getArg("insert-slave-query"); d_InsertRecordQuery=getArg("insert-record-query"+authswitch); + d_InsertEntQuery=getArg("insert-ent-query"+authswitch); d_UpdateSerialOfZoneQuery=getArg("update-serial-query"); d_UpdateLastCheckofZoneQuery=getArg("update-lastcheck-query"); d_ZoneLastChangeQuery=getArg("zone-lastchange-query"); @@ -287,6 +290,9 @@ GSQLBackend::GSQLBackend(const string &mode, const string &suffix) if (d_dnssecQueries) { + d_InsertRecordOrderQuery=getArg("insert-record-order-query-auth"); + d_InsertEntOrderQuery=getArg("insert-ent-order-query-auth"); + d_firstOrderQuery = getArg("get-order-first-query"); d_beforeOrderQuery = getArg("get-order-before-query"); d_afterOrderQuery = getArg("get-order-after-query"); @@ -884,15 +890,18 @@ bool GSQLBackend::replaceRRSet(uint32_t domain_id, const string& qname, const QT return true; } -bool GSQLBackend::feedRecord(const DNSResourceRecord &r) +bool GSQLBackend::feedRecord(const DNSResourceRecord &r, string *ordername) { string output; if(d_dnssecQueries) { - output = (boost::format(d_InsertRecordQuery) % sqlEscape(r.content) % r.ttl % r.priority % sqlEscape(r.qtype.getName()) % r.domain_id % toLower(sqlEscape(r.qname)) % (int)r.auth).str(); + if(ordername) + output = (boost::format(d_InsertRecordOrderQuery) % sqlEscape(r.content) % r.ttl % r.priority % sqlEscape(r.qtype.getName()) % r.domain_id % toLower(sqlEscape(r.qname)) % sqlEscape(*ordername) % (int)r.auth).str(); + else + output = (boost::format(d_InsertRecordQuery) % sqlEscape(r.content) % r.ttl % r.priority % sqlEscape(r.qtype.getName()) % r.domain_id % toLower(sqlEscape(r.qname)) % (int)r.auth).str(); } else { output = (boost::format(d_InsertRecordQuery) % sqlEscape(r.content) % r.ttl % r.priority % sqlEscape(r.qtype.getName()) % r.domain_id % toLower(sqlEscape(r.qname))).str(); } - + try { d_db->doCommand(output.c_str()); } @@ -902,6 +911,46 @@ bool GSQLBackend::feedRecord(const DNSResourceRecord &r) return true; // XXX FIXME this API should not return 'true' I think -ahu } +bool GSQLBackend::feedEnts(int domain_id, set& nonterm) +{ + string output; + BOOST_FOREACH(const string qname, nonterm) { + output = (boost::format(d_InsertEntQuery) % domain_id % toLower(sqlEscape(qname))).str(); + + try { + d_db->doCommand(output.c_str()); + } + catch (SSqlException &e) { + throw AhuException("GSQLBackend unable to feed empty non-terminal: "+e.txtReason()); + } + } + return true; +} + +bool GSQLBackend::feedEnts3(int domain_id, const string &domain, set &nonterm, unsigned int times, const string &salt, bool narrow) +{ + if(!d_dnssecQueries) + return false; + + string ordername, output; + BOOST_FOREACH(const string qname, nonterm) { + if(narrow) { + output = (boost::format(d_InsertEntQuery) % domain_id % toLower(sqlEscape(qname))).str(); + } else { + ordername=toBase32Hex(hashQNameWithSalt(times, salt, qname)); + output = (boost::format(d_InsertEntOrderQuery) % domain_id % toLower(sqlEscape(qname)) % toLower(sqlEscape(ordername))).str(); + } + + try { + d_db->doCommand(output.c_str()); + } + catch (SSqlException &e) { + throw AhuException("GSQLBackend unable to feed empty non-terminal: "+e.txtReason()); + } + } + return true; +} + bool GSQLBackend::startTransaction(const string &domain, int domain_id) { char output[1024]; diff --git a/pdns/backends/gsql/gsqlbackend.hh b/pdns/backends/gsql/gsqlbackend.hh index 7bd3a0836b..ad35a341bd 100644 --- a/pdns/backends/gsql/gsqlbackend.hh +++ b/pdns/backends/gsql/gsqlbackend.hh @@ -32,7 +32,9 @@ public: bool startTransaction(const string &domain, int domain_id=-1); bool commitTransaction(); bool abortTransaction(); - bool feedRecord(const DNSResourceRecord &r); + bool feedRecord(const DNSResourceRecord &r, string *ordername=0); + bool feedEnts(int domain_id, set& nonterm); + bool feedEnts3(int domain_id, const string &domain, set &nonterm, unsigned int times, const string &salt, bool narrow); bool createSlaveDomain(const string &ip, const string &domain, const string &account); bool superMasterBackend(const string &ip, const string &domain, const vector&nsset, string *account, DNSBackend **db); void setFresh(uint32_t domain_id); @@ -86,6 +88,9 @@ private: string d_SuperMasterInfoQuery; string d_InsertSlaveZoneQuery; string d_InsertRecordQuery; + string d_InsertEntQuery; + string d_InsertRecordOrderQuery; + string d_InsertEntOrderQuery; string d_UpdateSerialOfZoneQuery; string d_UpdateLastCheckofZoneQuery; string d_InfoOfAllMasterDomainsQuery; diff --git a/pdns/dnsbackend.hh b/pdns/dnsbackend.hh index cea51c6612..2dae53cba0 100644 --- a/pdns/dnsbackend.hh +++ b/pdns/dnsbackend.hh @@ -201,10 +201,19 @@ public: } //! feeds a record to a zone, needs a call to startTransaction first - virtual bool feedRecord(const DNSResourceRecord &rr) + virtual bool feedRecord(const DNSResourceRecord &rr, string *ordername=0) { return false; // no problem! } + virtual bool feedEnts(int domain_id, set &nonterm) + { + return false; + } + virtual bool feedEnts3(int domain_id, const string &domain, set &nonterm, unsigned int times, const string &salt, bool narrow) + { + return false; + } + //! if this returns true, DomainInfo di contains information about the domain virtual bool getDomainInfo(const string &domain, DomainInfo &di) { diff --git a/pdns/slavecommunicator.cc b/pdns/slavecommunicator.cc index e20d033e27..e610f227d1 100644 --- a/pdns/slavecommunicator.cc +++ b/pdns/slavecommunicator.cc @@ -92,15 +92,6 @@ void CommunicatorClass::suck(const string &domain,const string &remote) const bool hadPresigned = dk.isPresigned(domain); const bool hadDnssecZone = dnssecZone; - if(dnssecZone) { - if(!haveNSEC3) - L<getDomainInfo(domain, di) || !di.backend) { // di.backend and B are mostly identical L< nsset, qnames, dsnames, nonterm, delnonterm; + set nsset, qnames; ComboAddress raddr(remote, 53); @@ -163,26 +154,22 @@ void CommunicatorClass::suck(const string &domain,const string &remote) bool gotNSEC3 = false; bool gotOptOutFlag = false; unsigned int soa_serial = 0; + vector rrs; while(retriever.getChunk(recs)) { if(first) { - L<startTransaction(domain, domain_id); + L<qtype.getCode() == QType::OPT || i->qtype.getCode() == QType::TSIG) // ignore EDNS0 & TSIG continue; - - if(i->qtype.getCode() == QType::SOA) { - if(soa_serial != 0) - continue; //skip the last SOA - SOAData sd; - fillSOAData(i->content,sd); - soa_serial = sd.serial; + + if(!endsOn(i->qname, domain)) { + L<qname<<"'|"<qtype.getName()<<" during AXFR of zone '"<qtype.getCode() == QType::NSEC3PARAM) { ns3pr = NSEC3PARAMRecordContent(i->content); narrow = false; @@ -197,126 +184,123 @@ void CommunicatorClass::suck(const string &domain,const string &remote) continue; } - if(!endsOn(i->qname, domain)) { - L<qname<<"'|"<qtype.getName()<<" during AXFR of zone '"<qtype.getCode() == QType::SOA) { + if(soa_serial != 0) + continue; //skip the last SOA + SOAData sd; + fillSOAData(i->content,sd); + soa_serial = sd.serial; } - + i->domain_id=domain_id; if (i->qtype.getCode() == QType::SRV) i->content = stripDot(i->content); + #if 0 if(i->qtype.getCode()>=60000) throw DBException("Database can't store unknown record type "+lexical_cast(i->qtype.getCode()-1024)); #endif + vector out; if(pdl && pdl->axfrfilter(raddr, domain, *i, out)) { BOOST_FOREACH(const DNSResourceRecord& rr, out) { - di.backend->feedRecord(rr); - if(rr.qtype.getCode() == QType::NS && !pdns_iequals(rr.qname, domain)) - nsset.insert(rr.qname); - if(rr.qtype.getCode() != QType::RRSIG) // this excludes us hashing RRSIGs for NSEC(3) - qnames.insert(rr.qname); - if(i->qtype.getCode() == QType::DS) - dsnames.insert(i->qname); + rrs.push_back(rr); } - } - else { - di.backend->feedRecord(*i); - if(i->qtype.getCode() == QType::NS && !pdns_iequals(i->qname, domain)) - nsset.insert(i->qname); - if(i->qtype.getCode() != QType::RRSIG) // this excludes us hashing RRSIGs for NSEC(3) - qnames.insert(i->qname); - if(i->qtype.getCode() == QType::DS) - dsnames.insert(i->qname); + } else { + rrs.push_back(*i); } } } - if (hadPresigned && !gotNSEC3) - { + BOOST_FOREACH(const DNSResourceRecord& rr, rrs) { + if(rr.qtype.getCode() == QType::NS && !pdns_iequals(rr.qname, domain)) + nsset.insert(rr.qname); + qnames.insert(rr.qname); + } + + if(dnssecZone) { + if(!haveNSEC3) + L<startTransaction(domain, domain_id); - if(haveNSEC3) - { - if(!narrow) { - hashed=toLower(toBase32Hex(hashQNameWithSalt(ns3pr.d_iterations, ns3pr.d_salt, qname))); - di.backend->updateDNSSECOrderAndAuthAbsolute(domain_id, qname, hashed, auth); + bool doent=true; + uint32_t maxent = ::arg().asNum("max-ent-entries"); + string ordername, shorter; + set nonterm, rrterm; + + BOOST_FOREACH(DNSResourceRecord& rr, rrs) { + + // Figure out auth and ents + rr.auth=true; + shorter=rr.qname; + rrterm.clear(); + do { + if(doent) { + if (!qnames.count(shorter) && !nonterm.count(shorter) && !rrterm.count(shorter)) + rrterm.insert(shorter); } - else - di.backend->nullifyDNSSECOrderNameAndUpdateAuth(domain_id, qname, auth); - } - else // NSEC - { - di.backend->updateDNSSECOrderAndAuth(domain_id, domain, qname, auth); - if (!realrr) - di.backend->nullifyDNSSECOrderNameAndUpdateAuth(domain_id, qname, auth); - } - - if(realrr) - { - if (dsnames.count(qname)) - di.backend->setDNSSECAuthOnDsRecord(domain_id, qname); - if (!auth || nsset.count(qname)) { - if(haveNSEC3 && gotOptOutFlag) - di.backend->nullifyDNSSECOrderNameAndAuth(domain_id, qname, "NS"); - di.backend->nullifyDNSSECOrderNameAndAuth(domain_id, qname, "A"); - di.backend->nullifyDNSSECOrderNameAndAuth(domain_id, qname, "AAAA"); + if(nsset.count(shorter) && rr.qtype.getCode() != QType::DS) { + rr.auth=false; + break; } - - if(auth && doent) - { - shorter=qname; - while(!pdns_iequals(shorter, domain) && chopOff(shorter)) - { - if(!qnames.count(shorter) && !nonterm.count(shorter)) - { - if(!(maxent)) - { - L< maxent) { + L<feedRecord(rr, &ordername); + } else + di.backend->feedRecord(rr); + } else { + // NSEC + if (rr.auth || rr.qtype.getCode() == QType::NS) { + ordername=toLower(labelReverse(makeRelative(rr.qname, domain))); + di.backend->feedRecord(rr, &ordername); + } else + di.backend->feedRecord(rr); + } + } else + di.backend->feedRecord(rr); } - if(!nonterm.empty() && realrr && doent) - { - if(di.backend->updateEmptyNonTerminals(domain_id, domain, nonterm, delnonterm, false)) - { - realrr=false; - qnames=nonterm; - goto dononterm; - } + // Insert empty non-terminals + if(doent && !nonterm.empty()) { + if (haveNSEC3) { + di.backend->feedEnts3(domain_id, domain, nonterm, ns3pr.d_iterations, ns3pr.d_salt, narrow); + } else + di.backend->feedEnts(domain_id, nonterm); } // now we also need to update the presigned flag and NSEC3PARAM -- 2.47.2