From 06ea90158e37de98e04deb0a158b34e26bebe497 Mon Sep 17 00:00:00 2001 From: bert hubert Date: Wed, 23 Apr 2014 22:33:18 +0200 Subject: [PATCH] make pdns-distributes-queries use a hash so queries get sent to the same thread. Original ideal from https://github.com/paddg/pdns/commit/9e176976e2499998787e15a45fbe5da250d20fc1 Astoundingly effective, approximately halves CPU usage. --- pdns/dns.cc | 19 ++++++++++++++++ pdns/dns.hh | 1 + pdns/dnspcap.hh | 1 - pdns/misc.cc | 53 +++++++++++++++++++++++++++++++++++++++++++ pdns/misc.hh | 1 + pdns/pdns_recursor.cc | 17 ++++++++------ pdns/syncres.hh | 2 +- 7 files changed, 85 insertions(+), 9 deletions(-) diff --git a/pdns/dns.cc b/pdns/dns.cc index 34999b550a..db23944a28 100644 --- a/pdns/dns.cc +++ b/pdns/dns.cc @@ -128,6 +128,25 @@ bool dnspacketLessThan(const std::string& a, const std::string& b) return boost::tie(aQtype, aQclass) < boost::tie(bQtype, bQclass); } +// goal is to has based purely on the question name, and turn error into 'default' +uint32_t hashQuestion(const char* packet, uint16_t len, uint32_t init) +{ + if(len < 12) + return init; + + uint32_t ret=init; + const unsigned char* end = (const unsigned char*)packet+len; + const unsigned char* pos = (const unsigned char*)packet+12; + + unsigned char labellen; + while((labellen=*pos++) && pos < end) { + if(pos + labellen + 1> end) + return 0; + ret=burtle(pos, labellen+1, ret); + pos += labellen; + } + return ret; +} string questionExpand(const char* packet, uint16_t len, uint16_t& type) { diff --git a/pdns/dns.hh b/pdns/dns.hh index 7f7146f202..2e9d5ef6e1 100644 --- a/pdns/dns.hh +++ b/pdns/dns.hh @@ -283,6 +283,7 @@ struct dnsheader { #define L theL() extern time_t s_starttime; std::string questionExpand(const char* packet, uint16_t len, uint16_t& type); +uint32_t hashQuestion(const char* packet, uint16_t len, uint32_t init); bool dnspacketLessThan(const std::string& a, const std::string& b); /** helper function for both DNSPacket and addSOARecord() - converts a line into a struct, for easier parsing */ diff --git a/pdns/dnspcap.hh b/pdns/dnspcap.hh index 1abc40c5a8..d18b69a7f2 100644 --- a/pdns/dnspcap.hh +++ b/pdns/dnspcap.hh @@ -83,7 +83,6 @@ public: bool getUDPPacket(); - ComboAddress getSource() const; ComboAddress getDest() const; diff --git a/pdns/misc.cc b/pdns/misc.cc index 53e9802552..db9ea96751 100644 --- a/pdns/misc.cc +++ b/pdns/misc.cc @@ -847,3 +847,56 @@ void setFilenumLimit(unsigned int lim) if(setrlimit(RLIMIT_NOFILE, &rlim) < 0) unixDie("Setting number of available file descriptors"); } + +#define burtlemix(a,b,c) \ +{ \ + a -= b; a -= c; a ^= (c>>13); \ + b -= c; b -= a; b ^= (a<<8); \ + c -= a; c -= b; c ^= (b>>13); \ + a -= b; a -= c; a ^= (c>>12); \ + b -= c; b -= a; b ^= (a<<16); \ + c -= a; c -= b; c ^= (b>>5); \ + a -= b; a -= c; a ^= (c>>3); \ + b -= c; b -= a; b ^= (a<<10); \ + c -= a; c -= b; c ^= (b>>15); \ +} + +uint32_t burtle(const unsigned char* k, uint32_t length, uint32_t initval) +{ + uint32_t a,b,c,len; + + /* Set up the internal state */ + len = length; + a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ + c = initval; /* the previous hash value */ + + /*---------------------------------------- handle most of the key */ + while (len >= 12) { + a += (k[0] +((uint32_t)k[1]<<8) +((uint32_t)k[2]<<16) +((uint32_t)k[3]<<24)); + b += (k[4] +((uint32_t)k[5]<<8) +((uint32_t)k[6]<<16) +((uint32_t)k[7]<<24)); + c += (k[8] +((uint32_t)k[9]<<8) +((uint32_t)k[10]<<16)+((uint32_t)k[11]<<24)); + burtlemix(a,b,c); + k += 12; len -= 12; + } + + /*------------------------------------- handle the last 11 bytes */ + c += length; + switch(len) { /* all the case statements fall through */ + case 11: c+=((uint32_t)k[10]<<24); + case 10: c+=((uint32_t)k[9]<<16); + case 9 : c+=((uint32_t)k[8]<<8); + /* the first byte of c is reserved for the length */ + case 8 : b+=((uint32_t)k[7]<<24); + case 7 : b+=((uint32_t)k[6]<<16); + case 6 : b+=((uint32_t)k[5]<<8); + case 5 : b+=k[4]; + case 4 : a+=((uint32_t)k[3]<<24); + case 3 : a+=((uint32_t)k[2]<<16); + case 2 : a+=((uint32_t)k[1]<<8); + case 1 : a+=k[0]; + /* case 0: nothing left to add */ + } + burtlemix(a,b,c); + /*-------------------------------------------- report the result */ + return c; +} diff --git a/pdns/misc.hh b/pdns/misc.hh index 917fd36ccf..4712e28ac1 100644 --- a/pdns/misc.hh +++ b/pdns/misc.hh @@ -528,4 +528,5 @@ void addCMsgSrcAddr(struct msghdr* msgh, void* cmsgbuf, ComboAddress* source); unsigned int getFilenumLimit(bool hardOrSoft=0); void setFilenumLimit(unsigned int lim); bool readFileIfThere(const char* fname, std::string* line); +uint32_t burtle(const unsigned char* k, uint32_t lengh, uint32_t init); #endif diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index cd16f00f1a..595c674ee5 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -947,7 +947,7 @@ void handleNewUDPQuestion(int fd, FDMultiplexer::funcparam_t& var) else { string question(data, len); if(g_weDistributeQueries) - distributeAsyncFunction(boost::bind(doProcessUDPQuestion, question, fromaddr, fd)); + distributeAsyncFunction(question, boost::bind(doProcessUDPQuestion, question, fromaddr, fd)); else doProcessUDPQuestion(question, fromaddr, fd); } @@ -1272,11 +1272,13 @@ void broadcastFunction(const pipefunc_t& func, bool skipSelf) } } } -void distributeAsyncFunction(const pipefunc_t& func) + +uint32_t g_disthashseed; +void distributeAsyncFunction(const std::string& question, const pipefunc_t& func) { - static unsigned int counter; - unsigned int target = 1 + (++counter % (g_pipes.size()-1)); - // cerr<<"Sending to: "<wantAnswer = false; if(write(tps.writeToThread, &tmsg, sizeof(tmsg)) != sizeof(tmsg)) - unixDie("write to thread pipe returned wrong size or error"); - + unixDie("write to thread pipe returned wrong size or error"); } void handlePipeRequest(int fd, FDMultiplexer::funcparam_t& var) @@ -1759,6 +1760,8 @@ int serviceMain(int argc, char*argv[]) showProductVersion(); seedRandom(::arg()["entropy-source"]); + g_disthashseed=dns_random(0xffffffff); + parseACLs(); if(!::arg()["dont-query"].empty()) { diff --git a/pdns/syncres.hh b/pdns/syncres.hh index 6a8840b838..4c855b01d5 100644 --- a/pdns/syncres.hh +++ b/pdns/syncres.hh @@ -622,7 +622,7 @@ ComboAddress parseIPAndPort(const std::string& input, uint16_t port); ComboAddress getQueryLocalAddress(int family, uint16_t port); typedef boost::function pipefunc_t; void broadcastFunction(const pipefunc_t& func, bool skipSelf = false); -void distributeAsyncFunction(const pipefunc_t& func); +void distributeAsyncFunction(const std::string& question, const pipefunc_t& func); int directResolve(const std::string& qname, const QType& qtype, int qclass, vector& ret); -- 2.47.2