From: Remi Gacogne Date: Mon, 24 Dec 2018 13:06:09 +0000 (+0100) Subject: rec: Add support for loading the Public Suffix List from a file X-Git-Tag: rec-4.2.0-alpha1~74^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d6f3fcfa4e559b9d14027da315067f6f6043512d;p=thirdparty%2Fpdns.git rec: Add support for loading the Public Suffix List from a file --- diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 28360e42b2..783a847e33 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -87,6 +87,7 @@ #include "rec-lua-conf.hh" #include "ednsoptions.hh" #include "gettime.hh" +#include "pubsuffix.hh" #ifdef NOD_ENABLED #include "nod.hh" #endif /* NOD_ENABLED */ @@ -3517,7 +3518,7 @@ static int serviceMain(int argc, char*argv[]) } parseACLs(); - sortPublicSuffixList(); + initPublicSuffixList(::arg()["public-suffix-list-file"]); if(!::arg()["dont-query"].empty()) { vector ips; @@ -4215,6 +4216,7 @@ int main(int argc, char **argv) ::arg().set("udp-source-port-max", "Maximum UDP port to bind on")="65535"; ::arg().set("udp-source-port-avoid", "List of comma separated UDP port number to avoid")="11211"; ::arg().set("rng", "Specify random number generator to use. Valid values are auto,sodium,openssl,getrandom,arc4random,urandom.")="auto"; + ::arg().set("public-suffix-list-file", "Path to the Public Suffix List file, if any")=""; #ifdef NOD_ENABLED ::arg().set("new-domain-tracking", "Track newly observed domains (i.e. never seen before).")="no"; ::arg().set("new-domain-log", "Log newly observed domains.")="yes"; diff --git a/pdns/pubsuffix.hh b/pdns/pubsuffix.hh index 721200ded7..cc5f7c4272 100644 --- a/pdns/pubsuffix.hh +++ b/pdns/pubsuffix.hh @@ -19,6 +19,15 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -#ifndef PDNS_PUBSUFFIX_HH -extern const char* g_pubsuffix[]; -#endif +#pragma once + +#include +#include + +extern std::vector> g_pubs; + +/* initialize the g_pubs variable with the public suffix list, + using the file passed in parameter if any, or the built-in + list otherwise. +*/ +void initPublicSuffixList(const std::string& file); diff --git a/pdns/rec_channel.hh b/pdns/rec_channel.hh index 0fadd41259..0f1dd7973e 100644 --- a/pdns/rec_channel.hh +++ b/pdns/rec_channel.hh @@ -65,7 +65,6 @@ public: std::map getAllStatsMap(); extern pthread_mutex_t g_carbon_config_lock; -void sortPublicSuffixList(); std::vector >* pleaseGetQueryRing(); std::vector >* pleaseGetServfailQueryRing(); std::vector >* pleaseGetBogusQueryRing(); diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc index 53eebc3d8c..c318948cb8 100644 --- a/pdns/rec_channel_rec.cc +++ b/pdns/rec_channel_rec.cc @@ -1208,24 +1208,6 @@ string doGenericTopRemotes(pleaseremotefunc_t func) return ret.str(); } -namespace { - typedef vector > pubs_t; - pubs_t g_pubs; -} - -void sortPublicSuffixList() -{ - for(const char** p=g_pubsuffix; *p; ++p) { - string low=toLower(*p); - - vector parts; - stringtok(parts, low, "."); - reverse(parts.begin(), parts.end()); - g_pubs.push_back(parts); - } - sort(g_pubs.begin(), g_pubs.end()); -} - // XXX DNSName Pain - this function should benefit from native DNSName methods DNSName getRegisteredName(const DNSName& dom) { diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am index f3ae8afe2a..cca58a8ad8 100644 --- a/pdns/recursordist/Makefile.am +++ b/pdns/recursordist/Makefile.am @@ -134,6 +134,7 @@ pdns_recursor_SOURCES = \ pollmplexer.cc \ protobuf.cc protobuf.hh \ pubsuffix.hh pubsuffix.cc \ + pubsuffixloader.cc \ qtype.hh qtype.cc \ rcpgenerator.cc rcpgenerator.hh \ rec-carbon.cc \ diff --git a/pdns/recursordist/docs/settings.rst b/pdns/recursordist/docs/settings.rst index 31f66737de..4dcec27b0f 100644 --- a/pdns/recursordist/docs/settings.rst +++ b/pdns/recursordist/docs/settings.rst @@ -1075,6 +1075,17 @@ maximizing the cache hit ratio. Starting with version 4.2.0, more than one distr setting. Improves performance on Linux. +.. _settting-public-suffix-list-file: + +``public-suffix-list-file`` +--------------------------- +.. versionadded:: 4.2.0 + +- Path +- Default: unset + +Path to the Public Suffix List file, if any. If set, PowerDNS will try to load the Public Suffix List from this file instead of using the built-in list. The PSL is used to group the queries by relevant domain names when displaying the top queries. + .. _setting-query-local-address: ``query-local-address`` diff --git a/pdns/recursordist/pubsuffixloader.cc b/pdns/recursordist/pubsuffixloader.cc new file mode 100644 index 0000000000..1bf1f54183 --- /dev/null +++ b/pdns/recursordist/pubsuffixloader.cc @@ -0,0 +1,84 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include + +#include "dnsname.hh" +#include "logger.hh" +#include "misc.hh" +#include "pubsuffix.hh" + +extern const char* g_pubsuffix[]; +std::vector> g_pubs; + +void initPublicSuffixList(const std::string& file) +{ + std::vector> pbList; + + bool loaded = false; + if (!file.empty()) { + try { + std::ifstream suffixFile(file); + if (!suffixFile.is_open()) { + throw std::runtime_error("Error opening the public suffix list file"); + } + + std::string line; + while (std::getline(suffixFile, line)) { + if (line.empty() || (line.rfind("//", 0) == 0)) { + /* skip empty and commented lines */ + continue; + } + try { + DNSName name(toLower(line)); + pbList.push_back(name.labelReverse().getRawLabels()); + } + catch(...) { + /* not a DNS name, ignoring */ + } + } + + g_log< parts; + stringtok(parts, low, "."); + reverse(parts.begin(), parts.end()); + pbList.push_back(parts); + } + } + + sort(pbList.begin(), pbList.end()); + g_pubs = std::move(pbList); +}