From: Otto Date: Wed, 12 May 2021 13:56:57 +0000 (+0200) Subject: Add a periodic zones-to-cache function. X-Git-Tag: auth-4.6.0-alpha1~5^2~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=841128ba26d86e4e85273a243486943f449285f2;p=thirdparty%2Fpdns.git Add a periodic zones-to-cache function. No signatures are validated, that will happen on-demand if the records are used. --- diff --git a/pdns/minicurl.cc b/pdns/minicurl.cc index b38413d47a..91b561cd01 100644 --- a/pdns/minicurl.cc +++ b/pdns/minicurl.cc @@ -74,7 +74,7 @@ static string extractHostFromURL(const std::string& url) return url.substr(pos, endpos-pos); } -void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen) +void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen, bool verify) { if(rem) { struct curl_slist *hostlist = nullptr; // THIS SHOULD BE FREED @@ -105,8 +105,8 @@ void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const C curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, true); /* only allow HTTP and HTTPS */ curl_easy_setopt(d_curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); - curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, false); - curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, false); + curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, verify); + curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, verify ? 2 : 0); curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, true); curl_easy_setopt(d_curl, CURLOPT_URL, str.c_str()); curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -121,9 +121,9 @@ void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const C d_data.clear(); } -std::string MiniCurl::getURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout) +std::string MiniCurl::getURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen, bool verify) { - setupURL(str, rem, src, timeout); + setupURL(str, rem, src, timeout, fastopen, verify); auto res = curl_easy_perform(d_curl); long http_code = 0; curl_easy_getinfo(d_curl, CURLINFO_RESPONSE_CODE, &http_code); @@ -136,9 +136,9 @@ std::string MiniCurl::getURL(const std::string& str, const ComboAddress* rem, co return ret; } -std::string MiniCurl::postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout, bool fastopen) +std::string MiniCurl::postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout, bool fastopen, bool verify) { - setupURL(str, nullptr, nullptr, timeout, fastopen); + setupURL(str, nullptr, nullptr, timeout, fastopen, verify); setHeaders(headers); curl_easy_setopt(d_curl, CURLOPT_POSTFIELDSIZE, postdata.size()); curl_easy_setopt(d_curl, CURLOPT_POSTFIELDS, postdata.c_str()); diff --git a/pdns/minicurl.hh b/pdns/minicurl.hh index c2550d6bd2..b59656ae71 100644 --- a/pdns/minicurl.hh +++ b/pdns/minicurl.hh @@ -38,14 +38,14 @@ public: MiniCurl(const string& useragent="MiniCurl/0.0"); ~MiniCurl(); MiniCurl& operator=(const MiniCurl&) = delete; - std::string getURL(const std::string& str, const ComboAddress* rem=nullptr, const ComboAddress* src=nullptr, int timeout = 2); - std::string postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout = 2, bool fastopen = false); + std::string getURL(const std::string& str, const ComboAddress* rem=nullptr, const ComboAddress* src=nullptr, int timeout = 2, bool fastopen = false, bool verify = false); + std::string postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout = 2, bool fastopen = false, bool verify = false); private: CURL *d_curl; static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata); std::string d_data; struct curl_slist* d_header_list = nullptr; - void setupURL(const std::string& str, const ComboAddress* rem=nullptr, const ComboAddress* src=nullptr, int timeout = 2, bool fastopen = false); + void setupURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen, bool verify); void setHeaders(const MiniCurlHeaders& headers); void clearHeaders(); }; diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 0011b55180..ceb32001a4 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -115,6 +115,7 @@ #include "rec-protozero.hh" #include "xpf.hh" +#include "rec-zonetocache.hh" typedef map tcpClientCounts_t; @@ -135,6 +136,7 @@ static thread_local uint64_t t_frameStreamServersGeneration; thread_local std::unique_ptr MT; // the big MTasker std::unique_ptr g_recCache; std::unique_ptr g_negCache; +std::map g_zones_to_cache; thread_local std::unique_ptr t_packetCache; thread_local FDMultiplexer* t_fdm{nullptr}; @@ -3701,6 +3703,8 @@ static void houseKeeping(void *) static thread_local int cleanCounter=0; static thread_local bool s_running; // houseKeeping can get suspended in secpoll, and be restarted, which makes us do duplicate work static time_t last_RC_prune = 0; + static time_t last_zone_to_cache = 0; + static time_t zones_to_cache_interval = 0; auto luaconfsLocal = g_luaconfs.getLocal(); if (last_trustAnchorUpdate == 0 && !luaconfsLocal->trustAnchorFileInfo.fname.empty() && luaconfsLocal->trustAnchorFileInfo.interval != 0) { @@ -3772,6 +3776,11 @@ static void houseKeeping(void *) g_log< zones_to_cache_interval * 9 / 10) { + last_zone_to_cache = now.tv_sec; + zones_to_cache_interval = RecZoneToCache::ZonesToCache(g_zones_to_cache); + } } if(now.tv_sec - last_secpoll >= 3600) { @@ -5018,6 +5027,19 @@ static int serviceMain(int argc, char*argv[]) exit(1); } } + { + std::vector parts; + stringtok(parts, ::arg()["zones-to-cache"], " ,\t\n\r"); + for (const auto& p : parts) { + if (p.find('=') == string::npos) { + throw PDNSException("Error parsing '" + p + "', missing ="); + } + auto [zone, url] = splitField(p, '='); + boost::trim(zone); + boost::trim(url); + g_zones_to_cache[zone] = url; + } + } g_networkTimeoutMsec = ::arg().asNum("network-timeout"); @@ -5941,6 +5963,7 @@ int main(int argc, char **argv) ::arg().setSwitch("dot-to-port-853", "Force DoT connection to target port 853 if DoT compiled in")="yes"; ::arg().set("dot-to-auth-names", "Use DoT to authoritative servers with these names or suffixes")=""; + ::arg().set("zones-to-cache", "Zones to prefill the cache with, comma separated domain=url pairs")=""; ::arg().set("tcp-out-max-idle-ms", "Time TCP/DoT connections are left idle in milliseconds or 0 if no limit") = "10000"; ::arg().set("tcp-out-max-idle-per-auth", "Maximum number of idle TCP/DoT connections to a specific IP per thread, 0 means do not keep idle connections open") = "10"; diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am index bcdb7454a4..d004b4154b 100644 --- a/pdns/recursordist/Makefile.am +++ b/pdns/recursordist/Makefile.am @@ -164,6 +164,7 @@ pdns_recursor_SOURCES = \ rec-snmp.hh rec-snmp.cc \ rec-taskqueue.cc rec-taskqueue.hh \ rec-tcpout.cc rec-tcpout.hh \ + rec-zonetocache.cc rec-zonetocache.hh \ rec_channel.cc rec_channel.hh rec_metrics.hh \ rec_channel_rec.cc \ recpacketcache.cc recpacketcache.hh \ @@ -447,6 +448,13 @@ AM_CPPFLAGS += $(GNUTLS_CFLAGS) pdns_recursor_LDADD += $(GNUTLS_LIBS) endif endif +if HAVE_LIBCURL +pdns_recursor_SOURCES += minicurl.cc minicurl.hh +pdns_recursor_LDADD += $(LIBCURL) +testrunner_SOURCES += minicurl.cc minicurl.hh +testrunner_LDADD += $(LIBCURL) +endif + rec_control_SOURCES = \ arguments.cc arguments.hh \ diff --git a/pdns/recursordist/configure.ac b/pdns/recursordist/configure.ac index 2727cbb517..d54177cdee 100644 --- a/pdns/recursordist/configure.ac +++ b/pdns/recursordist/configure.ac @@ -85,6 +85,7 @@ PDNS_CHECK_LIBCRYPTO_EDDSA PDNS_WITH_LIBSODIUM PDNS_WITH_LIBDECAF PDNS_WITH_LIBCAP +PDNS_CHECK_LIBCURL PDNS_WITH_NET_SNMP @@ -254,5 +255,9 @@ AS_IF([test "x$enable_dns_over_tls" != "xno"], [ [AC_MSG_NOTICE([OpenSSL: no])] )] ) +AS_IF([test "x$HAVE_LIBCURL" != "x"], + [AC_MSG_NOTICE([libcurl: yes])], + [AC_MSG_NOTICE([libcurl: no])] +) AC_MSG_NOTICE([Context library: $pdns_context_library]) AC_MSG_NOTICE([]) diff --git a/pdns/recursordist/m4/libcurl.m4 b/pdns/recursordist/m4/libcurl.m4 new file mode 120000 index 0000000000..91d8843a09 --- /dev/null +++ b/pdns/recursordist/m4/libcurl.m4 @@ -0,0 +1 @@ +../../../m4/libcurl.m4 \ No newline at end of file diff --git a/pdns/recursordist/m4/pdns_check_libcurl.m4 b/pdns/recursordist/m4/pdns_check_libcurl.m4 new file mode 120000 index 0000000000..0e21701cec --- /dev/null +++ b/pdns/recursordist/m4/pdns_check_libcurl.m4 @@ -0,0 +1 @@ +../../../m4/pdns_check_libcurl.m4 \ No newline at end of file diff --git a/pdns/recursordist/minicurl.cc b/pdns/recursordist/minicurl.cc new file mode 120000 index 0000000000..3f48b8c54e --- /dev/null +++ b/pdns/recursordist/minicurl.cc @@ -0,0 +1 @@ +../minicurl.cc \ No newline at end of file diff --git a/pdns/recursordist/minicurl.hh b/pdns/recursordist/minicurl.hh new file mode 120000 index 0000000000..bba06c379e --- /dev/null +++ b/pdns/recursordist/minicurl.hh @@ -0,0 +1 @@ +../minicurl.hh \ No newline at end of file diff --git a/pdns/recursordist/rec-zonetocache.cc b/pdns/recursordist/rec-zonetocache.cc new file mode 100644 index 0000000000..1835e89cbf --- /dev/null +++ b/pdns/recursordist/rec-zonetocache.cc @@ -0,0 +1,260 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "rec-zonetocache.hh" + +#include "syncres.hh" +#include "minicurl.hh" +#include "zoneparser-tng.hh" +#include "query-local-address.hh" +#include "axfr-retriever.hh" +#include "validate-recursor.hh" + +time_t RecZoneToCache::ZonesToCache(const std::map& map) +{ + // By default and max once per 24 hours + time_t refresh = 24 * 3600; + + struct timeval now; + gettimeofday(&now, nullptr); + SyncRes sr(now); + bool dnssec = g_dnssecmode != DNSSECMode::Off; + sr.setDoDNSSEC(dnssec); + sr.setDNSSECValidationRequested(g_dnssecmode != DNSSECMode::Off && g_dnssecmode != DNSSECMode::ProcessNoValidate); + + for (const auto& [zone, url] : map) { + const string msg = "zones-to-cache error while loading " + zone + " from " + url + ": "; + try { + refresh = min(refresh, ZoneToCache(zone, url, dnssec)); + } + catch (const PDNSException& e) { + g_log << Logger::Error << msg << e.reason << endl; + } + catch (const std::runtime_error& e) { + g_log << Logger::Error << msg << e.what() << endl; + } + catch (...) { + g_log << Logger::Error << msg << "unexpected exception" << endl; + } + + // We do not want to refresh more than once per hour + refresh = max(refresh, 3600LL); + } + + return refresh; +} + +struct ZoneData +{ + std::map, vector> d_all; + std::map, vector>> d_sigs; + std::set d_delegations; + time_t d_refresh; + time_t d_now; + DNSName d_zone; + + bool isRRSetAuth(const DNSName& qname, QType qtype); + void parseDRForCache(DNSRecord& dr); + void getByAXFR(const std::string& url); +}; + +bool ZoneData::isRRSetAuth(const DNSName& qname, QType qtype) +{ + DNSName delegatedZone(qname); + if (qtype == QType::DS) { + delegatedZone.chopOff(); + } + bool isDelegated = false; + for (;;) { + if (d_delegations.count(delegatedZone) > 0) { + isDelegated = true; + break; + } + delegatedZone.chopOff(); + if (delegatedZone == g_rootdnsname || delegatedZone == d_zone) + break; + } + return !isDelegated; +} + +void ZoneData::parseDRForCache(DNSRecord& dr) +{ + const auto key = make_pair(dr.d_name, dr.d_type); + + d_refresh = min(d_refresh, static_cast(dr.d_ttl)); + dr.d_ttl += d_now; + + switch (dr.d_type) { + case QType::NSEC: + case QType::NSEC3: + break; + case QType::RRSIG: { + const auto& rr = getRR(dr); + const auto sigkey = make_pair(key.first, rr->d_type); + auto found = d_sigs.find(sigkey); + if (found != d_sigs.end()) { + found->second.push_back(rr); + } + else { + vector> sigsrr; + sigsrr.push_back(rr); + d_sigs.insert({sigkey, sigsrr}); + } + break; + } + case QType::NS: + if (dr.d_name != d_zone) { + d_delegations.insert(dr.d_name); + } + break; + default: + break; + } + + auto found = d_all.find(key); + if (found != d_all.end()) { + found->second.push_back(dr); + } + else { + vector v; + v.push_back(dr); + d_all.insert({key, v}); + } +} + +void ZoneData::getByAXFR(const std::string& url) +{ + ComboAddress primary = ComboAddress(url.substr(7), 53); + uint16_t axfrTimeout = 20; + size_t maxReceivedBytes = 0; + const TSIGTriplet tt; + ComboAddress local; //(localAddress); + if (local == ComboAddress()) { + local = pdns::getQueryLocalAddress(primary.sin4.sin_family, 0); + } + AXFRRetriever axfr(primary, d_zone, tt, &local, maxReceivedBytes, axfrTimeout); + Resolver::res_t nop; + vector chunk; + time_t axfrStart = time(nullptr); + time_t axfrNow = time(nullptr); + shared_ptr sr; + while (axfr.getChunk(nop, &chunk, (axfrStart + axfrTimeout - axfrNow))) { + for (auto& dr : chunk) { + parseDRForCache(dr); + } + axfrNow = time(nullptr); + if (axfrNow < axfrStart || axfrNow - axfrStart > axfrTimeout) { + throw PDNSException("Total AXFR time for zoneToCache exceeded!"); + } + } +} + +static std::vector getLinesFromFile(const std::string& url) +{ + std::vector lines; + std::ifstream stream(url); + std::string line; + while (std::getline(stream, line)) { + lines.push_back(line); + } + return lines; +} + +static std::vector getURL(const std::string& url) +{ + std::vector lines; +#ifdef HAVE_LIBCURL + MiniCurl mc; + std::string reply = mc.getURL(url, nullptr, nullptr, 10, false, true); + std::istringstream stream(reply); + string line; + while (std::getline(stream, line)) { + lines.push_back(line); + } +#endif + return lines; +} + +time_t RecZoneToCache::ZoneToCache(const string& name, const string& url, bool dnssec) +{ + ZoneData data; + data.d_refresh = std::numeric_limits::max(); + data.d_zone = DNSName(name); + data.d_now = time(nullptr); + + // We do not do validation, it will happen on-demand if an Indeterminate record is encountered when the caches are queried + // First scan all records collecting info about delegations ans sigs + // A this moment, we ignore NSEC and NSEC3 records. It is not clear to me yet under which conditions + // they could be entered in into the (neg)cache. + + if (url.substr(0, 7) == "axfr://") { + data.getByAXFR(url); + } + else { + vector lines; + if (url.substr(0, 8) != "https://" && url.substr(0, 7) != "http://") { + lines = getLinesFromFile(url); + } + else { + lines = getURL(url); + } + DNSResourceRecord drr; + ZoneParserTNG zpt(lines, data.d_zone); + zpt.setMaxGenerateSteps(0); + + while (zpt.get(drr)) { + DNSRecord dr(drr); + data.parseDRForCache(dr); + } + } + + // Rerun, now inserting the rrsets into the cache with associated sigs + data.d_now = time(nullptr); + for (const auto& [key, v] : data.d_all) { + const auto& [qname, qtype] = key; + switch (qtype) { + case QType::NSEC: + case QType::NSEC3: + break; + case QType::RRSIG: + break; + default: { + vector> sigsrr; + auto it = data.d_sigs.find(key); + if (it != data.d_sigs.end()) { + for (const auto& sig : it->second) { + sigsrr.push_back(sig); + } + } + bool auth = data.isRRSetAuth(qname, qtype); + // Same decision as updateCacheFromRecords() (we do not test for NSEC since we skip those completely) + if (auth || (qtype == QType::NS || qtype == QType::A || qtype == QType::AAAA || qtype == QType::DS)) { + g_recCache->replace(data.d_now, qname, qtype, v, sigsrr, + std::vector>(), auth, data.d_zone); + } + break; + } + } + } + + return data.d_refresh; +} diff --git a/pdns/recursordist/rec-zonetocache.hh b/pdns/recursordist/rec-zonetocache.hh new file mode 100644 index 0000000000..7f777dcf9f --- /dev/null +++ b/pdns/recursordist/rec-zonetocache.hh @@ -0,0 +1,34 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "namespaces.hh" + +class RecZoneToCache +{ +public: + static time_t ZonesToCache(const std::map& map); + +private: + static time_t ZoneToCache(const string& name, const string& url, bool dnssec); +};