From: Jeremy Clerc Date: Thu, 21 Nov 2019 23:57:48 +0000 (+0100) Subject: geoipbackend: accept custom lookup mapping X-Git-Tag: dnsdist-1.6.0-alpha0~9^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F8608%2Fhead;p=thirdparty%2Fpdns.git geoipbackend: accept custom lookup mapping If for example you want a per country granularity, but still want to group by custom regions: uk, fr, be -> eu-central, pt, es -> eu-south; you need to to use %cc and create identical country records. It means you zones file can become huge and powerdns does not like it too much. For a user to have a custom mapping without the need to rewrite the GeoIP database, with this commit, he/she can specify a custom mapping, so if we get from the GeoIP database fr, we will look in the custom mapping and find eu-central. So we only need to create the eu-central record and it will be used by for uk, fr and be as per our first example. Definition of mapping_lookup_formats or custom_mapping at the domain level has priority, but if not defined the global config will be used as default. The custom lookup formats and mapping are specified in the zones file: --- mapping_lookup_formats: ['%cc-%re', '%cc'] custom_mapping: 'fr': 'eu-central' 'be': 'eu-central' 'es': 'eu-south' 'pt': 'eu-south' 'us-ca': 'us-west' 'us-tx': 'us-south' domains: - domain: example.com services: www.example.com: [ '%mp.www.example.com' ] records: eu-central.www.example.com: - A: 1.1.1.1 eu-south.www.example.com: - A: 1.1.1.2 us-west.www.example.com: - A: 1.1.1.3 us-south.www.example.com: - A: 1.1.1.4 - domain: example2.com mapping_lookup_formats: ['%cc'] --- diff --git a/docs/backends/geoip.rst b/docs/backends/geoip.rst index 402809a89b..7186a25bfe 100644 --- a/docs/backends/geoip.rst +++ b/docs/backends/geoip.rst @@ -145,6 +145,14 @@ that the ``‐`` before certain keys is part of the syntax. service.geo.example.com: default: [ '%co.%cn.service.geo.example.com', '%cn.service.geo.example.com' ] 10.0.0.0/8: 'internal.service.geo.example.com' + mapping_lookup_formats: ['%cc-%re', '%cc'] + custom_mapping: + fr: eu-central + be: eu-central + es: eu-south + pt: eu-south + us-tx: us-south + us-ca: us-south Keys explained ~~~~~~~~~~~~~~ @@ -165,6 +173,16 @@ Keys explained :services: Defines one or more services for querying. Each service name may have one or more placeholders. + :mapping_lookup_formats: Defines which format to interpolate when using the ``%mp`` placeholder. Each entry + is looked up in the given order and stops at first match. + This allows using a fine granularity, (e.g. per country), while limiting the number + of records to create. + You can use any placeholder, except ``%mp`` to avoid recursion, within the given + format (e.g. %cc). + :custom_mapping: Defines the mapping between the lookup format and a custom value to replace ``%mp`` placeholder. + +:mapping_lookup_formats: Same as per domain, but used as default value if not defined at the domain level. +:custom_mapping: Same as per domain, but used as default value if not defined at the domain level. .. note:: @@ -204,6 +222,16 @@ These placeholders disable caching for the record completely: :%ip4: Client IPv4 address :%ip6: Client IPv6 address +Following placeholder allows custom mapping: + +:%mp: Use formats in ``mapping_lookup_formats`` and use user defined ``custom_mapping`` + +.. versionadded:: 4.4.0 + + These placeholders have been added in version 4.4.0: + + - %mp to expand user defined custom formats. + .. versionadded:: 4.2.0 These placeholders have been added in version 4.2.0: diff --git a/modules/geoipbackend/geoipbackend.cc b/modules/geoipbackend/geoipbackend.cc index 83a86bc06b..119069b310 100644 --- a/modules/geoipbackend/geoipbackend.cc +++ b/modules/geoipbackend/geoipbackend.cc @@ -51,6 +51,8 @@ struct GeoIPDomain { int ttl; map services; map > records; + vector mapping_lookup_formats; + map custom_mapping; }; static vector s_domains; @@ -91,6 +93,25 @@ static vector > s_geoip_files; string getGeoForLua(const std::string& ip, int qaint); static string queryGeoIP(const Netmask& addr, GeoIPInterface::GeoIPQueryAttribute attribute, GeoIPNetmask& gl); +// validateMappingLookupFormats validates any custom format provided by the +// user does not use the custom mapping placeholder again, else it would do an +// infinite recursion. +bool validateMappingLookupFormats(const vector& formats) { + string::size_type cur,last; + for (const auto& lookupFormat : formats) { + last=0; + while((cur = lookupFormat.find("%", last)) != string::npos) { + if (!lookupFormat.compare(cur,3,"%mp")) { + return false; + } else if (!lookupFormat.compare(cur,2,"%%")) { // Ensure escaped % is also accepted + last = cur + 2; continue; + } + last = cur + 1; // move to next attribute + } + } + return true; +} + void GeoIPBackend::initialize() { YAML::Node config; vector tmp_domains; @@ -116,6 +137,19 @@ void GeoIPBackend::initialize() { } } + // Global lookup formats and mapping will be used + // if none defined at the domain level. + vector global_mapping_lookup_formats; + map global_custom_mapping; + if (YAML::Node formats = config["mapping_lookup_formats"]) { + global_mapping_lookup_formats = formats.as>(); + if (!validateMappingLookupFormats(global_mapping_lookup_formats)) + throw PDNSException(string("%mp is not allowed in mapping lookup")); + } + if (YAML::Node mapping = config["custom_mapping"]) { + global_custom_mapping = mapping.as>(); + } + for(YAML::Node domain : config["domains"]) { GeoIPDomain dom; dom.id = tmp_domains.size(); @@ -210,6 +244,23 @@ void GeoIPBackend::initialize() { nmt.insert(Netmask("::/0")).second.swap(value); } + // Allow per domain override of mapping_lookup_formats and custom_mapping. + // If not defined, the global values will be used. + if (YAML::Node formats = domain["mapping_lookup_formats"]) { + vector mapping_lookup_formats = formats.as>(); + if (!validateMappingLookupFormats(mapping_lookup_formats)) + throw PDNSException(string("%mp is not allowed in mapping lookup formats of domain ") + dom.domain.toLogString()); + + dom.mapping_lookup_formats = mapping_lookup_formats; + } else { + dom.mapping_lookup_formats = global_mapping_lookup_formats; + } + if (YAML::Node mapping = domain["custom_mapping"]) { + dom.custom_mapping = mapping.as>(); + } else { + dom.custom_mapping = global_custom_mapping; + } + dom.services[srvName].netmask4 = netmask4; dom.services[srvName].netmask6 = netmask6; dom.services[srvName].masks.swap(nmt); @@ -329,7 +380,7 @@ bool GeoIPBackend::lookup_static(const GeoIPDomain &dom, const DNSName &search, if (rr.weight == 0 || probability_rnd < comp || probability_rnd > (comp + rr.weight)) continue; } - const string& content = format2str(rr.content, addr, gl); + const string& content = format2str(rr.content, addr, gl, dom); if (rr.qtype != QType::ENT && rr.qtype != QType::TXT && content.empty()) continue; d_result.push_back(rr); d_result.back().content = content; @@ -409,7 +460,7 @@ void GeoIPBackend::lookup(const QType &qtype, const DNSName& qdomain, int zoneId // note that this means the array format won't work with indirect for(auto it = node->second.begin(); it != node->second.end(); it++) { - sformat = DNSName(format2str(*it, addr, gl)); + sformat = DNSName(format2str(*it, addr, gl, *dom)); // see if the record can be found if (this->lookup_static((*dom), sformat, qtype, qdomain, addr, gl)) @@ -539,7 +590,7 @@ static bool queryGeoLocation(const Netmask& addr, GeoIPNetmask& gl, double& lat, return false; } -string GeoIPBackend::format2str(string sformat, const Netmask& addr, GeoIPNetmask& gl) { +string GeoIPBackend::format2str(string sformat, const Netmask& addr, GeoIPNetmask& gl, const GeoIPDomain &dom) { string::size_type cur,last; boost::optional alt, prec; double lat, lon; @@ -553,7 +604,16 @@ string GeoIPBackend::format2str(string sformat, const Netmask& addr, GeoIPNetmas string rep; int nrep=3; tmp_gl.netmask = 0; - if (!sformat.compare(cur,3,"%cn")) { + if (!sformat.compare(cur,3,"%mp")) { + rep = "unknown"; + for (const auto& lookupFormat : dom.mapping_lookup_formats) { + auto it = dom.custom_mapping.find(format2str(lookupFormat, addr, gl, dom)); + if (it != dom.custom_mapping.end()) { + rep = it->second; + break; + } + } + } else if (!sformat.compare(cur,3,"%cn")) { rep = queryGeoIP(addr, GeoIPInterface::Continent, tmp_gl); } else if (!sformat.compare(cur,3,"%co")) { rep = queryGeoIP(addr, GeoIPInterface::Country, tmp_gl); diff --git a/modules/geoipbackend/geoipbackend.hh b/modules/geoipbackend/geoipbackend.hh index a45bbda4e5..ec0126f4fe 100644 --- a/modules/geoipbackend/geoipbackend.hh +++ b/modules/geoipbackend/geoipbackend.hh @@ -70,7 +70,7 @@ private: static ReadWriteLock s_state_lock; void initialize(); - string format2str(string format, const Netmask &addr, GeoIPNetmask& gl); + string format2str(string format, const Netmask &addr, GeoIPNetmask& gl, const GeoIPDomain &dom); bool d_dnssec; bool hasDNSSECkey(const DNSName& name); bool lookup_static(const GeoIPDomain &dom, const DNSName &search, const QType &qtype, const DNSName& qdomain, const Netmask &addr, GeoIPNetmask& gl); diff --git a/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/command b/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/command new file mode 100755 index 0000000000..056a78de23 --- /dev/null +++ b/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/command @@ -0,0 +1,3 @@ +#!/bin/sh +cleandig map.geo.example.com TXT ednssubnet $geoipregionip +cleandig map.geo2.example.com TXT ednssubnet $geoipregionip \ No newline at end of file diff --git a/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/description b/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/description new file mode 100644 index 0000000000..c770423eff --- /dev/null +++ b/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/description @@ -0,0 +1,2 @@ +This test tries to resolve a straight TXT record that is available via +custom mapping the database. diff --git a/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/expected_result b/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/expected_result new file mode 100644 index 0000000000..2930214ef2 --- /dev/null +++ b/modules/geoipbackend/regression-tests/custom-mapping-txt-resolution/expected_result @@ -0,0 +1,8 @@ +0 map.geo.example.com. IN TXT 30 "custom mapping" +2 . IN OPT 0 AAgACAABIBgBAQEB +Rcode: 0 (No Error), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 +Reply to question for qname='map.geo.example.com.', qtype=TXT +0 map.geo2.example.com. IN TXT 30 "overridden moon mapping" +2 . IN OPT 0 AAgACAABIBgBAQEB +Rcode: 0 (No Error), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 +Reply to question for qname='map.geo2.example.com.', qtype=TXT diff --git a/regression-tests/backends/geoip-master b/regression-tests/backends/geoip-master index 60ec0ed889..876edaeb38 100644 --- a/regression-tests/backends/geoip-master +++ b/regression-tests/backends/geoip-master @@ -51,11 +51,31 @@ domains: - a: $geoipregionip unknown.service.geo.example.com: - a: 127.0.0.1 + earth.map.geo.example.com: + - txt: "custom mapping" services: geo.example.com: '%cn.service.geo.example.com' www.geo.example.com: '%cn.service.geo.example.com' indirect.geo.example.com: '%cn.elsewhere.example.com' city.geo.example.com: '%ci.%re.%cc.city.geo.example.com' + map.geo.example.com: '%mp.map.geo.example.com' +- domain: geo2.example.com + ttl: 30 + records: + geo2.example.com: + - soa: ns1.example.com hostmaster.example.com 2014090125 7200 3600 1209600 3600 + - ns: ns1.example.com + - ns: ns2.example.com + - mx: 10 mx.example.com + moon.map.geo2.example.com: + - txt: "overridden moon mapping" + services: + map.geo2.example.com: '%mp.map.geo2.example.com' + custom_mapping: + $geoipregion: moon +mapping_lookup_formats: ['%cn'] +custom_mapping: + $geoipregion: earth EOF cat > $testsdir/region-a-resolution/expected_result <