[thirdparty/pdns.git] / pdns / rpzloader.cc

#include "syncres.hh"
#include "dnsparser.hh"
#include "dnsrecords.hh"
#include "ixfr.hh"
#include "resolver.hh"
#include "logger.hh"
#include "rec-lua-conf.hh"
#include "rpzloader.hh"
#include "zoneparser-tng.hh"

static Netmask makeNetmaskFromRPZ(const DNSName& name)
{
  auto parts = name.getRawLabels();
  /*
   * why 2?, the minimally valid IPv6 address that can be encoded in an RPZ is
   * $NETMASK.zz (::/$NETMASK)
   * Terrible right?
   */
  if(parts.size() < 2 || parts.size() > 9)
    throw PDNSException("Invalid IP address in RPZ: "+name.toString());

  bool isV6 = (stoi(parts[0]) > 32);
  bool hadZZ = false;

  for (auto &part : parts) {
    // Check if we have an IPv4 octet
    for (auto c : part)
      if (!isdigit(c))
        isV6 = true;

    if (pdns_iequals(part,"zz")) {
      if (hadZZ)
        throw PDNSException("more than one 'zz' label found in RPZ name"+name.toString());
      part = "";
      isV6 = true;
      hadZZ = true;
    }
  }

  if (isV6 && parts.size() < 9 && !hadZZ)
    throw PDNSException("No 'zz' label found in an IPv6 RPZ name shorter than 9 elements: "+name.toString());

  if (parts.size() == 5 && !isV6)
    return Netmask(parts[4]+"."+parts[3]+"."+parts[2]+"."+parts[1]+"/"+parts[0]);

  string v6;

  for (uint8_t i = parts.size()-1 ; i > 0; i--) {
    v6 += parts[i];
    if (parts[i] == "" && i == 1 && i == parts.size()-1)
        v6+= "::";
    if (parts[i] == "" && i != parts.size()-1)
        v6+= ":";
    if (parts[i] != "" && i != 1)
      v6 += ":";
  }
  v6 += "/" + parts[0];

  return Netmask(v6);
}

void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr<DNSFilterEngine::Zone> zone, bool addOrRemove, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL)
{
  static const DNSName drop("rpz-drop."), truncate("rpz-tcp-only."), noaction("rpz-passthru.");
  static const DNSName rpzClientIP("rpz-client-ip"), rpzIP("rpz-ip"),
    rpzNSDname("rpz-nsdname"), rpzNSIP("rpz-nsip.");
  static const std::string rpzPrefix("rpz-");

  DNSFilterEngine::Policy pol;

  if(dr.d_class != QClass::IN) {
    return;
  }

  if(dr.d_type == QType::CNAME) {
    auto crc = getRR<CNAMERecordContent>(dr);
    if (!crc) {
      return;
    }
    auto crcTarget=crc->getTarget();
    if(defpol) {
      pol=*defpol;
    }
    else if(crcTarget.isRoot()) {
      // cerr<<"Wants NXDOMAIN for "<<dr.d_name<<": ";
      pol.d_kind = DNSFilterEngine::PolicyKind::NXDOMAIN;
    } else if(crcTarget==g_wildcarddnsname) {
      // cerr<<"Wants NODATA for "<<dr.d_name<<": ";
      pol.d_kind = DNSFilterEngine::PolicyKind::NODATA;
    }
    else if(crcTarget==drop) {
      // cerr<<"Wants DROP for "<<dr.d_name<<": ";
      pol.d_kind = DNSFilterEngine::PolicyKind::Drop;
    }
    else if(crcTarget==truncate) {
      // cerr<<"Wants TRUNCATE for "<<dr.d_name<<": ";
      pol.d_kind = DNSFilterEngine::PolicyKind::Truncate;
    }
    else if(crcTarget==noaction) {
      // cerr<<"Wants NOACTION for "<<dr.d_name<<": ";
      pol.d_kind = DNSFilterEngine::PolicyKind::NoAction;
    }
    /* "The special RPZ encodings which are not to be taken as Local Data are
       CNAMEs with targets that are:
       +  "."  (NXDOMAIN action),
       +  "*." (NODATA action),
       +  a top level domain starting with "rpz-",
       +  a child of a top level domain starting with "rpz-".
    */
    else if(!crcTarget.empty() && !crcTarget.isRoot() && crcTarget.getRawLabel(crcTarget.countLabels() - 1).compare(0, rpzPrefix.length(), rpzPrefix) == 0) {
      /* this is very likely an higher format number or a configuration error,
         let's just ignore it. */
      L<<Logger::Info<<"Discarding unsupported RPZ entry "<<crcTarget.toString()<<" for "<<dr.d_name<<endl;
      return;
    }
    else {
      pol.d_kind = DNSFilterEngine::PolicyKind::Custom;
      pol.d_custom = dr.d_content;
      // cerr<<"Wants custom "<<crcTarget<<" for "<<dr.d_name<<": ";
    }
  }
  else {
    if (defpol) {
      pol=*defpol;
    }
    else {
      pol.d_kind = DNSFilterEngine::PolicyKind::Custom;
      pol.d_custom = dr.d_content;
      // cerr<<"Wants custom "<<dr.d_content->getZoneRepresentation()<<" for "<<dr.d_name<<": ";
    }
  }

  if (!defpol || defpol->d_ttl < 0) {
    pol.d_ttl = static_cast<int32_t>(std::min(maxTTL, dr.d_ttl));
  } else {
    pol.d_ttl = static_cast<int32_t>(std::min(maxTTL, static_cast<uint32_t>(pol.d_ttl)));
  }

  // now to DO something with that

  if(dr.d_name.isPartOf(rpzNSDname)) {
    DNSName filt=dr.d_name.makeRelative(rpzNSDname);
    if(addOrRemove)
      zone->addNSTrigger(filt, pol);
    else
      zone->rmNSTrigger(filt, pol);
  } else if(dr.d_name.isPartOf(rpzClientIP)) {
    DNSName filt=dr.d_name.makeRelative(rpzClientIP);
    auto nm=makeNetmaskFromRPZ(filt);
    if(addOrRemove)
      zone->addClientTrigger(nm, pol);
    else
      zone->rmClientTrigger(nm, pol);
    
  } else if(dr.d_name.isPartOf(rpzIP)) {
    // cerr<<"Should apply answer content IP policy: "<<dr.d_name<<endl;
    DNSName filt=dr.d_name.makeRelative(rpzIP);
    auto nm=makeNetmaskFromRPZ(filt);
    if(addOrRemove)
      zone->addResponseTrigger(nm, pol);
    else
      zone->rmResponseTrigger(nm, pol);
  } else if(dr.d_name.isPartOf(rpzNSIP)) {
    DNSName filt=dr.d_name.makeRelative(rpzNSIP);
    auto nm=makeNetmaskFromRPZ(filt);
    if(addOrRemove)
      zone->addNSIPTrigger(nm, pol);
    else
      zone->rmNSIPTrigger(nm, pol);
  } else {
    if(addOrRemove)
      zone->addQNameTrigger(dr.d_name, pol);
    else
      zone->rmQNameTrigger(dr.d_name, pol);
  }
}

shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout)
{
  L<<Logger::Warning<<"Loading RPZ zone '"<<zoneName<<"' from "<<master.toStringWithPort()<<endl;
  if(!tt.name.empty())
    L<<Logger::Warning<<"With TSIG key '"<<tt.name<<"' of algorithm '"<<tt.algo<<"'"<<endl;

  ComboAddress local(localAddress);
  if (local == ComboAddress())
    local = getQueryLocalAddress(master.sin4.sin_family, 0);

  AXFRRetriever axfr(master, zoneName, tt, &local, maxReceivedBytes, axfrTimeout);
  unsigned int nrecords=0;
  Resolver::res_t nop;
  vector<DNSRecord> chunk;
  time_t last=0;
  time_t axfrStart = time(nullptr);
  time_t axfrNow = time(nullptr);
  shared_ptr<SOARecordContent> sr;
  while(axfr.getChunk(nop, &chunk, (axfrStart + axfrTimeout - axfrNow))) {
    for(auto& dr : chunk) {
      if(dr.d_type==QType::NS || dr.d_type==QType::TSIG) {
	continue;
      }

      dr.d_name.makeUsRelative(zoneName);
      if(dr.d_type==QType::SOA) {
	sr = getRR<SOARecordContent>(dr);
	continue;
      }

      RPZRecordToPolicy(dr, zone, true, defpol, maxTTL);
      nrecords++;
    } 
    axfrNow = time(nullptr);
    if (axfrNow < axfrStart || axfrNow - axfrStart > axfrTimeout) {
      throw PDNSException("Total AXFR time exceeded!");
    }
    if(last != time(0)) {
      L<<Logger::Info<<"Loaded & indexed "<<nrecords<<" policy records so far"<<endl;
      last=time(0);
    }
  }
  L<<Logger::Info<<"Done: "<<nrecords<<" policy records active, SOA: "<<sr->getZoneRepresentation()<<endl;
  return sr;
}

// this function is silent - you do the logging
void loadRPZFromFile(const std::string& fname, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL)
{
  ZoneParserTNG zpt(fname);
  DNSResourceRecord drr;
  DNSName domain;
  while(zpt.get(drr)) {
    try {
      if(drr.qtype.getCode() == QType::CNAME && drr.content.empty())
	drr.content=".";
      DNSRecord dr(drr);
      if(dr.d_type == QType::SOA) {
        domain = dr.d_name;
        zone->setDomain(domain);
      }
      else if(dr.d_type == QType::NS) {
	continue;
      }
      else {
	dr.d_name=dr.d_name.makeRelative(domain);
	RPZRecordToPolicy(dr, zone, true, defpol, maxTTL);
      }
    }
    catch(const PDNSException& pe) {
      throw PDNSException("Issue parsing '"+drr.qname.toString()+"' '"+drr.content+"' at "+zpt.getLineOfFile()+": "+pe.reason);
    }
  }
}

static std::unordered_map<std::string, shared_ptr<rpzStats> > s_rpzStats;
static std::mutex s_rpzStatsMutex;

shared_ptr<rpzStats> getRPZZoneStats(const std::string& zone)
{
  std::lock_guard<std::mutex> l(s_rpzStatsMutex);
  if (s_rpzStats.find(zone) == s_rpzStats.end()) {
    s_rpzStats[zone] = std::make_shared<rpzStats>();
  }
  return s_rpzStats[zone];
}

static void incRPZFailedTransfers(const std::string& zone)
{
  auto stats = getRPZZoneStats(zone);
  if (stats != nullptr)
    stats->d_failedTransfers++;
}

static void setRPZZoneNewState(const std::string& zone, uint32_t serial, uint64_t numberOfRecords, bool wasAXFR)
{
  auto stats = getRPZZoneStats(zone);
  if (stats == nullptr)
    return;
  stats->d_successfulTransfers++;
  if (wasAXFR) {
    stats->d_fullTransfers++;
  }
  stats->d_lastUpdate = time(nullptr);
  stats->d_serial = serial;
  stats->d_numberOfRecords = numberOfRecords;
}

void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout)
{
  auto luaconfsLocal = g_luaconfs.getLocal();
  /* we can _never_ modify this zone directly, we need to do a full copy then replace the existing zone */
  std::shared_ptr<DNSFilterEngine::Zone> oldZone = luaconfsLocal->dfe.getZone(zoneIdx);
  if (!oldZone) {
    L<<Logger::Error<<"Unable to retrieve RPZ zone with index "<<zoneIdx<<" from the configuration, exiting"<<endl;
    return;
  }
  uint32_t refresh = oldZone->getRefresh();
  DNSName zoneName = oldZone->getDomain();
  std::string polName = oldZone->getName() ? *(oldZone->getName()) : zoneName.toString();
  shared_ptr<SOARecordContent> sr;

  while (!sr) {
    /* full copy, as promised */
    std::shared_ptr<DNSFilterEngine::Zone> newZone = std::make_shared<DNSFilterEngine::Zone>(*oldZone);

    try {
      sr=loadRPZFromServer(master, zoneName, newZone, defpol, maxTTL, tt, maxReceivedBytes, localAddress, axfrTimeout);
      if(refresh == 0) {
        refresh = sr->d_st.refresh;
      }
      newZone->setSerial(sr->d_st.serial);
      setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), true);

      g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
        lci.dfe.setZone(zoneIdx, newZone);
      });
    }
    catch(const std::exception& e) {
      theL()<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.what()<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
      incRPZFailedTransfers(polName);
    }
    catch(const PDNSException& e) {
      theL()<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.reason<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
      incRPZFailedTransfers(polName);
    }

    if (!sr) {
      if (refresh == 0) {
        sleep(10);
      } else {
        sleep(refresh);
      }
    }
  }

  for(;;) {
    DNSRecord dr;
    dr.d_content=sr;

    sleep(refresh);

    L<<Logger::Info<<"Getting IXFR deltas for "<<zoneName<<" from "<<master.toStringWithPort()<<", our serial: "<<getRR<SOARecordContent>(dr)->d_st.serial<<endl;
    vector<pair<vector<DNSRecord>, vector<DNSRecord> > > deltas;

    ComboAddress local(localAddress);
    if (local == ComboAddress())
      local = getQueryLocalAddress(master.sin4.sin_family, 0);

    try {
      deltas = getIXFRDeltas(master, zoneName, dr, tt, &local, maxReceivedBytes);
    } catch(std::runtime_error& e ){
      L<<Logger::Warning<<e.what()<<endl;
      incRPZFailedTransfers(polName);
      continue;
    }
    if(deltas.empty())
      continue;
    L<<Logger::Info<<"Processing "<<deltas.size()<<" delta"<<addS(deltas)<<" for RPZ "<<zoneName<<endl;

    oldZone = luaconfsLocal->dfe.getZone(zoneIdx);
    /* we need to make a _full copy_ of the zone we are going to work on */
    std::shared_ptr<DNSFilterEngine::Zone> newZone = std::make_shared<DNSFilterEngine::Zone>(*oldZone);

    int totremove=0, totadd=0;
    bool fullUpdate = false;
    for(const auto& delta : deltas) {
      const auto& remove = delta.first;
      const auto& add = delta.second;
      if(remove.empty()) {
        L<<Logger::Warning<<"IXFR update is a whole new zone"<<endl;
        newZone->clear();
        fullUpdate = true;
      }
      for(const auto& rr : remove) { // should always contain the SOA
        if(rr.d_type == QType::NS)
          continue;
	if(rr.d_type == QType::SOA) {
	  auto oldsr = getRR<SOARecordContent>(rr);
	  if(oldsr && oldsr->d_st.serial == sr->d_st.serial) {
	    //	    cout<<"Got good removal of SOA serial "<<oldsr->d_st.serial<<endl;
	  }
	  else
	    L<<Logger::Error<<"GOT WRONG SOA SERIAL REMOVAL, SHOULD TRIGGER WHOLE RELOAD"<<endl;
	}
	else {
          totremove++;
	  L<<(g_logRPZChanges ? Logger::Info : Logger::Debug)<<"Had removal of "<<rr.d_name<<" from RPZ zone "<<zoneName<<endl;
	  RPZRecordToPolicy(rr, newZone, false, defpol, maxTTL);
	}
      }

      for(const auto& rr : add) { // should always contain the new SOA
        if(rr.d_type == QType::NS)
          continue;
	if(rr.d_type == QType::SOA) {
	  auto newsr = getRR<SOARecordContent>(rr);
	  //	  L<<Logger::Info<<"New SOA serial for "<<zoneName<<": "<<newsr->d_st.serial<<endl;
	  if (newsr) {
	    sr = newsr;
	  }
	}
	else {
          totadd++;
	  L<<(g_logRPZChanges ? Logger::Info : Logger::Debug)<<"Had addition of "<<rr.d_name<<" to RPZ zone "<<zoneName<<endl;
	  RPZRecordToPolicy(rr, newZone, true, defpol, maxTTL);
	}
      }
    }
    L<<Logger::Info<<"Had "<<totremove<<" RPZ removal"<<addS(totremove)<<", "<<totadd<<" addition"<<addS(totadd)<<" for "<<zoneName<<" New serial: "<<sr->d_st.serial<<endl;
    newZone->setSerial(sr->d_st.serial);
    setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), fullUpdate);

    /* we need to replace the existing zone with the new one,
       but we don't want to touch anything else, especially other zones,
       since they might have been updated by another RPZ IXFR tracker thread.
    */
    g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
                        lci.dfe.setZone(zoneIdx, newZone);
                      });
  }
}
Commit	Line	Data
dad05827	1	#include "syncres.hh"
644dd1da	2	#include "dnsparser.hh"
644dd1da	3	#include "dnsrecords.hh"
0240ae08	4	#include "ixfr.hh"
39ec5d29	5	#include "resolver.hh"
39ec5d29	6	#include "logger.hh"
ad42489c	7	#include "rec-lua-conf.hh"
0240ae08 RG	8	#include "rpzloader.hh"
0240ae08 RG	9	#include "zoneparser-tng.hh"
644dd1da	10
	11	static Netmask makeNetmaskFromRPZ(const DNSName& name)
	12	{
	13	auto parts = name.getRawLabels();
b8470add PL	14	/*
	15	* why 2?, the minimally valid IPv6 address that can be encoded in an RPZ is
	16	* $NETMASK.zz (::/$NETMASK)
	17	* Terrible right?
	18	*/
	19	if(parts.size() < 2 \|\| parts.size() > 9)
644dd1da	20	throw PDNSException("Invalid IP address in RPZ: "+name.toString());
b8470add PL	21
	22	bool isV6 = (stoi(parts[0]) > 32);
	23	bool hadZZ = false;
	24
	25	for (auto &part : parts) {
	26	// Check if we have an IPv4 octet
	27	for (auto c : part)
	28	if (!isdigit(c))
	29	isV6 = true;
	30
	31	if (pdns_iequals(part,"zz")) {
	32	if (hadZZ)
	33	throw PDNSException("more than one 'zz' label found in RPZ name"+name.toString());
	34	part = "";
	35	isV6 = true;
	36	hadZZ = true;
	37	}
	38	}
	39
	40	if (isV6 && parts.size() < 9 && !hadZZ)
	41	throw PDNSException("No 'zz' label found in an IPv6 RPZ name shorter than 9 elements: "+name.toString());
	42
	43	if (parts.size() == 5 && !isV6)
	44	return Netmask(parts[4]+"."+parts[3]+"."+parts[2]+"."+parts[1]+"/"+parts[0]);
	45
	46	string v6;
	47
	48	for (uint8_t i = parts.size()-1 ; i > 0; i--) {
	49	v6 += parts[i];
	50	if (parts[i] == "" && i == 1 && i == parts.size()-1)
	51	v6+= "::";
	52	if (parts[i] == "" && i != parts.size()-1)
	53	v6+= ":";
	54	if (parts[i] != "" && i != 1)
	55	v6 += ":";
	56	}
	57	v6 += "/" + parts[0];
	58
	59	return Netmask(v6);
644dd1da	60	}
644dd1da	61
6b972d59	62	void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr<DNSFilterEngine::Zone> zone, bool addOrRemove, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL)
644dd1da	63	{
644dd1da	64	static const DNSName drop("rpz-drop."), truncate("rpz-tcp-only."), noaction("rpz-passthru.");
644dd1da	65	static const DNSName rpzClientIP("rpz-client-ip"), rpzIP("rpz-ip"),
644dd1da	66	rpzNSDname("rpz-nsdname"), rpzNSIP("rpz-nsip.");
39c9bef5	67	static const std::string rpzPrefix("rpz-");
39ec5d29	68
1f1ca368	69	DNSFilterEngine::Policy pol;
39ec5d29	70
ba3c54cb RG	71	if(dr.d_class != QClass::IN) {
	72	return;
	73	}
	74
39ec5d29	75	if(dr.d_type == QType::CNAME) {
ba3c54cb RG	76	auto crc = getRR<CNAMERecordContent>(dr);
	77	if (!crc) {
	78	return;
	79	}
dd079764	80	auto crcTarget=crc->getTarget();
ad42489c	81	if(defpol) {
	82	pol=*defpol;
	83	}
dd079764	84	else if(crcTarget.isRoot()) {
39ec5d29	85	// cerr<<"Wants NXDOMAIN for "<<dr.d_name<<": ";
39ec5d29	86	pol.d_kind = DNSFilterEngine::PolicyKind::NXDOMAIN;
dd079764	87	} else if(crcTarget==g_wildcarddnsname) {
39ec5d29	88	// cerr<<"Wants NODATA for "<<dr.d_name<<": ";
	89	pol.d_kind = DNSFilterEngine::PolicyKind::NODATA;
	90	}
dd079764	91	else if(crcTarget==drop) {
39ec5d29	92	// cerr<<"Wants DROP for "<<dr.d_name<<": ";
	93	pol.d_kind = DNSFilterEngine::PolicyKind::Drop;
	94	}
dd079764	95	else if(crcTarget==truncate) {
39ec5d29	96	// cerr<<"Wants TRUNCATE for "<<dr.d_name<<": ";
	97	pol.d_kind = DNSFilterEngine::PolicyKind::Truncate;
	98	}
dd079764	99	else if(crcTarget==noaction) {
39ec5d29	100	// cerr<<"Wants NOACTION for "<<dr.d_name<<": ";
	101	pol.d_kind = DNSFilterEngine::PolicyKind::NoAction;
	102	}
39c9bef5 RG	103	/* "The special RPZ encodings which are not to be taken as Local Data are
	104	CNAMEs with targets that are:
	105	+ "." (NXDOMAIN action),
	106	+ "*." (NODATA action),
	107	+ a top level domain starting with "rpz-",
	108	+ a child of a top level domain starting with "rpz-".
	109	*/
	110	else if(!crcTarget.empty() && !crcTarget.isRoot() && crcTarget.getRawLabel(crcTarget.countLabels() - 1).compare(0, rpzPrefix.length(), rpzPrefix) == 0) {
	111	/* this is very likely an higher format number or a configuration error,
	112	let's just ignore it. */
	113	L<<Logger::Info<<"Discarding unsupported RPZ entry "<<crcTarget.toString()<<" for "<<dr.d_name<<endl;
	114	return;
	115	}
39ec5d29	116	else {
	117	pol.d_kind = DNSFilterEngine::PolicyKind::Custom;
	118	pol.d_custom = dr.d_content;
dd079764	119	// cerr<<"Wants custom "<<crcTarget<<" for "<<dr.d_name<<": ";
39ec5d29	120	}
	121	}
	122	else {
db7dcbb1 RG	123	if (defpol) {
	124	pol=*defpol;
	125	}
	126	else {
	127	pol.d_kind = DNSFilterEngine::PolicyKind::Custom;
	128	pol.d_custom = dr.d_content;
	129	// cerr<<"Wants custom "<<dr.d_content->getZoneRepresentation()<<" for "<<dr.d_name<<": ";
	130	}
39ec5d29	131	}
39ec5d29	132
8f618901 RG	133	if (!defpol \|\| defpol->d_ttl < 0) {
	134	pol.d_ttl = static_cast<int32_t>(std::min(maxTTL, dr.d_ttl));
	135	} else {
	136	pol.d_ttl = static_cast<int32_t>(std::min(maxTTL, static_cast<uint32_t>(pol.d_ttl)));
	137	}
3876ee44	138
39ec5d29	139	// now to DO something with that
39c9bef5	140
39ec5d29	141	if(dr.d_name.isPartOf(rpzNSDname)) {
	142	DNSName filt=dr.d_name.makeRelative(rpzNSDname);
	143	if(addOrRemove)
6b972d59	144	zone->addNSTrigger(filt, pol);
39ec5d29	145	else
6b972d59	146	zone->rmNSTrigger(filt, pol);
6791663c	147	} else if(dr.d_name.isPartOf(rpzClientIP)) {
b8470add PL	148	DNSName filt=dr.d_name.makeRelative(rpzClientIP);
b8470add PL	149	auto nm=makeNetmaskFromRPZ(filt);
39ec5d29	150	if(addOrRemove)
6b972d59	151	zone->addClientTrigger(nm, pol);
39ec5d29	152	else
6b972d59	153	zone->rmClientTrigger(nm, pol);
39ec5d29	154
6791663c	155	} else if(dr.d_name.isPartOf(rpzIP)) {
39ec5d29	156	// cerr<<"Should apply answer content IP policy: "<<dr.d_name<<endl;
b8470add PL	157	DNSName filt=dr.d_name.makeRelative(rpzIP);
b8470add PL	158	auto nm=makeNetmaskFromRPZ(filt);
39ec5d29	159	if(addOrRemove)
6b972d59	160	zone->addResponseTrigger(nm, pol);
39ec5d29	161	else
6b972d59	162	zone->rmResponseTrigger(nm, pol);
39ec5d29	163	} else if(dr.d_name.isPartOf(rpzNSIP)) {
b8470add PL	164	DNSName filt=dr.d_name.makeRelative(rpzNSIP);
	165	auto nm=makeNetmaskFromRPZ(filt);
	166	if(addOrRemove)
6b972d59	167	zone->addNSIPTrigger(nm, pol);
b8470add	168	else
6b972d59	169	zone->rmNSIPTrigger(nm, pol);
39ec5d29	170	} else {
39ec5d29	171	if(addOrRemove)
6b972d59	172	zone->addQNameTrigger(dr.d_name, pol);
39ec5d29	173	else
6b972d59	174	zone->rmQNameTrigger(dr.d_name, pol);
39ec5d29	175	}
	176	}
	177
5d81d096	178	shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout)
39ec5d29	179	{
6b972d59	180	L<<Logger::Warning<<"Loading RPZ zone '"<<zoneName<<"' from "<<master.toStringWithPort()<<endl;
98c9ec39	181	if(!tt.name.empty())
	182	L<<Logger::Warning<<"With TSIG key '"<<tt.name<<"' of algorithm '"<<tt.algo<<"'"<<endl;
	183
f6a8f7d7 PL	184	ComboAddress local(localAddress);
	185	if (local == ComboAddress())
	186	local = getQueryLocalAddress(master.sin4.sin_family, 0);
	187
3dcb082a	188	AXFRRetriever axfr(master, zoneName, tt, &local, maxReceivedBytes, axfrTimeout);
39ec5d29	189	unsigned int nrecords=0;
	190	Resolver::res_t nop;
	191	vector<DNSRecord> chunk;
	192	time_t last=0;
699ba3ea PL	193	time_t axfrStart = time(nullptr);
699ba3ea PL	194	time_t axfrNow = time(nullptr);
39ec5d29	195	shared_ptr<SOARecordContent> sr;
5d81d096	196	while(axfr.getChunk(nop, &chunk, (axfrStart + axfrTimeout - axfrNow))) {
39ec5d29	197	for(auto& dr : chunk) {
98c9ec39	198	if(dr.d_type==QType::NS \|\| dr.d_type==QType::TSIG) {
	199	continue;
	200	}
	201
6b972d59	202	dr.d_name.makeUsRelative(zoneName);
39ec5d29	203	if(dr.d_type==QType::SOA) {
ba3c54cb	204	sr = getRR<SOARecordContent>(dr);
39ec5d29	205	continue;
39ec5d29	206	}
39ec5d29	207
6b972d59	208	RPZRecordToPolicy(dr, zone, true, defpol, maxTTL);
39ec5d29	209	nrecords++;
39ec5d29	210	}
5d81d096	211	axfrNow = time(nullptr);
699ba3ea	212	if (axfrNow < axfrStart \|\| axfrNow - axfrStart > axfrTimeout) {
5d81d096 PL	213	throw PDNSException("Total AXFR time exceeded!");
5d81d096 PL	214	}
39ec5d29	215	if(last != time(0)) {
	216	L<<Logger::Info<<"Loaded & indexed "<<nrecords<<" policy records so far"<<endl;
	217	last=time(0);
	218	}
	219	}
	220	L<<Logger::Info<<"Done: "<<nrecords<<" policy records active, SOA: "<<sr->getZoneRepresentation()<<endl;
	221	return sr;
	222	}
	223
c823f860	224	// this function is silent - you do the logging
6b972d59	225	void loadRPZFromFile(const std::string& fname, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL)
39ec5d29	226	{
	227	ZoneParserTNG zpt(fname);
	228	DNSResourceRecord drr;
644dd1da	229	DNSName domain;
644dd1da	230	while(zpt.get(drr)) {
644dd1da	231	try {
	232	if(drr.qtype.getCode() == QType::CNAME && drr.content.empty())
	233	drr.content=".";
	234	DNSRecord dr(drr);
	235	if(dr.d_type == QType::SOA) {
6791663c RG	236	domain = dr.d_name;
6791663c RG	237	zone->setDomain(domain);
644dd1da	238	}
39ec5d29	239	else if(dr.d_type == QType::NS) {
	240	continue;
	241	}
	242	else {
644dd1da	243	dr.d_name=dr.d_name.makeRelative(domain);
6b972d59	244	RPZRecordToPolicy(dr, zone, true, defpol, maxTTL);
644dd1da	245	}
644dd1da	246	}
6791663c	247	catch(const PDNSException& pe) {
c823f860	248	throw PDNSException("Issue parsing '"+drr.qname.toString()+"' '"+drr.content+"' at "+zpt.getLineOfFile()+": "+pe.reason);
644dd1da	249	}
644dd1da	250	}
644dd1da	251	}
0240ae08	252
0e2cf8bd	253	static std::unordered_map<std::string, shared_ptr<rpzStats> > s_rpzStats;
0240ae08 RG	254	static std::mutex s_rpzStatsMutex;
0240ae08 RG	255
0e2cf8bd	256	shared_ptr<rpzStats> getRPZZoneStats(const std::string& zone)
0240ae08 RG	257	{
0240ae08 RG	258	std::lock_guard<std::mutex> l(s_rpzStatsMutex);
0e2cf8bd PL	259	if (s_rpzStats.find(zone) == s_rpzStats.end()) {
	260	s_rpzStats[zone] = std::make_shared<rpzStats>();
	261	}
0240ae08 RG	262	return s_rpzStats[zone];
	263	}
	264
	265	static void incRPZFailedTransfers(const std::string& zone)
	266	{
0e2cf8bd PL	267	auto stats = getRPZZoneStats(zone);
	268	if (stats != nullptr)
	269	stats->d_failedTransfers++;
0240ae08 RG	270	}
	271
	272	static void setRPZZoneNewState(const std::string& zone, uint32_t serial, uint64_t numberOfRecords, bool wasAXFR)
	273	{
0e2cf8bd PL	274	auto stats = getRPZZoneStats(zone);
	275	if (stats == nullptr)
	276	return;
	277	stats->d_successfulTransfers++;
0240ae08	278	if (wasAXFR) {
0e2cf8bd	279	stats->d_fullTransfers++;
0240ae08	280	}
0e2cf8bd PL	281	stats->d_lastUpdate = time(nullptr);
	282	stats->d_serial = serial;
	283	stats->d_numberOfRecords = numberOfRecords;
0240ae08 RG	284	}
0240ae08 RG	285
31394254	286	void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout)
0240ae08	287	{
31394254 RG	288	auto luaconfsLocal = g_luaconfs.getLocal();
	289	/* we can _never_ modify this zone directly, we need to do a full copy then replace the existing zone */
	290	std::shared_ptr<DNSFilterEngine::Zone> oldZone = luaconfsLocal->dfe.getZone(zoneIdx);
	291	if (!oldZone) {
	292	L<<Logger::Error<<"Unable to retrieve RPZ zone with index "<<zoneIdx<<" from the configuration, exiting"<<endl;
	293	return;
	294	}
	295	uint32_t refresh = oldZone->getRefresh();
	296	DNSName zoneName = oldZone->getDomain();
	297	std::string polName = oldZone->getName() ? *(oldZone->getName()) : zoneName.toString();
0240ae08 RG	298	shared_ptr<SOARecordContent> sr;
	299
	300	while (!sr) {
31394254 RG	301	/* full copy, as promised */
	302	std::shared_ptr<DNSFilterEngine::Zone> newZone = std::make_shared<DNSFilterEngine::Zone>(*oldZone);
	303
0240ae08	304	try {
cb0038f9	305	sr=loadRPZFromServer(master, zoneName, newZone, defpol, maxTTL, tt, maxReceivedBytes, localAddress, axfrTimeout);
0240ae08 RG	306	if(refresh == 0) {
	307	refresh = sr->d_st.refresh;
	308	}
cb0038f9 PL	309	newZone->setSerial(sr->d_st.serial);
	310	setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), true);
	311
	312	g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
	313	lci.dfe.setZone(zoneIdx, newZone);
	314	});
0240ae08 RG	315	}
0240ae08 RG	316	catch(const std::exception& e) {
9a7d779f	317	theL()<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.what()<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
0240ae08 RG	318	incRPZFailedTransfers(polName);
	319	}
	320	catch(const PDNSException& e) {
9a7d779f	321	theL()<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.reason<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
0240ae08 RG	322	incRPZFailedTransfers(polName);
	323	}
	324
	325	if (!sr) {
	326	if (refresh == 0) {
	327	sleep(10);
	328	} else {
	329	sleep(refresh);
	330	}
	331	}
	332	}
	333
	334	for(;;) {
	335	DNSRecord dr;
	336	dr.d_content=sr;
	337
	338	sleep(refresh);
	339
	340	L<<Logger::Info<<"Getting IXFR deltas for "<<zoneName<<" from "<<master.toStringWithPort()<<", our serial: "<<getRR<SOARecordContent>(dr)->d_st.serial<<endl;
	341	vector<pair<vector<DNSRecord>, vector<DNSRecord> > > deltas;
	342
	343	ComboAddress local(localAddress);
	344	if (local == ComboAddress())
	345	local = getQueryLocalAddress(master.sin4.sin_family, 0);
	346
	347	try {
	348	deltas = getIXFRDeltas(master, zoneName, dr, tt, &local, maxReceivedBytes);
	349	} catch(std::runtime_error& e ){
	350	L<<Logger::Warning<<e.what()<<endl;
	351	incRPZFailedTransfers(polName);
	352	continue;
	353	}
	354	if(deltas.empty())
	355	continue;
	356	L<<Logger::Info<<"Processing "<<deltas.size()<<" delta"<<addS(deltas)<<" for RPZ "<<zoneName<<endl;
	357
31394254	358	oldZone = luaconfsLocal->dfe.getZone(zoneIdx);
0240ae08 RG	359	/* we need to make a _full copy_ of the zone we are going to work on */
	360	std::shared_ptr<DNSFilterEngine::Zone> newZone = std::make_shared<DNSFilterEngine::Zone>(*oldZone);
	361
	362	int totremove=0, totadd=0;
	363	bool fullUpdate = false;
	364	for(const auto& delta : deltas) {
	365	const auto& remove = delta.first;
	366	const auto& add = delta.second;
	367	if(remove.empty()) {
	368	L<<Logger::Warning<<"IXFR update is a whole new zone"<<endl;
	369	newZone->clear();
	370	fullUpdate = true;
	371	}
	372	for(const auto& rr : remove) { // should always contain the SOA
	373	if(rr.d_type == QType::NS)
	374	continue;
	375	if(rr.d_type == QType::SOA) {
	376	auto oldsr = getRR<SOARecordContent>(rr);
	377	if(oldsr && oldsr->d_st.serial == sr->d_st.serial) {
	378	// cout<<"Got good removal of SOA serial "<<oldsr->d_st.serial<<endl;
	379	}
	380	else
	381	L<<Logger::Error<<"GOT WRONG SOA SERIAL REMOVAL, SHOULD TRIGGER WHOLE RELOAD"<<endl;
	382	}
	383	else {
	384	totremove++;
	385	L<<(g_logRPZChanges ? Logger::Info : Logger::Debug)<<"Had removal of "<<rr.d_name<<" from RPZ zone "<<zoneName<<endl;
	386	RPZRecordToPolicy(rr, newZone, false, defpol, maxTTL);
	387	}
	388	}
	389
	390	for(const auto& rr : add) { // should always contain the new SOA
	391	if(rr.d_type == QType::NS)
	392	continue;
	393	if(rr.d_type == QType::SOA) {
	394	auto newsr = getRR<SOARecordContent>(rr);
	395	// L<<Logger::Info<<"New SOA serial for "<<zoneName<<": "<<newsr->d_st.serial<<endl;
	396	if (newsr) {
	397	sr = newsr;
	398	}
	399	}
	400	else {
	401	totadd++;
	402	L<<(g_logRPZChanges ? Logger::Info : Logger::Debug)<<"Had addition of "<<rr.d_name<<" to RPZ zone "<<zoneName<<endl;
	403	RPZRecordToPolicy(rr, newZone, true, defpol, maxTTL);
	404	}
	405	}
	406	}
	407	L<<Logger::Info<<"Had "<<totremove<<" RPZ removal"<<addS(totremove)<<", "<<totadd<<" addition"<<addS(totadd)<<" for "<<zoneName<<" New serial: "<<sr->d_st.serial<<endl;
	408	newZone->setSerial(sr->d_st.serial);
	409	setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), fullUpdate);
	410
	411	/* we need to replace the existing zone with the new one,
	412	but we don't want to touch anything else, especially other zones,
	413	since they might have been updated by another RPZ IXFR tracker thread.
	414	*/
	415	g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
	416	lci.dfe.setZone(zoneIdx, newZone);
	417	});
	418	}
	419	}