From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Mon, 27 Aug 2018 13:33:24 +0000 (+0200)
Subject: ixfrdist: Reduce memory usage during zone retrieval
X-Git-Tag: dnsdist-1.3.3~143^2~7
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=451605af5cce5d10d93720d5f0d6b266ed174e82;p=thirdparty%2Fpdns.git

ixfrdist: Reduce memory usage during zone retrieval
---

diff --git a/pdns/ixfrdist.cc b/pdns/ixfrdist.cc
index 383665cd4d..779cb8608a 100644
--- a/pdns/ixfrdist.cc
+++ b/pdns/ixfrdist.cc
@@ -297,12 +297,12 @@ void updateThread(const string& workdir, const uint16_t& keep, const uint16_t& a
 
       // The *new* SOA
       shared_ptr<SOARecordContent> soa;
+      records_t records;
       try {
         AXFRRetriever axfr(master, domain, tt, &local);
         unsigned int nrecords=0;
         Resolver::res_t nop;
         vector<DNSRecord> chunk;
-        records_t records;
         time_t t_start = time(nullptr);
         time_t axfr_now = time(nullptr);
         while(axfr.getChunk(nop, &chunk, (axfr_now - t_start + axfrTimeout))) {
@@ -326,27 +326,39 @@ void updateThread(const string& workdir, const uint16_t& keep, const uint16_t& a
           continue;
         }
         g_log<<Logger::Notice<<"Retrieved all zone data for "<<domain<<". Received "<<nrecords<<" records."<<endl;
+      } catch (PDNSException &e) {
+        g_log<<Logger::Warning<<"Could not retrieve AXFR for '"<<domain<<"': "<<e.reason<<endl;
+      } catch (runtime_error &e) {
+        g_log<<Logger::Warning<<"Could not retrieve AXFR for zone '"<<domain<<"': "<<e.what()<<endl;
+      }
+
+      try {
+
         writeZoneToDisk(records, domain, dir);
         g_log<<Logger::Notice<<"Wrote zonedata for "<<domain<<" with serial "<<soa->d_st.serial<<" to "<<dir<<endl;
+
         {
           std::lock_guard<std::mutex> guard(g_soas_mutex);
           ixfrdiff_t diff;
           if (!g_soas[domain].latestAXFR.empty()) {
             makeIXFRDiff(g_soas[domain].latestAXFR, records, diff, g_soas[domain].soa, soa);
-            g_soas[domain].ixfrDiffs.push_back(diff);
+            g_soas[domain].ixfrDiffs.push_back(std::move(diff));
           }
+
           // Clean up the diffs
           while (g_soas[domain].ixfrDiffs.size() > keep) {
             g_soas[domain].ixfrDiffs.erase(g_soas[domain].ixfrDiffs.begin());
           }
-          g_soas[domain].latestAXFR = records;
+
+          g_soas[domain].latestAXFR = std::move(records);
           g_soas[domain].soa = soa;
         }
       } catch (PDNSException &e) {
-        g_log<<Logger::Warning<<"Could not retrieve AXFR for '"<<domain<<"': "<<e.reason<<endl;
+        g_log<<Logger::Warning<<"Could not save zone '"<<domain<<"' to disk: "<<e.reason<<endl;
       } catch (runtime_error &e) {
         g_log<<Logger::Warning<<"Could not save zone '"<<domain<<"' to disk: "<<e.what()<<endl;
       }
+
       // Now clean up the directory
       cleanUpDomain(domain, keep, workdir);
     } /* for (const auto &domain : domains) */
diff --git a/pdns/ixfrutils.cc b/pdns/ixfrutils.cc
index 04c832451d..c7757042ba 100644
--- a/pdns/ixfrutils.cc
+++ b/pdns/ixfrutils.cc
@@ -105,6 +105,17 @@ uint32_t getSerialFromRecords(const records_t& records, DNSRecord& soaret)
   return 0;
 }
 
+static void writeRecords(FILE* fp, const records_t& records)
+{
+  for(const auto& r: records) {
+    fprintf(fp, "%s\t%" PRIu32 "\tIN\t%s\t%s\n",
+            r.d_name.isRoot() ? "@" :  r.d_name.toStringNoDot().c_str(),
+            r.d_ttl,
+            DNSRecordContent::NumberToType(r.d_type).c_str(),
+            r.d_content->getZoneRepresentation().c_str());
+  }
+}
+
 void writeZoneToDisk(const records_t& records, const DNSName& zone, const std::string& directory)
 {
   DNSRecord soa;
@@ -117,15 +128,11 @@ void writeZoneToDisk(const records_t& records, const DNSName& zone, const std::s
   records_t soarecord;
   soarecord.insert(soa);
   fprintf(fp, "$ORIGIN %s\n", zone.toString().c_str());
-  for(const auto& outer : {soarecord, records, soarecord} ) {
-    for(const auto& r: outer) {
-      fprintf(fp, "%s\t%" PRIu32 "\tIN\t%s\t%s\n",
-          r.d_name.isRoot() ? "@" :  r.d_name.toStringNoDot().c_str(),
-          r.d_ttl,
-          DNSRecordContent::NumberToType(r.d_type).c_str(),
-          r.d_content->getZoneRepresentation().c_str());
-    }
-  }
+
+  writeRecords(fp, soarecord);
+  writeRecords(fp, records);
+  writeRecords(fp, soarecord);
+
   fclose(fp);
   rename( (fname+".partial").c_str(), fname.c_str());
 }