pdns/zone2sql.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22
  23 /* accepts a named.conf or a zone as parameter and outputs heaps of sql */
  24
  25 #ifdef HAVE_CONFIG_H
  26 #include "config.h"
  27 #endif
  28 #include <unistd.h>
  29 #include <string>
  30 #include <map>
  31
  32 #include <iostream>
  33 #include <stdio.h>
  34 #include "json11.hpp"
  35 #include "namespaces.hh"
  36 #include "dns.hh"
  37 #include "arguments.hh"
  38 #include "bindparserclasses.hh"
  39 #include "statbag.hh"
  40 #include "misc.hh"
  41 #include "dnspacket.hh"
  42 #include "zoneparser-tng.hh"
  43 #include "dnsrecords.hh"
  44 #include <boost/algorithm/string.hpp>
  45 #include <sys/types.h>
  46 #include <sys/stat.h>
  47 #include <unistd.h>
  48
  49
  50
  51 StatBag S;
  52
  53 enum dbmode_t {MYSQL, POSTGRES, SQLITE};
  54 static dbmode_t g_mode;
  55 static bool g_intransaction;
  56 static int g_numRecords;
  57
  58
  59 /* this is an official wart. We don't terminate domains on a . in PowerDNS,
  60    which is fine as it goes, except for encoding the root, it would end up as '',
  61    which leads to ambiguities in the content field. Therefore, if we encounter
  62    the root as a . in a BIND zone, we leave it as a ., and don't replace it by
  63    an empty string. Back in 1999 we made the wrong choice. */
  64
  65 static string stripDotContent(const string& content)
  66 {
  67   if(boost::ends_with(content, " .") || content==".")
  68     return content;
  69   return stripDot(content);
  70 }
  71
  72 static string sqlstr(const string &name)
  73 {
  74   if(g_mode == SQLITE)
  75     return "'"+boost::replace_all_copy(name, "'", "''")+"'";
  76
  77   string a;
  78
  79   for(char i : name) {
  80     if(i=='\'' || i=='\\'){
  81       a+='\\';
  82       a+=i;
  83     }
  84     else
  85       a+=i;
  86   }
  87   if(g_mode == POSTGRES)
  88     return "E'"+a+"'";
  89   else
  90     return "'"+a+"'";
  91 }
  92
  93 static void startNewTransaction()
  94 {
  95   if(!::arg().mustDo("transactions"))
  96     return;
  97
  98   if(g_intransaction) {
  99     if(g_mode==POSTGRES) {
 100       cout<<"COMMIT WORK;"<<endl;
 101     }
 102     else if(g_mode == MYSQL || g_mode == SQLITE) {
 103       cout<<"COMMIT;"<<endl;
 104     }
 105   }
 106   g_intransaction=true;
 107
 108   if(g_mode == MYSQL)
 109     cout<<"BEGIN;"<<endl;
 110   else
 111     cout<<"BEGIN TRANSACTION;"<<endl;
 112 }
 113
 114 static void emitDomain(const DNSName& domain, const vector<ComboAddress>* primaries = nullptr)
 115 {
 116   string iDomain = domain.toStringRootDot();
 117   if (!::arg().mustDo("secondary")) {
 118     cout<<"insert into domains (name,type) values ("<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 119   }
 120   else
 121   {
 122     string mstrs;
 123     if (primaries != nullptr && !primaries->empty()) {
 124       for (const auto& mstr : *primaries) {
 125         mstrs.append(mstr.toStringWithPortExcept(53));
 126         mstrs.append(1, ' ');
 127       }
 128     }
 129     if (mstrs.empty())
 130       cout<<"insert into domains (name,type) values ("<<sqlstr(iDomain)<<",'NATIVE');"<<endl;
 131     else
 132       cout<<"insert into domains (name,type,master) values ("<<sqlstr(iDomain)<<",'SLAVE'"<<", '"<<mstrs<<"');"<<endl;
 133   }
 134 }
 135
 136 bool g_doJSONComments;
 137 static void emitRecord(const DNSName& zoneName, const DNSName &DNSqname, const string &qtype, const string &ocontent, int ttl, const string& comment="")
 138 {
 139   string qname = DNSqname.toStringRootDot();
 140   string zname = zoneName.toStringRootDot();
 141   int prio=0;
 142   int disabled=0;
 143   string recordcomment;
 144
 145   if(g_doJSONComments && !comment.empty()) {
 146     string::size_type pos = comment.find("json={");
 147     if(pos!=string::npos) {
 148       string json = comment.substr(pos+5);
 149       string err;
 150       auto document = json11::Json::parse(json, err);
 151       if(document.is_null())
 152         throw runtime_error("Could not parse JSON '"+json+"': " + err);
 153
 154       disabled=document["disabled"].bool_value();
 155       recordcomment=document["comment"].string_value();
 156     }
 157   }
 158
 159   g_numRecords++;
 160   string content(ocontent);
 161
 162   if(qtype == "NSEC" || qtype == "NSEC3")
 163     return; // NSECs do not go in the database
 164
 165   if((qtype == "MX" || qtype == "SRV")) {
 166     pdns::checked_stoi_into(prio, content);
 167
 168     string::size_type pos = content.find_first_not_of("0123456789");
 169     if(pos != string::npos)
 170       boost::erase_head(content, pos);
 171     boost::trim_left(content);
 172   }
 173
 174   cout<<"insert into records (domain_id, name, type,content,ttl,prio,disabled) select id ,"<<
 175     sqlstr(toLower(qname))<<", "<<
 176     sqlstr(qtype)<<", "<<
 177     sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<(g_mode==POSTGRES ? (disabled ? "'t'" : "'f'") : std::to_string(disabled))<<
 178     " from domains where name="<<toLower(sqlstr(zname))<<";\n";
 179
 180   if(!recordcomment.empty()) {
 181     cout<<"insert into comments (domain_id,name,type,modified_at, comment) select id, "<<toLower(sqlstr(stripDot(qname)))<<", "<<sqlstr(qtype)<<", "<<time(nullptr)<<", "<<sqlstr(recordcomment)<<" from domains where name="<<toLower(sqlstr(zname))<<";\n";
 182   }
 183 }
 184
 185
 186 /* 2 modes of operation, either --named or --zone (the latter needs $ORIGIN)
 187    1 further mode: --mysql
 188 */
 189
 190 ArgvMap &arg()
 191 {
 192   static ArgvMap theArg;
 193   return theArg;
 194 }
 195
 196
 197 int main(int argc, char **argv) // NOLINT(readability-function-cognitive-complexity) 13379 https://github.com/PowerDNS/pdns/issues/13379 Habbie: zone2sql.cc, bindbackend2.cc: reduce complexity
 198 try
 199 {
 200     reportAllTypes();
 201     std::ios_base::sync_with_stdio(false);
 202
 203     ::arg().setSwitch("gpgsql","Output in format suitable for default gpgsqlbackend")="no";
 204     ::arg().setSwitch("gmysql","Output in format suitable for default gmysqlbackend")="no";
 205     ::arg().setSwitch("gsqlite","Output in format suitable for default gsqlitebackend")="no";
 206     ::arg().setSwitch("verbose","Verbose comments on operation")="no";
 207     ::arg().setSwitch("secondary", "Keep BIND secondaries as secondaries. Only works with named-conf.") = "no";
 208     ::arg().setSwitch("json-comments","Parse json={} field for disabled & comments")="no";
 209     ::arg().setSwitch("transactions","If target SQL supports it, use transactions")="no";
 210     ::arg().setSwitch("on-error-resume-next","Continue after errors")="no";
 211     ::arg().setSwitch("filter-duplicate-soa","Filter second SOA in zone")="yes";
 212     ::arg().set("zone","Zonefile to parse")="";
 213     ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")="";
 214     ::arg().set("named-conf","Bind 8/9 named.conf to parse")="";
 215
 216     ::arg().set("max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0";
 217     ::arg().set("max-include-depth", "Maximum nested $INCLUDE depth when loading a zone from a file")="20";
 218
 219     ::arg().setCmd("help","Provide a helpful message");
 220     ::arg().setCmd("version","Print the version");
 221
 222     S.declare("logmessages");
 223
 224     string namedfile="";
 225     string zonefile="";
 226
 227     ::arg().parse(argc, argv);
 228
 229     if(::arg().mustDo("version")) {
 230       cerr<<"zone2sql "<<VERSION<<endl;
 231       exit(0);
 232     }
 233
 234     if(::arg().mustDo("help")) {
 235       cout<<"syntax:"<<endl<<endl;
 236       cout<<::arg().helpstring()<<endl;
 237       exit(0);
 238     }
 239
 240     if(argc<2) {
 241       cerr<<"syntax:"<<endl<<endl;
 242       cerr<<::arg().helpstring()<<endl;
 243       exit(1);
 244     }
 245
 246     bool filterDupSOA = ::arg().mustDo("filter-duplicate-soa");
 247
 248     g_doJSONComments=::arg().mustDo("json-comments");
 249
 250     if(::arg().mustDo("gmysql"))
 251       g_mode=MYSQL;
 252     else if(::arg().mustDo("gpgsql"))
 253       g_mode=POSTGRES;
 254     else if(::arg().mustDo("gsqlite"))
 255       g_mode=SQLITE;
 256     else {
 257       cerr<<"Unknown SQL mode!\n\n";
 258       cerr<<"syntax:"<<endl<<endl;
 259       cerr<<::arg().helpstring()<<endl;
 260       exit(1);
 261     }
 262
 263     namedfile=::arg()["named-conf"];
 264     zonefile=::arg()["zone"];
 265
 266     int count=0, num_domainsdone=0;
 267
 268     if(zonefile.empty()) {
 269       BindParser BP;
 270       BP.setVerbose(::arg().mustDo("verbose"));
 271       BP.parse(namedfile.empty() ? "./named.conf" : namedfile);
 272
 273       vector<BindDomainInfo> domains=BP.getDomains();
 274       struct stat st;
 275       for(auto & domain : domains) {
 276         if(stat(domain.filename.c_str(), &st) == 0) {
 277           domain.d_dev = st.st_dev;
 278           domain.d_ino = st.st_ino;
 279         }
 280       }
 281
 282       sort(domains.begin(), domains.end()); // put stuff in inode order
 283
 284       int numdomains=domains.size();
 285       int tick=numdomains/100;
 286
 287       for(const auto & domain : domains)
 288         {
 289         if (domain.type != "primary" && domain.type != "secondary" && !domain.type.empty() && domain.type != "master" && domain.type != "slave") {
 290           cerr << " Warning! Skipping '" << domain.type << "' zone '" << domain.name << "'" << endl;
 291           continue;
 292         }
 293           try {
 294             startNewTransaction();
 295
 296             emitDomain(domain.name, &(domain.primaries));
 297
 298             ZoneParserTNG zpt(domain.filename, domain.name, BP.getDirectory());
 299             zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps"));
 300             zpt.setMaxIncludes(::arg().asNum("max-include-depth"));
 301             DNSResourceRecord rr;
 302             bool seenSOA=false;
 303             string comment;
 304             while(zpt.get(rr, &comment)) {
 305               if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 306                 continue;
 307               if(rr.qtype.getCode() == QType::SOA)
 308                 seenSOA=true;
 309
 310               emitRecord(domain.name, rr.qname, rr.qtype.toString(), rr.content, rr.ttl, comment);
 311             }
 312             num_domainsdone++;
 313           }
 314           catch(std::exception &ae) {
 315             if(!::arg().mustDo("on-error-resume-next"))
 316               throw;
 317             else
 318               cerr<<endl<<ae.what()<<endl;
 319           }
 320           catch(PDNSException &ae) {
 321             if(!::arg().mustDo("on-error-resume-next"))
 322               throw;
 323             else
 324               cerr<<ae.reason<<endl;
 325           }
 326
 327
 328           if(!tick || !((count++)%tick))
 329             cerr<<"\r"<<count*100/numdomains<<"% done ("<<domain.filename<<")\033\133\113";
 330         }
 331       cerr<<"\r100% done\033\133\113"<<endl;
 332     }
 333     else {
 334       DNSName zonename;
 335       if(!::arg()["zone-name"].empty())
 336         zonename = DNSName(::arg()["zone-name"]);
 337
 338       ZoneParserTNG zpt(zonefile, zonename);
 339       zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps"));
 340       DNSResourceRecord rr;
 341       startNewTransaction();
 342       string comment;
 343       bool seenSOA=false;
 344       bool haveEmittedZone = false;
 345       while(zpt.get(rr, &comment))  {
 346         if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 347           continue;
 348         if(rr.qtype.getCode() == QType::SOA)
 349           seenSOA=true;
 350         if(!haveEmittedZone) {
 351           if(!zpt.getZoneName().empty()){
 352             emitDomain(zpt.getZoneName());
 353             haveEmittedZone = true;
 354           } else {
 355             // We have no zonename yet, don't emit
 356             continue;
 357           }
 358         }
 359
 360         emitRecord(zpt.getZoneName(), rr.qname, rr.qtype.toString(), rr.content, rr.ttl, comment);
 361       }
 362       num_domainsdone=1;
 363     }
 364     cerr<<num_domainsdone<<" domains were fully parsed, containing "<<g_numRecords<<" records\n";
 365
 366   if(::arg().mustDo("transactions") && g_intransaction) {
 367     if(g_mode != SQLITE)
 368       cout<<"COMMIT WORK;"<<endl;
 369     else
 370       cout<<"COMMIT;"<<endl;
 371   }
 372   return 0;
 373 }
 374 catch(PDNSException &ae) {
 375   cerr<<"\nFatal error: "<<ae.reason<<endl;
 376   return 1;
 377 }
 378 catch(std::exception &e) {
 379   cerr<<"\ndied because of STL error: "<<e.what()<<endl;
 380   return 1;
 381 }
 382 catch(...) {
 383   cerr<<"\ndied because of unknown exception"<<endl;
 384   return 1;
 385 }