pdns/zone2sql.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22
  23 /* accepts a named.conf or a zone as parameter and outputs heaps of sql */
  24
  25 #ifdef HAVE_CONFIG_H
  26 #include "config.h"
  27 #endif
  28 #include <unistd.h>
  29 #include <string>
  30 #include <map>
  31
  32 #include <iostream>
  33 #include <stdio.h>
  34 #include "json11.hpp"
  35 #include "namespaces.hh"
  36 #include "dns.hh"
  37 #include "arguments.hh"
  38 #include "bindparserclasses.hh"
  39 #include "statbag.hh"
  40 #include "misc.hh"
  41 #include "dnspacket.hh"
  42 #include "zoneparser-tng.hh"
  43 #include "dnsrecords.hh"
  44 #include <boost/algorithm/string.hpp>
  45 #include <sys/types.h>
  46 #include <sys/stat.h>
  47 #include <unistd.h>
  48
  49
  50
  51 StatBag S;
  52
  53 enum dbmode_t {MYSQL, GORACLE, POSTGRES, SQLITE, MYDNS, ORACLE};
  54 static dbmode_t g_mode;
  55 static bool g_intransaction;
  56 static int g_numRecords;
  57
  58
  59 /* this is an official wart. We don't terminate domains on a . in PowerDNS,
  60    which is fine as it goes, except for encoding the root, it would end up as '',
  61    which leads to ambiguities in the content field. Therefore, if we encounter
  62    the root as a . in a BIND zone, we leave it as a ., and don't replace it by
  63    an empty string. Back in 1999 we made the wrong choice. */
  64
  65 static string stripDotContent(const string& content)
  66 {
  67   if(boost::ends_with(content, " .") || content==".")
  68     return content;
  69   return stripDot(content);
  70 }
  71
  72 static string sqlstr(const string &name)
  73 {
  74   if(g_mode == SQLITE || g_mode==GORACLE || g_mode==ORACLE)
  75     return "'"+boost::replace_all_copy(name, "'", "''")+"'";
  76
  77   string a;
  78
  79   for(string::const_iterator i=name.begin();i!=name.end();++i) {
  80     if(*i=='\'' || *i=='\\'){
  81       a+='\\';
  82       a+=*i;
  83     }
  84     else
  85       a+=*i;
  86   }
  87   if(g_mode == POSTGRES)
  88     return "E'"+a+"'";
  89   else
  90     return "'"+a+"'";
  91 }
  92
  93 static void startNewTransaction()
  94 {
  95   if(!::arg().mustDo("transactions"))
  96     return;
  97
  98   if(g_intransaction) {
  99     if(g_mode==POSTGRES) {
 100       cout<<"COMMIT WORK;"<<endl;
 101     }
 102     else if(g_mode == MYSQL || g_mode == SQLITE || g_mode == MYDNS) {
 103       cout<<"COMMIT;"<<endl;
 104     }
 105   }
 106   g_intransaction=1;
 107
 108   if(g_mode == MYSQL || g_mode == MYDNS)
 109     cout<<"BEGIN;"<<endl;
 110   else if (g_mode!=GORACLE && g_mode!=ORACLE)
 111     cout<<"BEGIN TRANSACTION;"<<endl;
 112 }
 113
 114 static void emitDomain(const DNSName& domain, const vector<ComboAddress> *masters = 0) {
 115   string iDomain = domain.toStringRootDot();
 116   if(!::arg().mustDo("slave")) {
 117     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 118       cout<<"insert into domains (name,type) values ("<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 119     }
 120     else if(g_mode==GORACLE) {
 121       cout<<"insert into domains (id,name,type) values (domains_id_sequence.nextval,"<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 122     }
 123     else if(g_mode==ORACLE) {
 124       cout<<"INSERT INTO Zones (id, name, type) VALUES (zones_id_seq.nextval, "<<sqlstr(toLower(iDomain))<<", 'NATIVE');"<<endl;
 125     }
 126   }
 127   else
 128   {
 129
 130     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 131       string mstrs;
 132       if (masters != 0 && ! masters->empty()) {
 133         for(const auto& mstr :  *masters) {
 134           mstrs.append(mstr.toStringWithPortExcept(53));
 135           mstrs.append(1, ' ');
 136         }
 137       }
 138       if (mstrs.empty())
 139         cout<<"insert into domains (name,type) values ("<<sqlstr(iDomain)<<",'NATIVE');"<<endl;
 140       else
 141         cout<<"insert into domains (name,type,master) values ("<<sqlstr(iDomain)<<",'SLAVE'"<<", '"<<mstrs<<"');"<<endl;
 142     }
 143     else if (g_mode == GORACLE || g_mode==ORACLE) {
 144       cerr<<"Slave import mode not supported with oracle."<<endl;
 145     }
 146   }
 147 }
 148
 149 bool g_doJSONComments;
 150 static void emitRecord(const DNSName& zoneName, const DNSName &DNSqname, const string &qtype, const string &ocontent, int ttl, const string& comment="")
 151 {
 152   string qname = DNSqname.toStringRootDot();
 153   string zname = zoneName.toStringRootDot();
 154   int prio=0;
 155   int disabled=0;
 156   string recordcomment;
 157
 158   if(g_doJSONComments & !comment.empty()) {
 159     string::size_type pos = comment.find("json={");
 160     if(pos!=string::npos) {
 161       string json = comment.substr(pos+5);
 162       string err;
 163       auto document = json11::Json::parse(json, err);
 164       if(document.is_null())
 165         throw runtime_error("Could not parse JSON '"+json+"': " + err);
 166
 167       disabled=document["disabled"].bool_value();
 168       recordcomment=document["comment"].string_value();
 169     }
 170   }
 171
 172   g_numRecords++;
 173   string content(ocontent);
 174
 175   if(qtype == "NSEC" || qtype == "NSEC3")
 176     return; // NSECs do not go in the database
 177
 178   if((qtype == "MX" || qtype == "SRV") && g_mode!=ORACLE) {
 179     prio=pdns_stou(content);
 180
 181     string::size_type pos = content.find_first_not_of("0123456789");
 182     if(pos != string::npos)
 183       boost::erase_head(content, pos);
 184     trim_left(content);
 185   }
 186
 187   bool auth = true;
 188   if(qtype == "NS" && !pdns_iequals(qname, zname)) {
 189     auth=false;
 190   }
 191
 192   if(g_mode==MYSQL || g_mode==SQLITE) {
 193     cout<<"insert into records (domain_id, name, type,content,ttl,prio,disabled) select id ,"<<
 194       sqlstr(toLower(qname))<<", "<<
 195       sqlstr(qtype)<<", "<<
 196       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<disabled<<
 197       " from domains where name="<<toLower(sqlstr(zname))<<";\n";
 198
 199     if(!recordcomment.empty()) {
 200       cout<<"insert into comments (domain_id,name,type,modified_at, comment) select id, "<<toLower(sqlstr(stripDot(qname)))<<", "<<sqlstr(qtype)<<", "<<time(0)<<", "<<sqlstr(recordcomment)<<" from domains where name="<<toLower(sqlstr(zname))<<";\n";
 201     }
 202   }
 203   else if(g_mode==POSTGRES) {
 204     cout<<"insert into records (domain_id, name, ordername, auth, type,content,ttl,prio,disabled) select id ,"<<
 205       sqlstr(toLower(qname))<<", "<<
 206       sqlstr(DNSName(qname).makeRelative(DNSName(zname)).makeLowerCase().labelReverse().toString(" ", false))<<", '"<< (auth  ? 't' : 'f') <<"', "<<
 207       sqlstr(qtype)<<", "<<
 208       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", '"<<(disabled ? 't': 'f') <<
 209       "' from domains where name="<<toLower(sqlstr(zname))<<";\n";
 210   }
 211   else if(g_mode==GORACLE) {
 212     cout<<"insert into Records (id, domain_id, name, type, content, ttl, prio, disabled) select RECORDS_ID_SEQUENCE.nextval,id ,"<<
 213       sqlstr(toLower(qname))<<", "<<
 214       sqlstr(qtype)<<", "<<
 215       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<disabled<<
 216       " from Domains where name="<<toLower(sqlstr(zname))<<";\n";
 217   }
 218   else if(g_mode==ORACLE) {
 219     cout<<"INSERT INTO Records (id, zone_id, fqdn, ttl, type, content) SELECT records_id_seq.nextval, id, "<<
 220       sqlstr(toLower(qname))<<", "<<
 221       ttl<<", "<<sqlstr(qtype)<<", "<<
 222       sqlstr(stripDotContent(content))<<
 223       " FROM Zones WHERE name="<<toLower(sqlstr(zname))<<";"<<endl;
 224   }
 225   else if (g_mode == MYDNS) {
 226     string zoneNameDot = zname + ".";
 227     if (qtype == "A" || qtype == "AAAA" || qtype == "CNAME" || qtype == "HINFO" || qtype == "MX" || qtype == "NAPTR" ||
 228         qtype == "NS" || qtype == "PTR" || qtype == "RP" || qtype == "SRV" || qtype == "TXT")
 229     {
 230       if ((qtype == "MX" || qtype == "NS" || qtype == "SRV" || qtype == "CNAME") && content[content.size()-1] != '.')
 231         content.append(".");
 232       cout<<"INSERT INTO rr(zone, name, type, data, aux, ttl) VALUES("<<
 233       "(SELECT id FROM soa WHERE origin = "<<
 234       sqlstr(toLower(zoneNameDot))<<"), "<<
 235       sqlstr(toLower(DNSqname.toString()))<<", "<<
 236       sqlstr(qtype)<<", "<<sqlstr(content)<<", "<<prio<<", "<<ttl<<");\n";
 237     }
 238     else if (qtype == "SOA") {
 239       //pdns CONTENT = ns1.wtest.com. ahu.example.com. 2005092501 28800 7200 604800 86400
 240       vector<string> parts;
 241       stringtok(parts, content);
 242
 243       cout<<"INSERT INTO soa(origin, ns, mbox, serial, refresh, retry, expire, minimum, ttl) VALUES("<<
 244       sqlstr(toLower(zoneNameDot))<<", "<<sqlstr(parts[0])<<", "<<sqlstr(parts[1])<<", "<<pdns_stou(parts[2])<<", "<<
 245       pdns_stou(parts[3])<<", "<<pdns_stou(parts[4])<<", "<<pdns_stou(parts[5])<<", "<<pdns_stou(parts[6])<<", "<<ttl<<");\n";
 246     }
 247     else
 248     {
 249       cerr<<"Record type "<<qtype<<" is not supported."<<endl;
 250     }
 251   }
 252 }
 253
 254
 255 /* 2 modes of operation, either --named or --zone (the latter needs $ORIGIN)
 256    2 further modes: --mysql, --goracle or --oracle
 257 */
 258
 259 ArgvMap &arg()
 260 {
 261   static ArgvMap theArg;
 262   return theArg;
 263 }
 264
 265
 266 int main(int argc, char **argv)
 267 try
 268 {
 269     reportAllTypes();
 270     std::ios_base::sync_with_stdio(false);
 271
 272     ::arg().setSwitch("gpgsql","Output in format suitable for default gpgsqlbackend")="no";
 273     ::arg().setSwitch("gmysql","Output in format suitable for default gmysqlbackend")="no";
 274     ::arg().setSwitch("mydns","Output in format suitable for default mydnsbackend")="no";
 275     ::arg().setSwitch("goracle","Output in format suitable for the goraclebackend")="no";
 276     ::arg().setSwitch("oracle","Output in format suitable for the oraclebackend")="no";
 277     ::arg().setSwitch("gsqlite","Output in format suitable for default gsqlitebackend")="no";
 278     ::arg().setSwitch("verbose","Verbose comments on operation")="no";
 279     ::arg().setSwitch("slave","Keep BIND slaves as slaves. Only works with named-conf.")="no";
 280     ::arg().setSwitch("json-comments","Parse json={} field for disabled & comments")="no";
 281     ::arg().setSwitch("transactions","If target SQL supports it, use transactions")="no";
 282     ::arg().setSwitch("on-error-resume-next","Continue after errors")="no";
 283     ::arg().setSwitch("filter-duplicate-soa","Filter second SOA in zone")="yes";
 284     ::arg().set("zone","Zonefile to parse")="";
 285     ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")="";
 286     ::arg().set("named-conf","Bind 8/9 named.conf to parse")="";
 287
 288     ::arg().set("soa-minimum-ttl","Do not change")="0";
 289     ::arg().set("soa-refresh-default","Do not change")="0";
 290     ::arg().set("soa-retry-default","Do not change")="0";
 291     ::arg().set("soa-expire-default","Do not change")="0";
 292
 293     ::arg().setCmd("help","Provide a helpful message");
 294     ::arg().setCmd("version","Print the version");
 295
 296     S.declare("logmessages");
 297
 298     string namedfile="";
 299     string zonefile="";
 300
 301     ::arg().parse(argc, argv);
 302
 303     if(::arg().mustDo("version")) {
 304       cerr<<"zone2sql "<<VERSION<<endl;
 305       exit(0);
 306     }
 307
 308     if(::arg().mustDo("help")) {
 309       cout<<"syntax:"<<endl<<endl;
 310       cout<<::arg().helpstring()<<endl;
 311       exit(0);
 312     }
 313
 314     if(argc<2) {
 315       cerr<<"syntax:"<<endl<<endl;
 316       cerr<<::arg().helpstring()<<endl;
 317       exit(1);
 318     }
 319
 320     bool filterDupSOA = ::arg().mustDo("filter-duplicate-soa");
 321
 322     g_doJSONComments=::arg().mustDo("json-comments");
 323
 324     if(::arg().mustDo("gmysql"))
 325       g_mode=MYSQL;
 326     else if(::arg().mustDo("gpgsql"))
 327       g_mode=POSTGRES;
 328     else if(::arg().mustDo("gsqlite"))
 329       g_mode=SQLITE;
 330     else if(::arg().mustDo("goracle"))
 331       g_mode=GORACLE;
 332     else if(::arg().mustDo("oracle"))
 333       g_mode=ORACLE;
 334     else if(::arg().mustDo("mydns"))
 335       g_mode=MYDNS;
 336     else {
 337       cerr<<"Unknown SQL mode!\n\n";
 338       cerr<<"syntax:"<<endl<<endl;
 339       cerr<<::arg().helpstring()<<endl;
 340       exit(1);
 341     }
 342
 343     if((g_mode==GORACLE || g_mode==ORACLE) && !::arg().mustDo("transactions"))
 344       cout<<"set autocommit on;"<<endl;
 345
 346     namedfile=::arg()["named-conf"];
 347     zonefile=::arg()["zone"];
 348
 349     int count=0, num_domainsdone=0;
 350
 351     if(zonefile.empty()) {
 352       BindParser BP;
 353       BP.setVerbose(::arg().mustDo("verbose"));
 354       BP.parse(namedfile.empty() ? "./named.conf" : namedfile);
 355
 356       vector<BindDomainInfo> domains=BP.getDomains();
 357       struct stat st;
 358       for(vector<BindDomainInfo>::iterator i=domains.begin(); i!=domains.end(); ++i) {
 359         if(stat(i->filename.c_str(), &st) == 0) {
 360           i->d_dev = st.st_dev;
 361           i->d_ino = st.st_ino;
 362         }
 363       }
 364
 365       sort(domains.begin(), domains.end()); // put stuff in inode order
 366
 367       int numdomains=domains.size();
 368       int tick=numdomains/100;
 369
 370       for(vector<BindDomainInfo>::const_iterator i=domains.begin();
 371           i!=domains.end();
 372           ++i)
 373         {
 374           if(i->type!="master" && i->type!="slave") {
 375             cerr<<" Warning! Skipping '"<<i->type<<"' zone '"<<i->name<<"'"<<endl;
 376             continue;
 377           }
 378           try {
 379             startNewTransaction();
 380
 381             emitDomain(i->name, &(i->masters));
 382
 383             ZoneParserTNG zpt(i->filename, i->name, BP.getDirectory());
 384             DNSResourceRecord rr;
 385             bool seenSOA=false;
 386             string comment;
 387             while(zpt.get(rr, &comment)) {
 388               if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 389                 continue;
 390               if(rr.qtype.getCode() == QType::SOA)
 391                 seenSOA=true;
 392
 393               emitRecord(i->name, rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 394             }
 395             num_domainsdone++;
 396           }
 397           catch(std::exception &ae) {
 398             if(!::arg().mustDo("on-error-resume-next"))
 399               throw;
 400             else
 401               cerr<<endl<<ae.what()<<endl;
 402           }
 403           catch(PDNSException &ae) {
 404             if(!::arg().mustDo("on-error-resume-next"))
 405               throw;
 406             else
 407               cerr<<ae.reason<<endl;
 408           }
 409
 410
 411           if(!tick || !((count++)%tick))
 412             cerr<<"\r"<<count*100/numdomains<<"% done ("<<i->filename<<")\033\133\113";
 413         }
 414       cerr<<"\r100% done\033\133\113"<<endl;
 415     }
 416     else {
 417       DNSName zonename;
 418       if(!::arg()["zone-name"].empty())
 419         zonename = DNSName(::arg()["zone-name"]);
 420
 421       ZoneParserTNG zpt(zonefile, zonename);
 422       DNSResourceRecord rr;
 423       startNewTransaction();
 424       string comment;
 425       bool seenSOA=false;
 426       bool haveEmittedZone = false;
 427       while(zpt.get(rr, &comment))  {
 428         if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 429           continue;
 430         if(rr.qtype.getCode() == QType::SOA)
 431           seenSOA=true;
 432         if(!haveEmittedZone) {
 433           if(!zpt.getZoneName().empty()){
 434             emitDomain(zpt.getZoneName());
 435             haveEmittedZone = true;
 436           } else {
 437             // We have no zonename yet, don't emit
 438             continue;
 439           }
 440         }
 441
 442         emitRecord(zpt.getZoneName(), rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 443       }
 444       num_domainsdone=1;
 445     }
 446     cerr<<num_domainsdone<<" domains were fully parsed, containing "<<g_numRecords<<" records\n";
 447
 448   if(::arg().mustDo("transactions") && g_intransaction) {
 449     if(g_mode != SQLITE)
 450       cout<<"COMMIT WORK;"<<endl;
 451     else
 452       cout<<"COMMIT;"<<endl;
 453   }
 454   return 0;
 455 }
 456 catch(PDNSException &ae) {
 457   cerr<<"\nFatal error: "<<ae.reason<<endl;
 458   return 1;
 459 }
 460 catch(std::exception &e) {
 461   cerr<<"\ndied because of STL error: "<<e.what()<<endl;
 462   return 1;
 463 }
 464 catch(...) {
 465   cerr<<"\ndied because of unknown exception"<<endl;
 466   return 1;
 467 }