pdns/zone2sql.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22
  23 /* accepts a named.conf or a zone as parameter and outputs heaps of sql */
  24
  25 #ifdef HAVE_CONFIG_H
  26 #include "config.h"
  27 #endif
  28 #include <unistd.h>
  29 #include <string>
  30 #include <map>
  31
  32 #include <iostream>
  33 #include <stdio.h>
  34 #include "json11.hpp"
  35 #include "namespaces.hh"
  36 #include "dns.hh"
  37 #include "arguments.hh"
  38 #include "bindparserclasses.hh"
  39 #include "statbag.hh"
  40 #include "misc.hh"
  41 #include "dnspacket.hh"
  42 #include "zoneparser-tng.hh"
  43 #include "dnsrecords.hh"
  44 #include <boost/algorithm/string.hpp>
  45 #include <sys/types.h>
  46 #include <sys/stat.h>
  47 #include <unistd.h>
  48
  49
  50
  51 StatBag S;
  52
  53 enum dbmode_t {MYSQL, GORACLE, POSTGRES, SQLITE, MYDNS, ORACLE};
  54 static dbmode_t g_mode;
  55 static bool g_intransaction;
  56 static int g_numRecords;
  57
  58
  59 /* this is an official wart. We don't terminate domains on a . in PowerDNS,
  60    which is fine as it goes, except for encoding the root, it would end up as '',
  61    which leads to ambiguities in the content field. Therefore, if we encounter
  62    the root as a . in a BIND zone, we leave it as a ., and don't replace it by
  63    an empty string. Back in 1999 we made the wrong choice. */
  64
  65 static string stripDotContent(const string& content)
  66 {
  67   if(boost::ends_with(content, " .") || content==".")
  68     return content;
  69   return stripDot(content);
  70 }
  71
  72 static string sqlstr(const string &name)
  73 {
  74   if(g_mode == SQLITE || g_mode==GORACLE || g_mode==ORACLE)
  75     return "'"+boost::replace_all_copy(name, "'", "''")+"'";
  76
  77   string a;
  78
  79   for(string::const_iterator i=name.begin();i!=name.end();++i) {
  80     if(*i=='\'' || *i=='\\'){
  81       a+='\\';
  82       a+=*i;
  83     }
  84     else
  85       a+=*i;
  86   }
  87   if(g_mode == POSTGRES)
  88     return "E'"+a+"'";
  89   else
  90     return "'"+a+"'";
  91 }
  92
  93 static void startNewTransaction()
  94 {
  95   if(!::arg().mustDo("transactions"))
  96     return;
  97
  98   if(g_intransaction) {
  99     if(g_mode==POSTGRES) {
 100       cout<<"COMMIT WORK;"<<endl;
 101     }
 102     else if(g_mode == MYSQL || g_mode == SQLITE || g_mode == MYDNS) {
 103       cout<<"COMMIT;"<<endl;
 104     }
 105   }
 106   g_intransaction=1;
 107
 108   if(g_mode == MYSQL || g_mode == MYDNS)
 109     cout<<"BEGIN;"<<endl;
 110   else if (g_mode!=GORACLE && g_mode!=ORACLE)
 111     cout<<"BEGIN TRANSACTION;"<<endl;
 112 }
 113
 114 static void emitDomain(const DNSName& domain, const vector<string> *masters = 0) {
 115   string iDomain = domain.toStringRootDot();
 116   if(!::arg().mustDo("slave")) {
 117     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 118       cout<<"insert into domains (name,type) values ("<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 119     }
 120     else if(g_mode==GORACLE) {
 121       cout<<"insert into domains (id,name,type) values (domains_id_sequence.nextval,"<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 122     }
 123     else if(g_mode==ORACLE) {
 124       cout<<"INSERT INTO Zones (id, name, type) VALUES (zones_id_seq.nextval, "<<sqlstr(toLower(iDomain))<<", 'NATIVE');"<<endl;
 125     }
 126   }
 127   else
 128   {
 129
 130     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 131       string mstrs;
 132       if (masters != 0 && ! masters->empty()) {
 133         for(const string& mstr :  *masters) {
 134           mstrs.append(mstr);
 135           mstrs.append(1, ' ');
 136         }
 137       }
 138       if (mstrs.empty())
 139         cout<<"insert into domains (name,type) values ("<<sqlstr(iDomain)<<",'NATIVE');"<<endl;
 140       else
 141         cout<<"insert into domains (name,type,master) values ("<<sqlstr(iDomain)<<",'SLAVE'"<<", '"<<mstrs<<"');"<<endl;
 142     }
 143     else if (g_mode == GORACLE || g_mode==ORACLE) {
 144       cerr<<"Slave import mode not supported with oracle."<<endl;
 145     }
 146   }
 147 }
 148
 149 bool g_doJSONComments;
 150 static void emitRecord(const DNSName& zoneName, const DNSName &DNSqname, const string &qtype, const string &ocontent, int ttl, const string& comment="")
 151 {
 152   string qname = DNSqname.toStringRootDot();
 153   string zname = zoneName.toStringRootDot();
 154   int prio=0;
 155   int disabled=0;
 156   string recordcomment;
 157
 158   if(g_doJSONComments & !comment.empty()) {
 159     string::size_type pos = comment.find("json={");
 160     if(pos!=string::npos) {
 161       string json = comment.substr(pos+5);
 162       string err;
 163       auto document = json11::Json::parse(json, err);
 164       if(document.is_null())
 165         throw runtime_error("Could not parse JSON '"+json+"': " + err);
 166
 167       disabled=document["disabled"].bool_value();
 168       recordcomment=document["comment"].string_value();
 169     }
 170   }
 171
 172   g_numRecords++;
 173   string content(ocontent);
 174
 175   if(qtype == "NSEC" || qtype == "NSEC3")
 176     return; // NSECs do not go in the database
 177
 178   if((qtype == "MX" || qtype == "SRV") && g_mode!=ORACLE) {
 179     prio=pdns_stou(content);
 180
 181     string::size_type pos = content.find_first_not_of("0123456789");
 182     if(pos != string::npos)
 183       boost::erase_head(content, pos);
 184     trim_left(content);
 185   }
 186
 187   bool auth = true;
 188   if(qtype == "NS" && !pdns_iequals(qname, zname)) {
 189     auth=false;
 190   }
 191
 192   if(g_mode==MYSQL || g_mode==SQLITE) {
 193     cout<<"insert into records (domain_id, name, type,content,ttl,prio,disabled) select id ,"<<
 194       sqlstr(toLower(qname))<<", "<<
 195       sqlstr(qtype)<<", "<<
 196       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<disabled<<
 197       " from domains where name="<<toLower(sqlstr(zname))<<";\n";
 198
 199     if(!recordcomment.empty()) {
 200       cout<<"insert into comments (domain_id,name,type,modified_at, comment) select id, "<<toLower(sqlstr(stripDot(qname)))<<", "<<sqlstr(qtype)<<", "<<time(0)<<", "<<sqlstr(recordcomment)<<" from domains where name="<<toLower(sqlstr(zname))<<";\n";
 201     }
 202   }
 203   else if(g_mode==POSTGRES) {
 204     cout<<"insert into records (domain_id, name, ordername, auth, type,content,ttl,prio,disabled) select id ,"<<
 205       sqlstr(toLower(qname))<<", "<<
 206       sqlstr(DNSName(qname).makeRelative(DNSName(zname)).makeLowerCase().labelReverse().toString(" ", false))<<", '"<< (auth  ? 't' : 'f') <<"', "<<
 207       sqlstr(qtype)<<", "<<
 208       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", '"<<(disabled ? 't': 'f') <<
 209       "' from domains where name="<<toLower(sqlstr(zname))<<";\n";
 210   }
 211   else if(g_mode==GORACLE) {
 212     cout<<"insert into Records (id, domain_id, name, type, content, ttl, prio, disabled) select RECORDS_ID_SEQUENCE.nextval,id ,"<<
 213       sqlstr(toLower(qname))<<", "<<
 214       sqlstr(qtype)<<", "<<
 215       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<disabled<<
 216       " from Domains where name="<<toLower(sqlstr(zname))<<";\n";
 217   }
 218   else if(g_mode==ORACLE) {
 219     cout<<"INSERT INTO Records (id, zone_id, fqdn, ttl, type, content) SELECT records_id_seq.nextval, id, "<<
 220       sqlstr(toLower(qname))<<", "<<
 221       ttl<<", "<<sqlstr(qtype)<<", "<<
 222       sqlstr(stripDotContent(content))<<
 223       " FROM Zones WHERE name="<<toLower(sqlstr(zname))<<";"<<endl;
 224   }
 225   else if (g_mode == MYDNS) {
 226     string zoneNameDot = zname + ".";
 227     if (qtype == "A" || qtype == "AAAA" || qtype == "CNAME" || qtype == "HINFO" || qtype == "MX" || qtype == "NAPTR" ||
 228         qtype == "NS" || qtype == "PTR" || qtype == "RP" || qtype == "SRV" || qtype == "TXT")
 229     {
 230       if ((qtype == "MX" || qtype == "NS" || qtype == "SRV" || qtype == "CNAME") && content[content.size()-1] != '.')
 231         content.append(".");
 232       cout<<"INSERT INTO rr(zone, name, type, data, aux, ttl) VALUES("<<
 233       "(SELECT id FROM soa WHERE origin = "<<
 234       sqlstr(toLower(zoneNameDot))<<"), "<<
 235       sqlstr(toLower(DNSqname.toString()))<<", "<<
 236       sqlstr(qtype)<<", "<<sqlstr(content)<<", "<<prio<<", "<<ttl<<");\n";
 237     }
 238     else if (qtype == "SOA") {
 239       //pdns CONTENT = ns1.wtest.com. ahu.example.com. 2005092501 28800 7200 604800 86400
 240       vector<string> parts;
 241       stringtok(parts, content);
 242
 243       cout<<"INSERT INTO soa(origin, ns, mbox, serial, refresh, retry, expire, minimum, ttl) VALUES("<<
 244       sqlstr(toLower(zoneNameDot))<<", "<<sqlstr(parts[0])<<", "<<sqlstr(parts[1])<<", "<<pdns_stou(parts[2])<<", "<<
 245       pdns_stou(parts[3])<<", "<<pdns_stou(parts[4])<<", "<<pdns_stou(parts[5])<<", "<<pdns_stou(parts[6])<<", "<<ttl<<");\n";
 246     }
 247     else
 248     {
 249       cerr<<"Record type "<<qtype<<" is not supported."<<endl;
 250     }
 251   }
 252 }
 253
 254
 255 /* 2 modes of operation, either --named or --zone (the latter needs $ORIGIN)
 256    2 further modes: --mysql, --goracle or --oracle
 257 */
 258
 259 ArgvMap &arg()
 260 {
 261   static ArgvMap theArg;
 262   return theArg;
 263 }
 264
 265
 266 int main(int argc, char **argv)
 267 try
 268 {
 269     reportAllTypes();
 270 #if __GNUC__ >= 3
 271     std::ios_base::sync_with_stdio(false);
 272 #endif
 273
 274     ::arg().setSwitch("gpgsql","Output in format suitable for default gpgsqlbackend")="no";
 275     ::arg().setSwitch("gmysql","Output in format suitable for default gmysqlbackend")="no";
 276     ::arg().setSwitch("mydns","Output in format suitable for default mydnsbackend")="no";
 277     ::arg().setSwitch("goracle","Output in format suitable for the goraclebackend")="no";
 278     ::arg().setSwitch("oracle","Output in format suitable for the oraclebackend")="no";
 279     ::arg().setSwitch("gsqlite","Output in format suitable for default gsqlitebackend")="no";
 280     ::arg().setSwitch("verbose","Verbose comments on operation")="no";
 281     ::arg().setSwitch("slave","Keep BIND slaves as slaves. Only works with named-conf.")="no";
 282     ::arg().setSwitch("json-comments","Parse json={} field for disabled & comments")="no";
 283     ::arg().setSwitch("transactions","If target SQL supports it, use transactions")="no";
 284     ::arg().setSwitch("on-error-resume-next","Continue after errors")="no";
 285     ::arg().setSwitch("filter-duplicate-soa","Filter second SOA in zone")="yes";
 286     ::arg().set("zone","Zonefile to parse")="";
 287     ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")="";
 288     ::arg().set("named-conf","Bind 8/9 named.conf to parse")="";
 289
 290     ::arg().set("soa-minimum-ttl","Do not change")="0";
 291     ::arg().set("soa-refresh-default","Do not change")="0";
 292     ::arg().set("soa-retry-default","Do not change")="0";
 293     ::arg().set("soa-expire-default","Do not change")="0";
 294
 295     ::arg().setCmd("help","Provide a helpful message");
 296     ::arg().setCmd("version","Print the version");
 297
 298     S.declare("logmessages");
 299
 300     string namedfile="";
 301     string zonefile="";
 302
 303     ::arg().parse(argc, argv);
 304
 305     if(::arg().mustDo("version")) {
 306       cerr<<"zone2sql "<<VERSION<<endl;
 307       exit(0);
 308     }
 309
 310     if(::arg().mustDo("help")) {
 311       cout<<"syntax:"<<endl<<endl;
 312       cout<<::arg().helpstring()<<endl;
 313       exit(0);
 314     }
 315
 316     if(argc<2) {
 317       cerr<<"syntax:"<<endl<<endl;
 318       cerr<<::arg().helpstring()<<endl;
 319       exit(1);
 320     }
 321
 322     bool filterDupSOA = ::arg().mustDo("filter-duplicate-soa");
 323
 324     g_doJSONComments=::arg().mustDo("json-comments");
 325
 326     if(::arg().mustDo("gmysql"))
 327       g_mode=MYSQL;
 328     else if(::arg().mustDo("gpgsql"))
 329       g_mode=POSTGRES;
 330     else if(::arg().mustDo("gsqlite"))
 331       g_mode=SQLITE;
 332     else if(::arg().mustDo("goracle"))
 333       g_mode=GORACLE;
 334     else if(::arg().mustDo("oracle"))
 335       g_mode=ORACLE;
 336     else if(::arg().mustDo("mydns"))
 337       g_mode=MYDNS;
 338     else {
 339       cerr<<"Unknown SQL mode!\n\n";
 340       cerr<<"syntax:"<<endl<<endl;
 341       cerr<<::arg().helpstring()<<endl;
 342       exit(1);
 343     }
 344
 345     if((g_mode==GORACLE || g_mode==ORACLE) && !::arg().mustDo("transactions"))
 346       cout<<"set autocommit on;"<<endl;
 347
 348     namedfile=::arg()["named-conf"];
 349     zonefile=::arg()["zone"];
 350
 351     int count=0, num_domainsdone=0;
 352
 353     if(zonefile.empty()) {
 354       BindParser BP;
 355       BP.setVerbose(::arg().mustDo("verbose"));
 356       BP.parse(namedfile.empty() ? "./named.conf" : namedfile);
 357
 358       vector<BindDomainInfo> domains=BP.getDomains();
 359       struct stat st;
 360       for(vector<BindDomainInfo>::iterator i=domains.begin(); i!=domains.end(); ++i) {
 361         if(stat(i->filename.c_str(), &st) == 0) {
 362           i->d_dev = st.st_dev;
 363           i->d_ino = st.st_ino;
 364         }
 365       }
 366
 367       sort(domains.begin(), domains.end()); // put stuff in inode order
 368
 369       int numdomains=domains.size();
 370       int tick=numdomains/100;
 371
 372       for(vector<BindDomainInfo>::const_iterator i=domains.begin();
 373           i!=domains.end();
 374           ++i)
 375         {
 376           if(i->type!="master" && i->type!="slave") {
 377             cerr<<" Warning! Skipping '"<<i->type<<"' zone '"<<i->name<<"'"<<endl;
 378             continue;
 379           }
 380           try {
 381             startNewTransaction();
 382
 383             emitDomain(i->name, &(i->masters));
 384
 385             ZoneParserTNG zpt(i->filename, i->name, BP.getDirectory());
 386             DNSResourceRecord rr;
 387             bool seenSOA=false;
 388             string comment;
 389             while(zpt.get(rr, &comment)) {
 390               if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 391                 continue;
 392               if(rr.qtype.getCode() == QType::SOA)
 393                 seenSOA=true;
 394
 395               emitRecord(i->name, rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 396             }
 397             num_domainsdone++;
 398           }
 399           catch(std::exception &ae) {
 400             if(!::arg().mustDo("on-error-resume-next"))
 401               throw;
 402             else
 403               cerr<<endl<<ae.what()<<endl;
 404           }
 405           catch(PDNSException &ae) {
 406             if(!::arg().mustDo("on-error-resume-next"))
 407               throw;
 408             else
 409               cerr<<ae.reason<<endl;
 410           }
 411
 412
 413           if(!tick || !((count++)%tick))
 414             cerr<<"\r"<<count*100/numdomains<<"% done ("<<i->filename<<")\033\133\113";
 415         }
 416       cerr<<"\r100% done\033\133\113"<<endl;
 417     }
 418     else {
 419       DNSName zonename;
 420       if(!::arg()["zone-name"].empty())
 421         zonename = DNSName(::arg()["zone-name"]);
 422
 423       ZoneParserTNG zpt(zonefile, zonename);
 424       DNSResourceRecord rr;
 425       startNewTransaction();
 426       string comment;
 427       bool seenSOA=false;
 428       bool haveEmittedZone = false;
 429       while(zpt.get(rr, &comment))  {
 430         if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 431           continue;
 432         if(rr.qtype.getCode() == QType::SOA)
 433           seenSOA=true;
 434         if(!haveEmittedZone) {
 435           if(!zpt.getZoneName().empty()){
 436             emitDomain(zpt.getZoneName());
 437             haveEmittedZone = true;
 438           } else {
 439             // We have no zonename yet, don't emit
 440             continue;
 441           }
 442         }
 443
 444         emitRecord(zpt.getZoneName(), rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 445       }
 446       num_domainsdone=1;
 447     }
 448     cerr<<num_domainsdone<<" domains were fully parsed, containing "<<g_numRecords<<" records\n";
 449
 450   if(::arg().mustDo("transactions") && g_intransaction) {
 451     if(g_mode != SQLITE)
 452       cout<<"COMMIT WORK;"<<endl;
 453     else
 454       cout<<"COMMIT;"<<endl;
 455   }
 456   return 0;
 457 }
 458 catch(PDNSException &ae) {
 459   cerr<<"\nFatal error: "<<ae.reason<<endl;
 460   return 1;
 461 }
 462 catch(std::exception &e) {
 463   cerr<<"\ndied because of STL error: "<<e.what()<<endl;
 464   return 1;
 465 }
 466 catch(...) {
 467   cerr<<"\ndied because of unknown exception"<<endl;
 468   return 1;
 469 }