pdns/zone2sql.cc

   1 /*
   2     PowerDNS Versatile Database Driven Nameserver
   3     Copyright (C) 2002 - 2011  PowerDNS.COM BV
   4
   5     This program is free software; you can redistribute it and/or modify
   6     it under the terms of the GNU General Public License version 2
   7     as published by the Free Software Foundation
   8
   9     Additionally, the license of this program contains a special
  10     exception which allows to distribute the program in binary form when
  11     it is linked against OpenSSL.
  12
  13     This program is distributed in the hope that it will be useful,
  14     but WITHOUT ANY WARRANTY; without even the implied warranty of
  15     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16     GNU General Public License for more details.
  17
  18     You should have received a copy of the GNU General Public License
  19     along with this program; if not, write to the Free Software
  20     Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  21 */
  22 /* accepts a named.conf or a zone as parameter and outputs heaps of sql */
  23
  24 #ifdef HAVE_CONFIG_H
  25 #include "config.h"
  26 #endif
  27 #include <unistd.h>
  28 #include <string>
  29 #include <map>
  30
  31 #include <iostream>
  32 #include <stdio.h>
  33 #include "json11.hpp"
  34 #include "namespaces.hh"
  35 #include "dns.hh"
  36 #include "arguments.hh"
  37 #include "bindparserclasses.hh"
  38 #include "statbag.hh"
  39 #include "misc.hh"
  40 #include "dnspacket.hh"
  41 #include "zoneparser-tng.hh"
  42 #include "dnsrecords.hh"
  43 #include <boost/algorithm/string.hpp>
  44 #include <sys/types.h>
  45 #include <sys/stat.h>
  46 #include <unistd.h>
  47
  48
  49
  50 StatBag S;
  51
  52 enum dbmode_t {MYSQL, GORACLE, POSTGRES, SQLITE, MYDNS, ORACLE};
  53 static dbmode_t g_mode;
  54 static bool g_intransaction;
  55 static int g_numRecords;
  56
  57
  58 /* this is an official wart. We don't terminate domains on a . in PowerDNS,
  59    which is fine as it goes, except for encoding the root, it would end up as '',
  60    which leads to ambiguities in the content field. Therefore, if we encounter
  61    the root as a . in a BIND zone, we leave it as a ., and don't replace it by
  62    an empty string. Back in 1999 we made the wrong choice. */
  63
  64 static string stripDotContent(const string& content)
  65 {
  66   if(boost::ends_with(content, " .") || content==".")
  67     return content;
  68   return stripDot(content);
  69 }
  70
  71 static string sqlstr(const string &name)
  72 {
  73   if(g_mode == SQLITE || g_mode==GORACLE || g_mode==ORACLE)
  74     return "'"+boost::replace_all_copy(name, "'", "''")+"'";
  75
  76   string a;
  77
  78   for(string::const_iterator i=name.begin();i!=name.end();++i) {
  79     if(*i=='\'' || *i=='\\'){
  80       a+='\\';
  81       a+=*i;
  82     }
  83     else
  84       a+=*i;
  85   }
  86   if(g_mode == POSTGRES)
  87     return "E'"+a+"'";
  88   else
  89     return "'"+a+"'";
  90 }
  91
  92 static void startNewTransaction()
  93 {
  94   if(!::arg().mustDo("transactions"))
  95     return;
  96
  97   if(g_intransaction) {
  98     if(g_mode==POSTGRES) {
  99       cout<<"COMMIT WORK;"<<endl;
 100     }
 101     else if(g_mode == MYSQL || g_mode == SQLITE || g_mode == MYDNS) {
 102       cout<<"COMMIT;"<<endl;
 103     }
 104   }
 105   g_intransaction=1;
 106
 107   if(g_mode == MYSQL || g_mode == MYDNS)
 108     cout<<"BEGIN;"<<endl;
 109   else if (g_mode!=GORACLE && g_mode!=ORACLE)
 110     cout<<"BEGIN TRANSACTION;"<<endl;
 111 }
 112
 113 static void emitDomain(const DNSName& domain, const vector<string> *masters = 0) {
 114   string iDomain = domain.toStringRootDot();
 115   if(!::arg().mustDo("slave")) {
 116     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 117       cout<<"insert into domains (name,type) values ("<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 118     }
 119     else if(g_mode==GORACLE) {
 120       cout<<"insert into domains (id,name,type) values (domains_id_sequence.nextval,"<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 121     }
 122     else if(g_mode==ORACLE) {
 123       cout<<"INSERT INTO Zones (id, name, type) VALUES (zones_id_seq.nextval, "<<sqlstr(toLower(iDomain))<<", 'NATIVE');"<<endl;
 124     }
 125   }
 126   else
 127   {
 128
 129     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 130       string mstrs;
 131       if (masters != 0 && ! masters->empty()) {
 132         for(const string& mstr :  *masters) {
 133           mstrs.append(mstr);
 134           mstrs.append(1, ' ');
 135         }
 136       }
 137       if (mstrs.empty())
 138         cout<<"insert into domains (name,type) values ("<<sqlstr(iDomain)<<",'NATIVE');"<<endl;
 139       else
 140         cout<<"insert into domains (name,type,master) values ("<<sqlstr(iDomain)<<",'SLAVE'"<<", '"<<mstrs<<"');"<<endl;
 141     }
 142     else if (g_mode == GORACLE || g_mode==ORACLE) {
 143       cerr<<"Slave import mode not supported with oracle."<<endl;
 144     }
 145   }
 146 }
 147
 148 bool g_doJSONComments;
 149 static void emitRecord(const DNSName& zoneName, const DNSName &DNSqname, const string &qtype, const string &ocontent, int ttl, const string& comment="")
 150 {
 151   string qname = DNSqname.toStringRootDot();
 152   string zname = zoneName.toStringRootDot();
 153   int prio=0;
 154   int disabled=0;
 155   string recordcomment;
 156
 157   if(g_doJSONComments & !comment.empty()) {
 158     string::size_type pos = comment.find("json={");
 159     if(pos!=string::npos) {
 160       string json = comment.substr(pos+5);
 161       string err;
 162       auto document = json11::Json::parse(json, err);
 163       if(document.is_null())
 164         throw runtime_error("Could not parse JSON '"+json+"': " + err);
 165
 166       disabled=document["disabled"].bool_value();
 167       recordcomment=document["comment"].string_value();
 168     }
 169   }
 170
 171   g_numRecords++;
 172   string content(ocontent);
 173
 174   if(qtype == "NSEC" || qtype == "NSEC3")
 175     return; // NSECs do not go in the database
 176
 177   if((qtype == "MX" || qtype == "SRV") && g_mode!=ORACLE) {
 178     prio=pdns_stou(content);
 179
 180     string::size_type pos = content.find_first_not_of("0123456789");
 181     if(pos != string::npos)
 182       boost::erase_head(content, pos);
 183     trim_left(content);
 184   }
 185
 186   bool auth = true;
 187   if(qtype == "NS" && !pdns_iequals(qname, zname)) {
 188     auth=false;
 189   }
 190
 191   if(g_mode==MYSQL || g_mode==SQLITE) {
 192     cout<<"insert into records (domain_id, name, type,content,ttl,prio,disabled) select id ,"<<
 193       sqlstr(toLower(qname))<<", "<<
 194       sqlstr(qtype)<<", "<<
 195       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<disabled<<
 196       " from domains where name="<<toLower(sqlstr(zname))<<";\n";
 197
 198     if(!recordcomment.empty()) {
 199       cout<<"insert into comments (domain_id,name,type,modified_at, comment) select id, "<<toLower(sqlstr(stripDot(qname)))<<", "<<sqlstr(qtype)<<", "<<time(0)<<", "<<sqlstr(recordcomment)<<" from domains where name="<<toLower(sqlstr(zname))<<";\n";
 200     }
 201   }
 202   else if(g_mode==POSTGRES) {
 203     cout<<"insert into records (domain_id, name, ordername, auth, type,content,ttl,prio,disabled) select id ,"<<
 204       sqlstr(toLower(qname))<<", "<<
 205       sqlstr(toLower(labelReverse(makeRelative(qname, zname))))<<", '"<< (auth  ? 't' : 'f') <<"', "<<
 206       sqlstr(qtype)<<", "<<
 207       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", '"<<(disabled ? 't': 'f') <<
 208       "' from domains where name="<<toLower(sqlstr(zname))<<";\n";
 209   }
 210   else if(g_mode==GORACLE) {
 211     cout<<"insert into Records (id, domain_id, name, type, content, ttl, prio, disabled) select RECORDS_ID_SEQUENCE.nextval,id ,"<<
 212       sqlstr(toLower(qname))<<", "<<
 213       sqlstr(qtype)<<", "<<
 214       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<disabled<<
 215       " from Domains where name="<<toLower(sqlstr(zname))<<";\n";
 216   }
 217   else if(g_mode==ORACLE) {
 218     cout<<"INSERT INTO Records (id, zone_id, fqdn, ttl, type, content) SELECT records_id_seq.nextval, id, "<<
 219       sqlstr(toLower(qname))<<", "<<
 220       ttl<<", "<<sqlstr(qtype)<<", "<<
 221       sqlstr(stripDotContent(content))<<
 222       " FROM Zones WHERE name="<<toLower(sqlstr(zname))<<";"<<endl;
 223   }
 224   else if (g_mode == MYDNS) {
 225     string zoneNameDot = zname + ".";
 226     if (qtype == "A" || qtype == "AAAA" || qtype == "CNAME" || qtype == "HINFO" || qtype == "MX" || qtype == "NAPTR" ||
 227         qtype == "NS" || qtype == "PTR" || qtype == "RP" || qtype == "SRV" || qtype == "TXT")
 228     {
 229       if ((qtype == "MX" || qtype == "NS" || qtype == "SRV" || qtype == "CNAME") && content[content.size()-1] != '.')
 230         content.append(".");
 231       cout<<"INSERT INTO rr(zone, name, type, data, aux, ttl) VALUES("<<
 232       "(SELECT id FROM soa WHERE origin = "<<
 233       sqlstr(toLower(zoneNameDot))<<"), "<<
 234       sqlstr(toLower(DNSqname.toString()))<<", "<<
 235       sqlstr(qtype)<<", "<<sqlstr(content)<<", "<<prio<<", "<<ttl<<");\n";
 236     }
 237     else if (qtype == "SOA") {
 238       //pdns CONTENT = ns1.wtest.com. ahu.example.com. 2005092501 28800 7200 604800 86400
 239       vector<string> parts;
 240       stringtok(parts, content);
 241
 242       cout<<"INSERT INTO soa(origin, ns, mbox, serial, refresh, retry, expire, minimum, ttl) VALUES("<<
 243       sqlstr(toLower(zoneNameDot))<<", "<<sqlstr(parts[0])<<", "<<sqlstr(parts[1])<<", "<<pdns_stou(parts[2])<<", "<<
 244       pdns_stou(parts[3])<<", "<<pdns_stou(parts[4])<<", "<<pdns_stou(parts[5])<<", "<<pdns_stou(parts[6])<<", "<<ttl<<");\n";
 245     }
 246     else
 247     {
 248       cerr<<"Record type "<<qtype<<" is not supported."<<endl;
 249     }
 250   }
 251 }
 252
 253
 254 /* 2 modes of operation, either --named or --zone (the latter needs $ORIGIN)
 255    2 further modes: --mysql, --goracle or --oracle
 256 */
 257
 258 ArgvMap &arg()
 259 {
 260   static ArgvMap theArg;
 261   return theArg;
 262 }
 263
 264
 265 int main(int argc, char **argv)
 266 try
 267 {
 268     reportAllTypes();
 269 #if __GNUC__ >= 3
 270     std::ios_base::sync_with_stdio(false);
 271 #endif
 272
 273     ::arg().setSwitch("gpgsql","Output in format suitable for default gpgsqlbackend")="no";
 274     ::arg().setSwitch("gmysql","Output in format suitable for default gmysqlbackend")="no";
 275     ::arg().setSwitch("mydns","Output in format suitable for default mydnsbackend")="no";
 276     ::arg().setSwitch("goracle","Output in format suitable for the goraclebackend")="no";
 277     ::arg().setSwitch("oracle","Output in format suitable for the oraclebackend")="no";
 278     ::arg().setSwitch("gsqlite","Output in format suitable for default gsqlitebackend")="no";
 279     ::arg().setSwitch("verbose","Verbose comments on operation")="no";
 280     ::arg().setSwitch("slave","Keep BIND slaves as slaves. Only works with named-conf.")="no";
 281     ::arg().setSwitch("json-comments","Parse json={} field for disabled & comments")="no";
 282     ::arg().setSwitch("transactions","If target SQL supports it, use transactions")="no";
 283     ::arg().setSwitch("on-error-resume-next","Continue after errors")="no";
 284     ::arg().setSwitch("filter-duplicate-soa","Filter second SOA in zone")="yes";
 285     ::arg().set("zone","Zonefile to parse")="";
 286     ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")="";
 287     ::arg().set("named-conf","Bind 8/9 named.conf to parse")="";
 288
 289     ::arg().set("soa-minimum-ttl","Do not change")="0";
 290     ::arg().set("soa-refresh-default","Do not change")="0";
 291     ::arg().set("soa-retry-default","Do not change")="0";
 292     ::arg().set("soa-expire-default","Do not change")="0";
 293
 294     ::arg().setCmd("help","Provide a helpful message");
 295
 296     S.declare("logmessages");
 297
 298     string namedfile="";
 299     string zonefile="";
 300
 301     ::arg().parse(argc, argv);
 302
 303     if(::arg().mustDo("help")) {
 304       cout<<"syntax:"<<endl<<endl;
 305       cout<<::arg().helpstring()<<endl;
 306       exit(0);
 307     }
 308
 309     if(argc<2) {
 310       cerr<<"syntax:"<<endl<<endl;
 311       cerr<<::arg().helpstring()<<endl;
 312       exit(1);
 313     }
 314
 315     bool filterDupSOA = ::arg().mustDo("filter-duplicate-soa");
 316
 317     g_doJSONComments=::arg().mustDo("json-comments");
 318
 319     if(::arg().mustDo("gmysql"))
 320       g_mode=MYSQL;
 321     else if(::arg().mustDo("gpgsql"))
 322       g_mode=POSTGRES;
 323     else if(::arg().mustDo("gsqlite"))
 324       g_mode=SQLITE;
 325     else if(::arg().mustDo("goracle"))
 326       g_mode=GORACLE;
 327     else if(::arg().mustDo("oracle"))
 328       g_mode=ORACLE;
 329     else if(::arg().mustDo("mydns"))
 330       g_mode=MYDNS;
 331     else {
 332       cerr<<"Unknown SQL mode!\n\n";
 333       cerr<<"syntax:"<<endl<<endl;
 334       cerr<<::arg().helpstring()<<endl;
 335       exit(1);
 336     }
 337
 338     if((g_mode==GORACLE || g_mode==ORACLE) && !::arg().mustDo("transactions"))
 339       cout<<"set autocommit on;"<<endl;
 340
 341     namedfile=::arg()["named-conf"];
 342     zonefile=::arg()["zone"];
 343
 344     int count=0, num_domainsdone=0;
 345
 346     if(zonefile.empty()) {
 347       BindParser BP;
 348       BP.setVerbose(::arg().mustDo("verbose"));
 349       BP.parse(namedfile.empty() ? "./named.conf" : namedfile);
 350
 351       vector<BindDomainInfo> domains=BP.getDomains();
 352       struct stat st;
 353       for(vector<BindDomainInfo>::iterator i=domains.begin(); i!=domains.end(); ++i) {
 354         if(stat(i->filename.c_str(), &st) == 0) {
 355           i->d_dev = st.st_dev;
 356           i->d_ino = st.st_ino;
 357         }
 358       }
 359
 360       sort(domains.begin(), domains.end()); // put stuff in inode order
 361
 362       int numdomains=domains.size();
 363       int tick=numdomains/100;
 364
 365       for(vector<BindDomainInfo>::const_iterator i=domains.begin();
 366           i!=domains.end();
 367           ++i)
 368         {
 369           if(i->type!="master" && i->type!="slave") {
 370             cerr<<" Warning! Skipping '"<<i->type<<"' zone '"<<i->name.toString()<<"'"<<endl;
 371             continue;
 372           }
 373           try {
 374             startNewTransaction();
 375
 376             emitDomain(i->name, &(i->masters));
 377
 378             ZoneParserTNG zpt(i->filename, i->name, BP.getDirectory());
 379             DNSResourceRecord rr;
 380             bool seenSOA=false;
 381             string comment;
 382             while(zpt.get(rr, &comment)) {
 383               if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 384                 continue;
 385               if(rr.qtype.getCode() == QType::SOA)
 386                 seenSOA=true;
 387
 388               emitRecord(i->name, rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 389             }
 390             num_domainsdone++;
 391           }
 392           catch(std::exception &ae) {
 393             if(!::arg().mustDo("on-error-resume-next"))
 394               throw;
 395             else
 396               cerr<<endl<<ae.what()<<endl;
 397           }
 398           catch(PDNSException &ae) {
 399             if(!::arg().mustDo("on-error-resume-next"))
 400               throw;
 401             else
 402               cerr<<ae.reason<<endl;
 403           }
 404
 405
 406           if(!tick || !((count++)%tick))
 407             cerr<<"\r"<<count*100/numdomains<<"% done ("<<i->filename<<")\033\133\113";
 408         }
 409       cerr<<"\r100% done\033\133\113"<<endl;
 410     }
 411     else {
 412       DNSName zonename;
 413       if(!::arg()["zone-name"].empty())
 414         zonename = DNSName(::arg()["zone-name"]);
 415
 416       ZoneParserTNG zpt(zonefile, zonename);
 417       DNSResourceRecord rr;
 418       startNewTransaction();
 419       string comment;
 420       bool seenSOA=false;
 421       bool haveEmittedZone = false;
 422       while(zpt.get(rr, &comment))  {
 423         if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 424           continue;
 425         if(rr.qtype.getCode() == QType::SOA)
 426           seenSOA=true;
 427         if(!haveEmittedZone) {
 428           if(!zpt.getZoneName().empty()){
 429             emitDomain(zpt.getZoneName());
 430             haveEmittedZone = true;
 431           } else {
 432             // We have no zonename yet, don't emit
 433             continue;
 434           }
 435         }
 436
 437         emitRecord(zpt.getZoneName(), rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 438       }
 439       num_domainsdone=1;
 440     }
 441     cerr<<num_domainsdone<<" domains were fully parsed, containing "<<g_numRecords<<" records\n";
 442
 443   if(::arg().mustDo("transactions") && g_intransaction) {
 444     if(g_mode != SQLITE)
 445       cout<<"COMMIT WORK;"<<endl;
 446     else
 447       cout<<"COMMIT;"<<endl;
 448   }
 449   return 0;
 450 }
 451 catch(PDNSException &ae) {
 452   cerr<<"\nFatal error: "<<ae.reason<<endl;
 453   return 1;
 454 }
 455 catch(std::exception &e) {
 456   cerr<<"\ndied because of STL error: "<<e.what()<<endl;
 457   return 1;
 458 }
 459 catch(...) {
 460   cerr<<"\ndied because of unknown exception"<<endl;
 461   return 1;
 462 }