pdns/zone2sql.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22
  23 /* accepts a named.conf or a zone as parameter and outputs heaps of sql */
  24
  25 #ifdef HAVE_CONFIG_H
  26 #include "config.h"
  27 #endif
  28 #include <unistd.h>
  29 #include <string>
  30 #include <map>
  31
  32 #include <iostream>
  33 #include <stdio.h>
  34 #include "json11.hpp"
  35 #include "namespaces.hh"
  36 #include "dns.hh"
  37 #include "arguments.hh"
  38 #include "bindparserclasses.hh"
  39 #include "statbag.hh"
  40 #include "misc.hh"
  41 #include "dnspacket.hh"
  42 #include "zoneparser-tng.hh"
  43 #include "dnsrecords.hh"
  44 #include <boost/algorithm/string.hpp>
  45 #include <sys/types.h>
  46 #include <sys/stat.h>
  47 #include <unistd.h>
  48
  49
  50
  51 StatBag S;
  52
  53 enum dbmode_t {MYSQL, POSTGRES, SQLITE};
  54 static dbmode_t g_mode;
  55 static bool g_intransaction;
  56 static int g_numRecords;
  57
  58
  59 /* this is an official wart. We don't terminate domains on a . in PowerDNS,
  60    which is fine as it goes, except for encoding the root, it would end up as '',
  61    which leads to ambiguities in the content field. Therefore, if we encounter
  62    the root as a . in a BIND zone, we leave it as a ., and don't replace it by
  63    an empty string. Back in 1999 we made the wrong choice. */
  64
  65 static string stripDotContent(const string& content)
  66 {
  67   if(boost::ends_with(content, " .") || content==".")
  68     return content;
  69   return stripDot(content);
  70 }
  71
  72 static string sqlstr(const string &name)
  73 {
  74   if(g_mode == SQLITE)
  75     return "'"+boost::replace_all_copy(name, "'", "''")+"'";
  76
  77   string a;
  78
  79   for(string::const_iterator i=name.begin();i!=name.end();++i) {
  80     if(*i=='\'' || *i=='\\'){
  81       a+='\\';
  82       a+=*i;
  83     }
  84     else
  85       a+=*i;
  86   }
  87   if(g_mode == POSTGRES)
  88     return "E'"+a+"'";
  89   else
  90     return "'"+a+"'";
  91 }
  92
  93 static void startNewTransaction()
  94 {
  95   if(!::arg().mustDo("transactions"))
  96     return;
  97
  98   if(g_intransaction) {
  99     if(g_mode==POSTGRES) {
 100       cout<<"COMMIT WORK;"<<endl;
 101     }
 102     else if(g_mode == MYSQL || g_mode == SQLITE) {
 103       cout<<"COMMIT;"<<endl;
 104     }
 105   }
 106   g_intransaction=1;
 107
 108   if(g_mode == MYSQL)
 109     cout<<"BEGIN;"<<endl;
 110   else
 111     cout<<"BEGIN TRANSACTION;"<<endl;
 112 }
 113
 114 static void emitDomain(const DNSName& domain, const vector<ComboAddress> *masters = 0) {
 115   string iDomain = domain.toStringRootDot();
 116   if(!::arg().mustDo("slave")) {
 117     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 118       cout<<"insert into domains (name,type) values ("<<toLower(sqlstr(iDomain))<<",'NATIVE');"<<endl;
 119     }
 120   }
 121   else
 122   {
 123
 124     if(g_mode==POSTGRES || g_mode==MYSQL || g_mode==SQLITE) {
 125       string mstrs;
 126       if (masters != 0 && ! masters->empty()) {
 127         for(const auto& mstr :  *masters) {
 128           mstrs.append(mstr.toStringWithPortExcept(53));
 129           mstrs.append(1, ' ');
 130         }
 131       }
 132       if (mstrs.empty())
 133         cout<<"insert into domains (name,type) values ("<<sqlstr(iDomain)<<",'NATIVE');"<<endl;
 134       else
 135         cout<<"insert into domains (name,type,master) values ("<<sqlstr(iDomain)<<",'SLAVE'"<<", '"<<mstrs<<"');"<<endl;
 136     }
 137   }
 138 }
 139
 140 bool g_doJSONComments;
 141 static void emitRecord(const DNSName& zoneName, const DNSName &DNSqname, const string &qtype, const string &ocontent, int ttl, const string& comment="")
 142 {
 143   string qname = DNSqname.toStringRootDot();
 144   string zname = zoneName.toStringRootDot();
 145   int prio=0;
 146   int disabled=0;
 147   string recordcomment;
 148
 149   if(g_doJSONComments & !comment.empty()) {
 150     string::size_type pos = comment.find("json={");
 151     if(pos!=string::npos) {
 152       string json = comment.substr(pos+5);
 153       string err;
 154       auto document = json11::Json::parse(json, err);
 155       if(document.is_null())
 156         throw runtime_error("Could not parse JSON '"+json+"': " + err);
 157
 158       disabled=document["disabled"].bool_value();
 159       recordcomment=document["comment"].string_value();
 160     }
 161   }
 162
 163   g_numRecords++;
 164   string content(ocontent);
 165
 166   if(qtype == "NSEC" || qtype == "NSEC3")
 167     return; // NSECs do not go in the database
 168
 169   if((qtype == "MX" || qtype == "SRV")) {
 170     prio=pdns_stou(content);
 171
 172     string::size_type pos = content.find_first_not_of("0123456789");
 173     if(pos != string::npos)
 174       boost::erase_head(content, pos);
 175     trim_left(content);
 176   }
 177
 178   bool auth = true;
 179   if(qtype == "NS" && !pdns_iequals(qname, zname)) {
 180     auth=false;
 181   }
 182
 183   if(g_mode==MYSQL || g_mode==SQLITE) {
 184     cout<<"insert into records (domain_id, name, type,content,ttl,prio,disabled) select id ,"<<
 185       sqlstr(toLower(qname))<<", "<<
 186       sqlstr(qtype)<<", "<<
 187       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", "<<disabled<<
 188       " from domains where name="<<toLower(sqlstr(zname))<<";\n";
 189
 190     if(!recordcomment.empty()) {
 191       cout<<"insert into comments (domain_id,name,type,modified_at, comment) select id, "<<toLower(sqlstr(stripDot(qname)))<<", "<<sqlstr(qtype)<<", "<<time(0)<<", "<<sqlstr(recordcomment)<<" from domains where name="<<toLower(sqlstr(zname))<<";\n";
 192     }
 193   }
 194   else if(g_mode==POSTGRES) {
 195     cout<<"insert into records (domain_id, name, ordername, auth, type,content,ttl,prio,disabled) select id ,"<<
 196       sqlstr(toLower(qname))<<", "<<
 197       sqlstr(DNSName(qname).makeRelative(DNSName(zname)).makeLowerCase().labelReverse().toString(" ", false))<<", '"<< (auth  ? 't' : 'f') <<"', "<<
 198       sqlstr(qtype)<<", "<<
 199       sqlstr(stripDotContent(content))<<", "<<ttl<<", "<<prio<<", '"<<(disabled ? 't': 'f') <<
 200       "' from domains where name="<<toLower(sqlstr(zname))<<";\n";
 201   }
 202 }
 203
 204
 205 /* 2 modes of operation, either --named or --zone (the latter needs $ORIGIN)
 206    1 further mode: --mysql
 207 */
 208
 209 ArgvMap &arg()
 210 {
 211   static ArgvMap theArg;
 212   return theArg;
 213 }
 214
 215
 216 int main(int argc, char **argv)
 217 try
 218 {
 219     reportAllTypes();
 220     std::ios_base::sync_with_stdio(false);
 221
 222     ::arg().setSwitch("gpgsql","Output in format suitable for default gpgsqlbackend")="no";
 223     ::arg().setSwitch("gmysql","Output in format suitable for default gmysqlbackend")="no";
 224     ::arg().setSwitch("gsqlite","Output in format suitable for default gsqlitebackend")="no";
 225     ::arg().setSwitch("verbose","Verbose comments on operation")="no";
 226     ::arg().setSwitch("slave","Keep BIND slaves as slaves. Only works with named-conf.")="no";
 227     ::arg().setSwitch("json-comments","Parse json={} field for disabled & comments")="no";
 228     ::arg().setSwitch("transactions","If target SQL supports it, use transactions")="no";
 229     ::arg().setSwitch("on-error-resume-next","Continue after errors")="no";
 230     ::arg().setSwitch("filter-duplicate-soa","Filter second SOA in zone")="yes";
 231     ::arg().set("zone","Zonefile to parse")="";
 232     ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")="";
 233     ::arg().set("named-conf","Bind 8/9 named.conf to parse")="";
 234
 235     ::arg().set("soa-minimum-ttl","Do not change")="0";
 236     ::arg().set("soa-refresh-default","Do not change")="0";
 237     ::arg().set("soa-retry-default","Do not change")="0";
 238     ::arg().set("soa-expire-default","Do not change")="0";
 239     ::arg().set("max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0";
 240
 241     ::arg().setCmd("help","Provide a helpful message");
 242     ::arg().setCmd("version","Print the version");
 243
 244     S.declare("logmessages");
 245
 246     string namedfile="";
 247     string zonefile="";
 248
 249     ::arg().parse(argc, argv);
 250
 251     if(::arg().mustDo("version")) {
 252       cerr<<"zone2sql "<<VERSION<<endl;
 253       exit(0);
 254     }
 255
 256     if(::arg().mustDo("help")) {
 257       cout<<"syntax:"<<endl<<endl;
 258       cout<<::arg().helpstring()<<endl;
 259       exit(0);
 260     }
 261
 262     if(argc<2) {
 263       cerr<<"syntax:"<<endl<<endl;
 264       cerr<<::arg().helpstring()<<endl;
 265       exit(1);
 266     }
 267
 268     bool filterDupSOA = ::arg().mustDo("filter-duplicate-soa");
 269
 270     g_doJSONComments=::arg().mustDo("json-comments");
 271
 272     if(::arg().mustDo("gmysql"))
 273       g_mode=MYSQL;
 274     else if(::arg().mustDo("gpgsql"))
 275       g_mode=POSTGRES;
 276     else if(::arg().mustDo("gsqlite"))
 277       g_mode=SQLITE;
 278     else {
 279       cerr<<"Unknown SQL mode!\n\n";
 280       cerr<<"syntax:"<<endl<<endl;
 281       cerr<<::arg().helpstring()<<endl;
 282       exit(1);
 283     }
 284
 285     namedfile=::arg()["named-conf"];
 286     zonefile=::arg()["zone"];
 287
 288     int count=0, num_domainsdone=0;
 289
 290     if(zonefile.empty()) {
 291       BindParser BP;
 292       BP.setVerbose(::arg().mustDo("verbose"));
 293       BP.parse(namedfile.empty() ? "./named.conf" : namedfile);
 294
 295       vector<BindDomainInfo> domains=BP.getDomains();
 296       struct stat st;
 297       for(vector<BindDomainInfo>::iterator i=domains.begin(); i!=domains.end(); ++i) {
 298         if(stat(i->filename.c_str(), &st) == 0) {
 299           i->d_dev = st.st_dev;
 300           i->d_ino = st.st_ino;
 301         }
 302       }
 303
 304       sort(domains.begin(), domains.end()); // put stuff in inode order
 305
 306       int numdomains=domains.size();
 307       int tick=numdomains/100;
 308
 309       for(vector<BindDomainInfo>::const_iterator i=domains.begin();
 310           i!=domains.end();
 311           ++i)
 312         {
 313           if(i->type!="master" && i->type!="slave") {
 314             cerr<<" Warning! Skipping '"<<i->type<<"' zone '"<<i->name<<"'"<<endl;
 315             continue;
 316           }
 317           try {
 318             startNewTransaction();
 319
 320             emitDomain(i->name, &(i->masters));
 321
 322             ZoneParserTNG zpt(i->filename, i->name, BP.getDirectory());
 323             zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps"));
 324             DNSResourceRecord rr;
 325             bool seenSOA=false;
 326             string comment;
 327             while(zpt.get(rr, &comment)) {
 328               if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 329                 continue;
 330               if(rr.qtype.getCode() == QType::SOA)
 331                 seenSOA=true;
 332
 333               emitRecord(i->name, rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 334             }
 335             num_domainsdone++;
 336           }
 337           catch(std::exception &ae) {
 338             if(!::arg().mustDo("on-error-resume-next"))
 339               throw;
 340             else
 341               cerr<<endl<<ae.what()<<endl;
 342           }
 343           catch(PDNSException &ae) {
 344             if(!::arg().mustDo("on-error-resume-next"))
 345               throw;
 346             else
 347               cerr<<ae.reason<<endl;
 348           }
 349
 350
 351           if(!tick || !((count++)%tick))
 352             cerr<<"\r"<<count*100/numdomains<<"% done ("<<i->filename<<")\033\133\113";
 353         }
 354       cerr<<"\r100% done\033\133\113"<<endl;
 355     }
 356     else {
 357       DNSName zonename;
 358       if(!::arg()["zone-name"].empty())
 359         zonename = DNSName(::arg()["zone-name"]);
 360
 361       ZoneParserTNG zpt(zonefile, zonename);
 362       zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps"));
 363       DNSResourceRecord rr;
 364       startNewTransaction();
 365       string comment;
 366       bool seenSOA=false;
 367       bool haveEmittedZone = false;
 368       while(zpt.get(rr, &comment))  {
 369         if(filterDupSOA && seenSOA && rr.qtype.getCode() == QType::SOA)
 370           continue;
 371         if(rr.qtype.getCode() == QType::SOA)
 372           seenSOA=true;
 373         if(!haveEmittedZone) {
 374           if(!zpt.getZoneName().empty()){
 375             emitDomain(zpt.getZoneName());
 376             haveEmittedZone = true;
 377           } else {
 378             // We have no zonename yet, don't emit
 379             continue;
 380           }
 381         }
 382
 383         emitRecord(zpt.getZoneName(), rr.qname, rr.qtype.getName(), rr.content, rr.ttl, comment);
 384       }
 385       num_domainsdone=1;
 386     }
 387     cerr<<num_domainsdone<<" domains were fully parsed, containing "<<g_numRecords<<" records\n";
 388
 389   if(::arg().mustDo("transactions") && g_intransaction) {
 390     if(g_mode != SQLITE)
 391       cout<<"COMMIT WORK;"<<endl;
 392     else
 393       cout<<"COMMIT;"<<endl;
 394   }
 395   return 0;
 396 }
 397 catch(PDNSException &ae) {
 398   cerr<<"\nFatal error: "<<ae.reason<<endl;
 399   return 1;
 400 }
 401 catch(std::exception &e) {
 402   cerr<<"\ndied because of STL error: "<<e.what()<<endl;
 403   return 1;
 404 }
 405 catch(...) {
 406   cerr<<"\ndied because of unknown exception"<<endl;
 407   return 1;
 408 }