modules/remotebackend/httpconnector.cc

   1 /*
   2  * This file is part of PowerDNS or dnsdist.
   3  * Copyright -- PowerDNS.COM B.V. and its contributors
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of version 2 of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * In addition, for the avoidance of any doubt, permission is granted to
  10  * link this program with OpenSSL and to (re)distribute the binaries
  11  * produced as the result of such linking.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21  */
  22 #ifdef HAVE_CONFIG_H
  23 #include "config.h"
  24 #endif
  25 #include "remotebackend.hh"
  26 #include <sys/socket.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include <sstream>
  31 #include "pdns/lock.hh"
  32
  33 #ifndef UNIX_PATH_MAX
  34 #define UNIX_PATH_MAX 108
  35 #endif
  36
  37 HTTPConnector::HTTPConnector(std::map<std::string,std::string> options) {
  38     this->d_url = options.find("url")->second;
  39     if (options.find("url-suffix") != options.end()) {
  40       this->d_url_suffix = options.find("url-suffix")->second;
  41     } else {
  42       this->d_url_suffix = "";
  43     }
  44     this->timeout = 2;
  45     this->d_post = false;
  46     this->d_post_json = false;
  47     this->d_socket = NULL;
  48
  49     if (options.find("timeout") != options.end()) {
  50       this->timeout = std::stoi(options.find("timeout")->second)/1000;
  51     }
  52     if (options.find("post") != options.end()) {
  53       std::string val = options.find("post")->second;
  54       if (val == "yes" || val == "true" || val == "on" || val == "1") {
  55         this->d_post = true;
  56       }
  57     }
  58     if (options.find("post_json") != options.end()) {
  59       std::string val = options.find("post_json")->second;
  60       if (val == "yes" || val == "true" || val == "on" || val == "1") {
  61         this->d_post_json = true;
  62       }
  63     }
  64 }
  65
  66 HTTPConnector::~HTTPConnector() {
  67     if (d_socket != NULL)
  68       delete d_socket;
  69 }
  70
  71 void HTTPConnector::addUrlComponent(const Json &parameters, const string& element, std::stringstream& ss) {
  72     std::string sparam;
  73     if (parameters[element] != Json())
  74        ss << "/" << asString(parameters[element]);
  75 }
  76
  77 std::string HTTPConnector::buildMemberListArgs(std::string prefix, const Json& args) {
  78     std::stringstream stream;
  79
  80     for(const auto& pair: args.object_items()) {
  81         if (pair.second.is_bool()) {
  82           stream << (pair.second.bool_value()?"1":"0");
  83         } else if (pair.second.is_null()) {
  84           stream << prefix << "[" << pair.first << "]=";
  85         } else {
  86           stream << prefix << "[" << pair.first << "]=" << this->asString(pair.second);
  87         }
  88         stream << "&";
  89     }
  90
  91     return stream.str().substr(0, stream.str().size()-1); // snip the trailing &
  92 }
  93
  94 // builds our request (near-restful)
  95 void HTTPConnector::restful_requestbuilder(const std::string &method, const Json& parameters, YaHTTP::Request& req)
  96 {
  97     std::stringstream ss;
  98     std::string sparam;
  99     std::string verb;
 100
 101     // special names are qname, name, zonename, kind, others go to headers
 102
 103     ss << d_url;
 104
 105     ss << "/" << method;
 106
 107     // add the url components, if found, in following order.
 108     // id must be first due to the fact that the qname/name can be empty
 109
 110     addUrlComponent(parameters, "id", ss);
 111     addUrlComponent(parameters, "domain_id", ss);
 112     addUrlComponent(parameters, "zonename", ss);
 113     addUrlComponent(parameters, "qname", ss);
 114     addUrlComponent(parameters, "name", ss);
 115     addUrlComponent(parameters, "kind", ss);
 116     addUrlComponent(parameters, "qtype", ss);
 117
 118     // set the correct type of request based on method
 119     if (method == "activateDomainKey" || method == "deactivateDomainKey") {
 120         // create an empty post
 121         req.preparePost();
 122         verb = "POST";
 123     } else if (method == "setTSIGKey") {
 124         req.POST()["algorithm"] = parameters["algorithm"].string_value();
 125         req.POST()["content"] = parameters["content"].string_value();
 126         req.preparePost();
 127         verb = "PATCH";
 128     } else if (method == "deleteTSIGKey") {
 129         verb = "DELETE";
 130     } else if (method == "addDomainKey") {
 131         const Json& param = parameters["key"];
 132         req.POST()["flags"] = asString(param["flags"]);
 133         req.POST()["active"] = (param["active"].bool_value() ? "1" : "0");
 134         req.POST()["content"] = param["content"].string_value();
 135         req.preparePost();
 136         verb = "PUT";
 137     } else if (method == "isMaster") {
 138         addUrlComponent(parameters, "ip", ss);
 139         verb = "GET";
 140     } else if (method == "superMasterBackend") {
 141         std::stringstream ss2;
 142         addUrlComponent(parameters, "ip", ss);
 143         addUrlComponent(parameters, "domain", ss);
 144         // then we need to serialize rrset payload into POST
 145         for(size_t index = 0; index < parameters["nsset"].array_items().size(); index++) {
 146             ss2 << buildMemberListArgs("nsset[" + std::to_string(index) + "]", parameters["nsset"][index]) << "&";
 147         }
 148         req.body = ss2.str().substr(0, ss2.str().size()-1);
 149         req.headers["content-type"] = "application/x-www-form-urlencoded; charset=utf-8";
 150         req.headers["content-length"] = std::to_string(req.body.size());
 151         verb = "POST";
 152     } else if (method == "createSlaveDomain") {
 153         addUrlComponent(parameters, "ip", ss);
 154         addUrlComponent(parameters, "domain", ss);
 155         if (parameters["account"].is_null() == false && parameters["account"].is_string()) {
 156            req.POST()["account"] = parameters["account"].string_value();
 157         }
 158         req.preparePost();
 159         verb = "PUT";
 160     } else if (method == "replaceRRSet") {
 161         std::stringstream ss2;
 162         for(size_t index = 0; index < parameters["rrset"].array_items().size(); index++) {
 163             ss2 << buildMemberListArgs("rrset[" + std::to_string(index) + "]", parameters["rrset"][index]);
 164         }
 165         req.body = ss2.str();
 166         req.headers["content-type"] = "application/x-www-form-urlencoded; charset=utf-8";
 167         req.headers["content-length"] = std::to_string(req.body.size());
 168         verb = "PATCH";
 169     } else if (method == "feedRecord") {
 170         addUrlComponent(parameters, "trxid", ss);
 171         req.body = buildMemberListArgs("rr", parameters["rr"]);
 172         req.headers["content-type"] = "application/x-www-form-urlencoded; charset=utf-8";
 173         req.headers["content-length"] = std::to_string(req.body.size());
 174         verb = "PATCH";
 175     } else if (method == "feedEnts") {
 176         std::stringstream ss2;
 177         addUrlComponent(parameters, "trxid", ss);
 178         for(const auto& param: parameters["nonterm"].array_items()) {
 179           ss2 << "nonterm[]=" << YaHTTP::Utility::encodeURL(param.string_value(), false) << "&";
 180         }
 181         for(const auto& param: parameters["auth"].array_items()) {
 182           ss2 << "auth[]=" << (param["auth"].bool_value()?"1":"0") << "&";
 183         }
 184         req.body = ss2.str().substr(0, ss2.str().size()-1);
 185         req.headers["content-type"] = "application/x-www-form-urlencoded; charset=utf-8";
 186         req.headers["content-length"] = std::to_string(req.body.size());
 187         verb = "PATCH";
 188     } else if (method == "feedEnts3") {
 189         std::stringstream ss2;
 190         addUrlComponent(parameters, "domain", ss);
 191         addUrlComponent(parameters, "trxid", ss);
 192         ss2 << "times=" << parameters["times"].int_value() << "&salt=" << YaHTTP::Utility::encodeURL(parameters["salt"].string_value(), false) << "&narrow=" << (parameters["narrow"].bool_value() ? 1 : 0) << "&";
 193         for(const auto& param: parameters["nonterm"].array_items()) {
 194           ss2 << "nonterm[]=" << YaHTTP::Utility::encodeURL(param.string_value(), false) << "&";
 195         }
 196         for(const auto& param: parameters["auth"].array_items()) {
 197           ss2 << "auth[]=" << (param["auth"].bool_value()?"1":"0") << "&";
 198         }
 199         req.body = ss2.str().substr(0, ss2.str().size()-1);
 200         req.headers["content-type"] = "application/x-www-form-urlencoded; charset=utf-8";
 201         req.headers["content-length"] = std::to_string(req.body.size());
 202         verb = "PATCH";
 203     } else if (method == "startTransaction") {
 204         addUrlComponent(parameters, "domain", ss);
 205         addUrlComponent(parameters, "trxid", ss);
 206         req.preparePost();
 207         verb = "POST";
 208     } else if (method == "commitTransaction" || method == "abortTransaction") {
 209         addUrlComponent(parameters, "trxid", ss);
 210         req.preparePost();
 211         verb = "POST";
 212     } else if (method == "calculateSOASerial") {
 213         addUrlComponent(parameters, "domain", ss);
 214         req.body = buildMemberListArgs("sd", parameters["sd"]);
 215         req.headers["content-type"] = "application/x-www-form-urlencoded; charset=utf-8";
 216         req.headers["content-length"] = std::to_string(req.body.size());
 217         verb = "POST";
 218     } else if (method == "setDomainMetadata") {
 219         // copy all metadata values into post
 220         std::stringstream ss2;
 221         // this one has values too
 222         if (parameters["value"].is_array()) {
 223            for(const auto& val: parameters["value"].array_items()) {
 224               ss2 << "value[]=" << YaHTTP::Utility::encodeURL(val.string_value(), false) << "&";
 225            }
 226         }
 227         req.body = ss2.str().substr(0, ss2.str().size()-1);
 228         req.headers["content-type"] = "application/x-www-form-urlencoded; charset=utf-8";
 229         req.headers["content-length"] = std::to_string(req.body.size());
 230         verb = "PATCH";
 231     } else if (method == "removeDomainKey") {
 232         // this one is delete
 233         verb = "DELETE";
 234     } else if (method == "setNotified") {
 235         req.POST()["serial"] = std::to_string(parameters["serial"].number_value());
 236         req.preparePost();
 237         verb = "PATCH";
 238     } else if (method == "directBackendCmd") {
 239         req.POST()["query"] = parameters["query"].string_value();
 240         req.preparePost();
 241         verb = "POST";
 242     } else if (method == "searchRecords" || method == "searchComments") {
 243         req.GET()["pattern"] = parameters["pattern"].string_value();
 244         req.GET()["maxResults"] = std::to_string(parameters["maxResults"].int_value());
 245         verb = "GET";
 246    } else if (method == "getAllDomains") {
 247         req.GET()["includeDisabled"] = (parameters["include_disabled"].bool_value()?"true":"false");
 248         verb = "GET";
 249     } else {
 250         // perform normal get
 251         verb = "GET";
 252     }
 253
 254     // put everything else into headers
 255     for(const auto& pair: parameters.object_items()) {
 256       std::string member = pair.first;
 257       // whitelist header parameters
 258       if ((member == "trxid" ||
 259            member == "local" ||
 260            member == "remote" ||
 261            member == "real-remote" ||
 262            member == "zone-id")) {
 263         std::string hdr = "x-remotebackend-" + member;
 264         req.headers[hdr] = asString(pair.second);
 265       }
 266     };
 267
 268     // finally add suffix and store url
 269     ss << d_url_suffix;
 270
 271     req.setup(verb, ss.str());
 272     req.headers["accept"] = "application/json";
 273 }
 274
 275 void HTTPConnector::post_requestbuilder(const Json& input, YaHTTP::Request& req) {
 276     if (this->d_post_json) {
 277         std::string out = input.dump();
 278         req.setup("POST", d_url);
 279         // simple case, POST JSON into url. nothing fancy.
 280         req.headers["Content-Type"] = "text/javascript; charset=utf-8";
 281         req.headers["Content-Length"] = std::to_string(out.size());
 282         req.headers["accept"] = "application/json";
 283         req.body = out;
 284     } else {
 285         std::stringstream url,content;
 286         // call url/method.suffix
 287         url << d_url << "/" << input["method"].string_value() << d_url_suffix;
 288         req.setup("POST", url.str());
 289         // then build content
 290         req.POST()["parameters"] = input["parameters"].dump();
 291         req.preparePost();
 292         req.headers["accept"] = "application/json";
 293     }
 294 }
 295
 296 int HTTPConnector::send_message(const Json& input) {
 297     int rv,ec,fd;
 298
 299     std::vector<std::string> members;
 300     std::string method;
 301     std::ostringstream out;
 302
 303     // perform request
 304     YaHTTP::Request req;
 305
 306     if (d_post)
 307       post_requestbuilder(input, req);
 308     else
 309       restful_requestbuilder(input["method"].string_value(), input["parameters"], req);
 310
 311     rv = -1;
 312     req.headers["connection"] = "Keep-Alive"; // see if we can streamline requests (not needed, strictly speaking)
 313
 314     out << req;
 315
 316     // try sending with current socket, if it fails retry with new socket
 317     if (this->d_socket != NULL) {
 318       fd = this->d_socket->getHandle();
 319       // there should be no data waiting
 320       if (waitForRWData(fd, true, 0, 1000) < 1) {
 321         try {
 322           d_socket->writenWithTimeout(out.str().c_str(), out.str().size(), timeout);
 323           rv = 1;
 324         } catch (NetworkError& ne) {
 325           g_log<<Logger::Error<<"While writing to HTTP endpoint "<<d_addr.toStringWithPort()<<": "<<ne.what()<<std::endl;
 326         } catch (...) {
 327           g_log<<Logger::Error<<"While writing to HTTP endpoint "<<d_addr.toStringWithPort()<<": exception caught"<<std::endl;
 328         }
 329       }
 330     }
 331
 332     if (rv == 1) return rv;
 333
 334     delete this->d_socket;
 335     this->d_socket = NULL;
 336
 337     if (req.url.protocol == "unix") {
 338       // connect using unix socket
 339     } else {
 340       // connect using tcp
 341       struct addrinfo *gAddr, *gAddrPtr, hints;
 342       std::string sPort = std::to_string(req.url.port);
 343       memset(&hints,0,sizeof hints);
 344       hints.ai_family = AF_UNSPEC;
 345       hints.ai_flags = AI_ADDRCONFIG;
 346       hints.ai_socktype = SOCK_STREAM;
 347       hints.ai_protocol = 6; // tcp
 348       if ((ec = getaddrinfo(req.url.host.c_str(), sPort.c_str(), &hints, &gAddr)) == 0) {
 349         // try to connect to each address.
 350         gAddrPtr = gAddr;
 351
 352         while(gAddrPtr) {
 353           try {
 354             d_socket = new Socket(gAddrPtr->ai_family, gAddrPtr->ai_socktype, gAddrPtr->ai_protocol);
 355             d_addr.setSockaddr(gAddrPtr->ai_addr, gAddrPtr->ai_addrlen);
 356             d_socket->connect(d_addr);
 357             d_socket->setNonBlocking();
 358             d_socket->writenWithTimeout(out.str().c_str(), out.str().size(), timeout);
 359             rv = 1;
 360           } catch (NetworkError& ne) {
 361             g_log<<Logger::Error<<"While writing to HTTP endpoint "<<d_addr.toStringWithPort()<<": "<<ne.what()<<std::endl;
 362           } catch (...) {
 363             g_log<<Logger::Error<<"While writing to HTTP endpoint "<<d_addr.toStringWithPort()<<": exception caught"<<std::endl;
 364           }
 365
 366           if (rv > -1) break;
 367           delete d_socket;
 368           d_socket = NULL;
 369           gAddrPtr = gAddrPtr->ai_next;
 370
 371         }
 372         freeaddrinfo(gAddr);
 373       } else {
 374         g_log<<Logger::Error<<"Unable to resolve " << req.url.host << ": " << gai_strerror(ec) << std::endl;
 375       }
 376     }
 377
 378     return rv;
 379 }
 380
 381 int HTTPConnector::recv_message(Json& output) {
 382     YaHTTP::AsyncResponseLoader arl;
 383     YaHTTP::Response resp;
 384
 385     if (d_socket == NULL ) return -1; // cannot receive :(
 386     char buffer[4096];
 387     int rd = -1;
 388     bool fail = false;
 389     time_t t0;
 390
 391     arl.initialize(&resp);
 392
 393     try {
 394       t0 = time((time_t*)NULL);
 395       while(arl.ready() == false && (labs(time((time_t*)NULL) - t0) <= timeout)) {
 396         rd = d_socket->readWithTimeout(buffer, sizeof(buffer), timeout);
 397         if (rd==0)
 398           throw NetworkError("EOF while reading");
 399         if (rd<0)
 400           throw NetworkError(std::string(strerror(rd)));
 401         arl.feed(std::string(buffer, rd));
 402       }
 403       // timeout occured.
 404       if (arl.ready() == false)
 405         throw NetworkError("timeout");
 406     } catch (NetworkError &ne) {
 407       g_log<<Logger::Error<<"While reading from HTTP endpoint "<<d_addr.toStringWithPort()<<": "<<ne.what()<<std::endl;
 408       delete d_socket;
 409       d_socket = NULL;
 410       fail = true;
 411     } catch (...) {
 412       g_log<<Logger::Error<<"While reading from HTTP endpoint "<<d_addr.toStringWithPort()<<": exception caught"<<std::endl;
 413       delete d_socket;
 414       fail = true;
 415     }
 416
 417     if (fail) {
 418       return -1;
 419     }
 420
 421     arl.finalize();
 422
 423     if (resp.status < 200 || resp.status >= 400) {
 424       // bad.
 425       return -1;
 426     }
 427
 428     int rv = -1;
 429     std::string err;
 430     output = Json::parse(resp.body, err);
 431     if (output != nullptr) return resp.body.size();
 432     g_log<<Logger::Error<<"Cannot parse JSON reply: "<<err<<endl;
 433
 434     return rv;
 435 }