pdns/dnsdistdist/doh.cc

   1 #include "config.h"
   2 #include "doh.hh"
   3
   4 #ifdef HAVE_DNS_OVER_HTTPS
   5 #define H2O_USE_EPOLL 1
   6
   7 #include <errno.h>
   8 #include <iostream>
   9 #include <thread>
  10
  11 #include <boost/algorithm/string.hpp>
  12 #include <h2o.h>
  13 //#include <h2o/http1.h>
  14 #include <h2o/http2.h>
  15
  16 #include <openssl/err.h>
  17 #include <openssl/ssl.h>
  18
  19 #include "base64.hh"
  20 #include "dnsname.hh"
  21 #undef CERT
  22 #include "dnsdist.hh"
  23 #include "misc.hh"
  24 #include "dns.hh"
  25 #include "dolog.hh"
  26 #include "dnsdist-ecs.hh"
  27 #include "dnsdist-rules.hh"
  28 #include "dnsdist-xpf.hh"
  29 #include "libssl.hh"
  30 #include "threadname.hh"
  31
  32 using namespace std;
  33
  34 /* So, how does this work. We use h2o for our http2 and TLS needs.
  35    If the operator has configured multiple IP addresses to listen on,
  36    we launch multiple h2o listener threads. We can hook in to multiple
  37    URLs though on the same IP. There is no SNI yet (I think).
  38
  39    h2o is event driven, so we get callbacks if a new DNS query arrived.
  40    When it does, we do some minimal parsing on it, and send it on to the
  41    dnsdist worker thread which we also launched.
  42
  43    This dnsdist worker thread injects the query into the normal dnsdist flow
  44    (as a datagram over a socketpair). The response also goes back over a
  45    (different) socketpair, where we pick it up and deliver it back to h2o.
  46
  47    For coordination, we use the h2o socket multiplexer, which is sensitive to our
  48    socketpair too.
  49 */
  50
  51 /* h2o notes.
  52    Paths and parameters etc just *happen* to be null-terminated in HTTP2.
  53    They are not in HTTP1. So you MUST use the length field!
  54 */
  55
  56 /* 'Intermediate' compatibility from https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29 */
  57 #define DOH_DEFAULT_CIPHERS "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS"
  58
  59 class DOHAcceptContext
  60 {
  61 public:
  62   DOHAcceptContext()
  63   {
  64     memset(&d_h2o_accept_ctx, 0, sizeof(d_h2o_accept_ctx));
  65   }
  66   DOHAcceptContext(const DOHAcceptContext&) = delete;
  67   DOHAcceptContext& operator=(const DOHAcceptContext&) = delete;
  68
  69   h2o_accept_ctx_t* get()
  70   {
  71     ++d_refcnt;
  72     return &d_h2o_accept_ctx;
  73   }
  74
  75   void release()
  76   {
  77     --d_refcnt;
  78     if (d_refcnt == 0) {
  79       SSL_CTX_free(d_h2o_accept_ctx.ssl_ctx);
  80       d_h2o_accept_ctx.ssl_ctx = nullptr;
  81       delete this;
  82     }
  83   }
  84
  85 private:
  86   h2o_accept_ctx_t d_h2o_accept_ctx;
  87   std::atomic<uint64_t> d_refcnt{1};
  88 };
  89
  90 // we create one of these per thread, and pass around a pointer to it
  91 // through the bowels of h2o
  92 struct DOHServerConfig
  93 {
  94   DOHServerConfig(uint32_t idleTimeout): accept_ctx(new DOHAcceptContext)
  95   {
  96     if(socketpair(AF_LOCAL, SOCK_DGRAM, 0, dohquerypair) < 0) {
  97       unixDie("Creating a socket pair for DNS over HTTPS");
  98     }
  99
 100     if (socketpair(AF_LOCAL, SOCK_DGRAM, 0, dohresponsepair) < 0) {
 101       close(dohquerypair[0]);
 102       close(dohquerypair[1]);
 103       unixDie("Creating a socket pair for DNS over HTTPS");
 104     }
 105
 106     h2o_config_init(&h2o_config);
 107     h2o_config.http2.idle_timeout = idleTimeout * 1000;
 108   }
 109   DOHServerConfig(const DOHServerConfig&) = delete;
 110   DOHServerConfig& operator=(const DOHServerConfig&) = delete;
 111
 112   ~DOHServerConfig()
 113   {
 114     if (accept_ctx) {
 115       accept_ctx->release();
 116     }
 117   }
 118
 119   LocalHolders holders;
 120   h2o_globalconf_t h2o_config;
 121   h2o_context_t h2o_ctx;
 122   DOHAcceptContext* accept_ctx{nullptr};
 123   ClientState* cs{nullptr};
 124   std::shared_ptr<DOHFrontend> df{nullptr};
 125   int dohquerypair[2]{-1,-1};
 126   int dohresponsepair[2]{-1,-1};
 127 };
 128
 129 void handleDOHTimeout(DOHUnit* oldDU)
 130 {
 131   if (oldDU == nullptr) {
 132     return;
 133   }
 134
 135 /* we are about to erase an existing DU */
 136   oldDU->error = true;
 137   oldDU->status_code = 502;
 138
 139   if (send(oldDU->rsock, &oldDU, sizeof(oldDU), 0) != sizeof(oldDU)) {
 140     delete oldDU;
 141     oldDU = nullptr;
 142   }
 143 }
 144
 145 static void on_socketclose(void *data)
 146 {
 147   DOHAcceptContext* ctx = reinterpret_cast<DOHAcceptContext*>(data);
 148   ctx->release();
 149 }
 150
 151 /* this duplicates way too much from the UDP handler. Sorry.
 152    this function calls 'return -1' to drop a query without sending it
 153    caller should make sure HTTPS thread hears of that
 154 */
 155
 156 static int processDOHQuery(DOHUnit* du)
 157 {
 158   uint16_t queryId = 0;
 159   ComboAddress remote;
 160   try {
 161     if(!du->req) {
 162       // we got closed meanwhile. XXX small race condition here
 163       return -1;
 164     }
 165     remote = du->remote;
 166     DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(du->req->conn->ctx->storage.entries[0].data);
 167     auto& holders = dsc->holders;
 168     ClientState& cs = *dsc->cs;
 169
 170     if (du->query.size() < sizeof(dnsheader)) {
 171       ++g_stats.nonCompliantQueries;
 172       du->status_code = 400;
 173       return -1;
 174     }
 175
 176     ++cs.queries;
 177     ++g_stats.queries;
 178
 179     /* we need an accurate ("real") value for the response and
 180        to store into the IDS, but not for insertion into the
 181        rings for example */
 182     struct timespec queryRealTime;
 183     gettime(&queryRealTime, true);
 184     uint16_t len = du->query.length();
 185     /* allocate a bit more memory to be able to spoof the content,
 186        or to add ECS without allocating a new buffer */
 187     du->query.resize(du->query.size() + 512);
 188     size_t bufferSize = du->query.size();
 189     auto query = const_cast<char*>(du->query.c_str());
 190     struct dnsheader* dh = reinterpret_cast<struct dnsheader*>(query);
 191
 192     if (!checkQueryHeaders(dh)) {
 193       du->status_code = 400;
 194       return -1; // drop
 195     }
 196
 197     uint16_t qtype, qclass;
 198     unsigned int consumed = 0;
 199     DNSName qname(query, len, sizeof(dnsheader), false, &qtype, &qclass, &consumed);
 200     DNSQuestion dq(&qname, qtype, qclass, consumed, &du->dest, &du->remote, dh, bufferSize, len, false, &queryRealTime);
 201     dq.ednsAdded = du->ednsAdded;
 202     dq.du = du;
 203     queryId = ntohs(dh->id);
 204 #ifdef HAVE_H2O_SOCKET_GET_SSL_SERVER_NAME
 205     h2o_socket_t* sock = du->req->conn->callbacks->get_socket(du->req->conn);
 206     const char * sni = h2o_socket_get_ssl_server_name(sock);
 207     if (sni != nullptr) {
 208       dq.sni = sni;
 209     }
 210 #endif /* HAVE_H2O_SOCKET_BET_SSL_SERVER_NAME */
 211
 212     std::shared_ptr<DownstreamState> ss{nullptr};
 213     auto result = processQuery(dq, cs, holders, ss);
 214
 215     if (result == ProcessQueryResult::Drop) {
 216       du->status_code = 403;
 217       return -1;
 218     }
 219
 220     if (result == ProcessQueryResult::SendAnswer) {
 221       du->response = std::string(reinterpret_cast<char*>(dq.dh), dq.len);
 222       send(du->rsock, &du, sizeof(du), 0);
 223       return 0;
 224     }
 225
 226     if (result != ProcessQueryResult::PassToBackend) {
 227       du->status_code = 500;
 228       return -1;
 229     }
 230
 231     if (ss == nullptr) {
 232       du->status_code = 502;
 233       return -1;
 234     }
 235
 236     ComboAddress dest = du->dest;
 237     unsigned int idOffset = (ss->idOffset++) % ss->idStates.size();
 238     IDState* ids = &ss->idStates[idOffset];
 239     ids->age = 0;
 240     DOHUnit* oldDU = nullptr;
 241     if (ids->usageIndicator != -1) {
 242       /* that means that the state was in use, possibly with an allocated
 243          DOHUnit that we will need to handle, but we can't touch it before
 244          confirming that we now own this state */
 245       oldDU = ids->du;
 246     }
 247
 248     /* we atomically replace the value, we now own this state */
 249     int64_t generation = ids->generation++;
 250     int64_t oldUsage = ids->usageIndicator.exchange(generation);
 251     if (oldUsage < 0) {
 252       /* the value was -1, meaning that the state was not in use.
 253          we reset 'oldDU' because it might have still been in use when we read it. */
 254       oldDU = nullptr;
 255       ++ss->outstanding;
 256     }
 257     else {
 258       ++ss->reuseds;
 259       ++g_stats.downstreamTimeouts;
 260       /* we are reusing a state, if there was an existing DOHUnit we need
 261          to handle it because it's about to be overwritten. */
 262       ids->du = nullptr;
 263       handleDOHTimeout(oldDU);
 264     }
 265
 266     ids->origFD = 0;
 267     ids->du = du;
 268
 269     ids->cs = &cs;
 270     ids->origID = dh->id;
 271     setIDStateFromDNSQuestion(*ids, dq, std::move(qname));
 272
 273     /* If we couldn't harvest the real dest addr, still
 274        write down the listening addr since it will be useful
 275        (especially if it's not an 'any' one).
 276        We need to keep track of which one it is since we may
 277        want to use the real but not the listening addr to reply.
 278     */
 279     if (dest.sin4.sin_family != 0) {
 280       ids->origDest = dest;
 281       ids->destHarvested = true;
 282     }
 283     else {
 284       ids->origDest = cs.local;
 285       ids->destHarvested = false;
 286     }
 287
 288     dh->id = idOffset;
 289
 290     int fd = pickBackendSocketForSending(ss);
 291     /* you can't touch du after this line, because it might already have been freed */
 292     ssize_t ret = udpClientSendRequestToBackend(ss, fd, query, dq.len);
 293
 294     if(ret < 0) {
 295       /* we are about to handle the error, make sure that
 296          this pointer is not accessed when the state is cleaned,
 297          but first check that it still belongs to us */
 298       if (ids->usageIndicator.compare_exchange_strong(generation, -1)) {
 299         ids->du = nullptr;
 300         --ss->outstanding;
 301       }
 302       ++ss->sendErrors;
 303       ++g_stats.downstreamSendErrors;
 304       du->status_code = 502;
 305       return -1;
 306     }
 307
 308     vinfolog("Got query for %s|%s from %s (https), relayed to %s", ids->qname.toString(), QType(ids->qtype).getName(), remote.toStringWithPort(), ss->getName());
 309   }
 310   catch(const std::exception& e) {
 311     vinfolog("Got an error in DOH question thread while parsing a query from %s, id %d: %s", remote.toStringWithPort(), queryId, e.what());
 312     du->status_code = 500;
 313     return -1;
 314   }
 315
 316   return 0;
 317 }
 318
 319 static void on_response_ready_cb(struct st_h2o_filter_t *self, h2o_req_t *req, h2o_ostream_t **slot)
 320 {
 321   if (req == nullptr) {
 322     return;
 323   }
 324
 325   DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(req->conn->ctx->storage.entries[0].data);
 326
 327   DOHFrontend::HTTPVersionStats* stats = nullptr;
 328   if (req->version < 0x200) {
 329     /* HTTP 1.x */
 330     stats = &dsc->df->d_http1Stats;
 331   }
 332   else {
 333     /* HTTP 2.0 */
 334     stats = &dsc->df->d_http2Stats;
 335   }
 336
 337   switch (req->res.status) {
 338   case 200:
 339     ++stats->d_nb200Responses;
 340     break;
 341   case 400:
 342     ++stats->d_nb400Responses;
 343     break;
 344   case 403:
 345     ++stats->d_nb403Responses;
 346     break;
 347   case 500:
 348     ++stats->d_nb500Responses;
 349     break;
 350   case 502:
 351     ++stats->d_nb502Responses;
 352     break;
 353   default:
 354     ++stats->d_nbOtherResponses;
 355     break;
 356   }
 357
 358   h2o_setup_next_ostream(req, slot);
 359 }
 360
 361 static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path, int (*on_req)(h2o_handler_t *, h2o_req_t *))
 362 {
 363   h2o_pathconf_t *pathconf = h2o_config_register_path(hostconf, path, 0);
 364   if (pathconf == nullptr) {
 365     return pathconf;
 366   }
 367   h2o_filter_t *filter = h2o_create_filter(pathconf, sizeof(*filter));
 368   if (filter) {
 369     filter->on_setup_ostream = on_response_ready_cb;
 370   }
 371
 372   h2o_handler_t *handler = h2o_create_handler(pathconf, sizeof(*handler));
 373   if (handler != nullptr) {
 374     handler->on_req = on_req;
 375   }
 376
 377   return pathconf;
 378 }
 379
 380 /* this is called by h2o when our request dies.
 381    We use this to signal to the 'du' that this req is no longer alive */
 382 static void on_generator_dispose(void *_self)
 383 {
 384   DOHUnit** du = (DOHUnit**)_self;
 385   if(*du) { // if 0, on_dnsdist cleaned up du already
 386 //    cout << "du "<<(void*)*du<<" containing req "<<(*du)->req<<" got killed"<<endl;
 387     (*du)->req = nullptr;
 388   }
 389 }
 390
 391 static void doh_dispatch_query(DOHServerConfig* dsc, h2o_handler_t* self, h2o_req_t* req, std::string&& query, const ComboAddress& local, const ComboAddress& remote)
 392 {
 393   try {
 394     uint16_t qtype;
 395     DNSName qname(query.c_str(), query.size(), sizeof(dnsheader), false, &qtype);
 396
 397     auto du = std::unique_ptr<DOHUnit>(new DOHUnit);
 398     du->req = req;
 399     du->dest = local;
 400     du->remote = remote;
 401     du->rsock = dsc->dohresponsepair[0];
 402     du->query = std::move(query);
 403     du->qtype = qtype;
 404     du->self = reinterpret_cast<DOHUnit**>(h2o_mem_alloc_shared(&req->pool, sizeof(*self), on_generator_dispose));
 405     auto ptr = du.release();
 406     *(ptr->self) = ptr;
 407     try  {
 408       if(send(dsc->dohquerypair[0], &ptr, sizeof(ptr), 0) != sizeof(ptr)) {
 409         delete ptr;
 410         ptr = nullptr;
 411         h2o_send_error_500(req, "Internal Server Error", "Internal Server Error", 0);
 412       }
 413     }
 414     catch(...) {
 415       delete ptr;
 416     }
 417   }
 418   catch(const std::exception& e) {
 419     vinfolog("Had error parsing DoH DNS packet from %s: %s", remote.toStringWithPort(), e.what());
 420     h2o_send_error_400(req, "Bad Request", "The DNS query could not be parsed", 0);
 421   }
 422 }
 423
 424 /*
 425    For GET, the base64url-encoded payload is in the 'dns' parameter, which might be the first parameter, or not.
 426    For POST, the payload is the payload.
 427  */
 428 static int doh_handler(h2o_handler_t *self, h2o_req_t *req)
 429 try
 430 {
 431   // g_logstream<<(void*)req<<" doh_handler"<<endl;
 432   if(!req->conn->ctx->storage.size) {
 433     return 0; // although we might was well crash on this
 434   }
 435   h2o_socket_t* sock = req->conn->callbacks->get_socket(req->conn);
 436   ComboAddress remote;
 437   ComboAddress local;
 438   h2o_socket_getpeername(sock, reinterpret_cast<struct sockaddr*>(&remote));
 439   h2o_socket_getsockname(sock, reinterpret_cast<struct sockaddr*>(&local));
 440   DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(req->conn->ctx->storage.entries[0].data);
 441
 442   auto& holders = dsc->holders;
 443   if (!holders.acl->match(remote)) {
 444     ++g_stats.aclDrops;
 445     vinfolog("Query from %s (DoH) dropped because of ACL", remote.toStringWithPort());
 446     h2o_send_error_403(req, "Forbidden", "dns query not allowed because of ACL", 0);
 447     return 0;
 448   }
 449
 450   if(auto tlsversion = h2o_socket_get_ssl_protocol_version(sock)) {
 451     if(!strcmp(tlsversion, "TLSv1.0"))
 452       ++dsc->df->d_tls10queries;
 453     else if(!strcmp(tlsversion, "TLSv1.1"))
 454       ++dsc->df->d_tls11queries;
 455     else if(!strcmp(tlsversion, "TLSv1.2"))
 456       ++dsc->df->d_tls12queries;
 457     else if(!strcmp(tlsversion, "TLSv1.3"))
 458       ++dsc->df->d_tls13queries;
 459     else
 460       ++dsc->df->d_tlsUnknownqueries;
 461   }
 462
 463   string path(req->path.base, req->path.len);
 464
 465   if (h2o_memis(req->method.base, req->method.len, H2O_STRLIT("POST"))) {
 466     ++dsc->df->d_postqueries;
 467     if(req->version >= 0x0200)
 468       ++dsc->df->d_http2Stats.d_nbQueries;
 469     else
 470       ++dsc->df->d_http1Stats.d_nbQueries;
 471
 472     std::string query;
 473     query.reserve(req->entity.len + 512);
 474     query.assign(req->entity.base, req->entity.len);
 475     doh_dispatch_query(dsc, self, req, std::move(query), local, remote);
 476   }
 477   else if(req->query_at != SIZE_MAX && (req->path.len - req->query_at > 5)) {
 478     auto pos = path.find("?dns=");
 479     if(pos == string::npos)
 480       pos = path.find("&dns=");
 481     if(pos != string::npos) {
 482       // need to base64url decode this
 483       string sdns(path.substr(pos+5));
 484       boost::replace_all(sdns,"-", "+");
 485       boost::replace_all(sdns,"_", "/");
 486       // re-add padding that may have been missing
 487       switch (sdns.size() % 4) {
 488       case 2:
 489         sdns.append(2, '=');
 490         break;
 491       case 3:
 492         sdns.append(1, '=');
 493         break;
 494       }
 495
 496       string decoded;
 497       /* rough estimate so we hopefully don't need a need allocation later */
 498       decoded.reserve(((sdns.size() * 3) / 4) + 512);
 499       if(B64Decode(sdns, decoded) < 0) {
 500         h2o_send_error_400(req, "Bad Request", "Unable to decode BASE64-URL", 0);
 501         ++dsc->df->d_badrequests;
 502         return 0;
 503       }
 504       else {
 505         ++dsc->df->d_getqueries;
 506         if(req->version >= 0x0200)
 507           ++dsc->df->d_http2Stats.d_nbQueries;
 508         else
 509           ++dsc->df->d_http1Stats.d_nbQueries;
 510
 511         doh_dispatch_query(dsc, self, req, std::move(decoded), local, remote);
 512       }
 513     }
 514     else
 515     {
 516       vinfolog("HTTP request without DNS parameter: %s", req->path.base);
 517       h2o_send_error_400(req, "Bad Request", "Unable to find the DNS parameter", 0);
 518       ++dsc->df->d_badrequests;
 519       return 0;
 520     }
 521   }
 522   else {
 523     h2o_send_error_400(req, "Bad Request", "Unable to parse the request", 0);
 524     ++dsc->df->d_badrequests;
 525   }
 526   return 0;
 527 }
 528 catch(const exception& e)
 529 {
 530   errlog("DOH Handler function failed with error %s", e.what());
 531   return 0;
 532 }
 533
 534 HTTPHeaderRule::HTTPHeaderRule(const std::string& header, const std::string& regex)
 535   :  d_regex(regex)
 536 {
 537   d_header = toLower(header);
 538   d_visual = "http[" + header+ "] ~ " + regex;
 539
 540 }
 541 bool HTTPHeaderRule::matches(const DNSQuestion* dq) const
 542 {
 543   if(!dq->du) {
 544     return false;
 545   }
 546
 547   for (unsigned int i = 0; i != dq->du->req->headers.size; ++i) {
 548     if(std::string(dq->du->req->headers.entries[i].name->base, dq->du->req->headers.entries[i].name->len) == d_header &&
 549        d_regex.match(std::string(dq->du->req->headers.entries[i].value.base, dq->du->req->headers.entries[i].value.len))) {
 550       return true;
 551     }
 552   }
 553   return false;
 554 }
 555
 556 string HTTPHeaderRule::toString() const
 557 {
 558   return d_visual;
 559 }
 560
 561 HTTPPathRule::HTTPPathRule(const std::string& path)
 562   :  d_path(path)
 563 {
 564
 565 }
 566
 567 bool HTTPPathRule::matches(const DNSQuestion* dq) const
 568 {
 569   if(!dq->du) {
 570     return false;
 571   }
 572
 573   if(dq->du->req->query_at == SIZE_MAX) {
 574     return dq->du->req->path.base == d_path;
 575   }
 576   else {
 577     return d_path.compare(0, d_path.size(), dq->du->req->path.base, dq->du->req->query_at) == 0;
 578   }
 579 }
 580
 581 string HTTPPathRule::toString() const
 582 {
 583   return "url path == " + d_path;
 584 }
 585
 586 void dnsdistclient(int qsock, int rsock)
 587 {
 588   setThreadName("dnsdist/doh-cli");
 589
 590   for(;;) {
 591     try {
 592       DOHUnit* du = nullptr;
 593       ssize_t got = recv(qsock, &du, sizeof(du), 0);
 594       if (got < 0) {
 595         warnlog("Error receiving internal DoH query: %s", strerror(errno));
 596         continue;
 597       }
 598       else if (static_cast<size_t>(got) < sizeof(du)) {
 599         continue;
 600       }
 601
 602       // if there was no EDNS, we add it with a large buffer size
 603       // so we can use UDP to talk to the backend.
 604       auto dh = const_cast<struct dnsheader*>(reinterpret_cast<const struct dnsheader*>(du->query.c_str()));
 605
 606       if(!dh->arcount) {
 607         std::string res;
 608         generateOptRR(std::string(), res, 4096, 0, false);
 609
 610         du->query += res;
 611         dh = const_cast<struct dnsheader*>(reinterpret_cast<const struct dnsheader*>(du->query.c_str())); // may have reallocated
 612         dh->arcount = htons(1);
 613         du->ednsAdded = true;
 614       }
 615       else {
 616         // we leave existing EDNS in place
 617       }
 618
 619       if(processDOHQuery(du) < 0) {
 620         du->error = true; // turns our drop into a 500
 621         if(send(du->rsock, &du, sizeof(du), 0) != sizeof(du))
 622           delete du;     // XXX but now what - will h2o time this out for us?
 623       }
 624     }
 625     catch(const std::exception& e) {
 626       errlog("Error while processing query received over DoH: %s", e.what());
 627     }
 628     catch(...) {
 629       errlog("Unspecified error while processing query received over DoH");
 630     }
 631   }
 632 }
 633
 634 // called if h2o finds that dnsdist gave us an answer
 635 static void on_dnsdist(h2o_socket_t *listener, const char *err)
 636 {
 637   DOHUnit *du = nullptr;
 638   DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(listener->data);
 639   ssize_t got = recv(dsc->dohresponsepair[1], &du, sizeof(du), 0);
 640
 641   if (got < 0) {
 642     warnlog("Error reading a DOH internal response: %s", strerror(errno));
 643     return;
 644   }
 645   else if (static_cast<size_t>(got) != sizeof(du)) {
 646     return;
 647   }
 648
 649   if(!du->req) { // it got killed in flight
 650 //    cout << "du "<<(void*)du<<" came back from dnsdist, but it was killed"<<endl;
 651     delete du;
 652     return;
 653   }
 654
 655   *du->self = nullptr; // so we don't clean up again in on_generator_dispose
 656   if (!du->error) {
 657     ++dsc->df->d_validresponses;
 658     du->req->res.status = 200;
 659     du->req->res.reason = "OK";
 660
 661     h2o_add_header(&du->req->pool, &du->req->res.headers, H2O_TOKEN_CONTENT_TYPE, nullptr, H2O_STRLIT("application/dns-message"));
 662
 663     //    struct dnsheader* dh = (struct dnsheader*)du->query.c_str();
 664     //    cout<<"Attempt to send out "<<du->query.size()<<" bytes over https, TC="<<dh->tc<<", RCODE="<<dh->rcode<<", qtype="<<du->qtype<<", req="<<(void*)du->req<<endl;
 665
 666     du->req->res.content_length = du->response.size();
 667     h2o_send_inline(du->req, du->response.c_str(), du->response.size());
 668   }
 669   else {
 670     switch(du->status_code) {
 671     case 400:
 672       h2o_send_error_400(du->req, "Bad Request", "invalid DNS query", 0);
 673       break;
 674     case 403:
 675       h2o_send_error_403(du->req, "Forbidden", "dns query not allowed", 0);
 676       break;
 677     case 502:
 678       h2o_send_error_502(du->req, "Bad Gateway", "no downstream server available", 0);
 679       break;
 680     case 500:
 681       /* fall-through */
 682     default:
 683       h2o_send_error_500(du->req, "Internal Server Error", "Internal Server Error", 0);
 684       break;
 685     }
 686
 687     ++dsc->df->d_errorresponses;
 688   }
 689
 690   delete du;
 691 }
 692
 693 static void on_accept(h2o_socket_t *listener, const char *err)
 694 {
 695   DOHServerConfig* dsc = reinterpret_cast<DOHServerConfig*>(listener->data);
 696   h2o_socket_t *sock = nullptr;
 697
 698   if (err != nullptr) {
 699     return;
 700   }
 701   // do some dnsdist rules here to filter based on IP address
 702   if ((sock = h2o_evloop_socket_accept(listener)) == nullptr) {
 703     return;
 704   }
 705
 706   ComboAddress remote;
 707   h2o_socket_getpeername(sock, reinterpret_cast<struct sockaddr*>(&remote));
 708   //  cout<<"New HTTP accept for client "<<remote.toStringWithPort()<<": "<< listener->data << endl;
 709
 710   sock->data = dsc;
 711   sock->on_close.cb = on_socketclose;
 712   auto accept_ctx = dsc->accept_ctx->get();
 713   sock->on_close.data = dsc->accept_ctx;
 714   ++dsc->df->d_httpconnects;
 715   h2o_accept(accept_ctx, sock);
 716 }
 717
 718 static int create_listener(const ComboAddress& addr, std::shared_ptr<DOHServerConfig>& dsc, int fd)
 719 {
 720   auto sock = h2o_evloop_socket_create(dsc->h2o_ctx.loop, fd, H2O_SOCKET_FLAG_DONT_READ);
 721   sock->data = dsc.get();
 722   h2o_socket_read_start(sock, on_accept);
 723
 724   return 0;
 725 }
 726
 727 static std::unique_ptr<SSL_CTX, void(*)(SSL_CTX*)> getTLSContext(const std::vector<std::pair<std::string, std::string>>& pairs, const std::string& ciphers, const std::string& ciphers13)
 728 {
 729   auto ctx = std::unique_ptr<SSL_CTX, void(*)(SSL_CTX*)>(SSL_CTX_new(SSLv23_server_method()), SSL_CTX_free);
 730
 731   int sslOptions =
 732     SSL_OP_NO_SSLv2 |
 733     SSL_OP_NO_SSLv3 |
 734     SSL_OP_NO_COMPRESSION |
 735     SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION |
 736     SSL_OP_SINGLE_DH_USE |
 737     SSL_OP_SINGLE_ECDH_USE;
 738
 739   SSL_CTX_set_options(ctx.get(), sslOptions);
 740
 741 #ifdef SSL_CTX_set_ecdh_auto
 742   SSL_CTX_set_ecdh_auto(ctx.get(), 1);
 743 #endif
 744
 745   /* load certificate and private key */
 746   for (const auto& pair : pairs) {
 747     if (SSL_CTX_use_certificate_chain_file(ctx.get(), pair.first.c_str()) != 1) {
 748       ERR_print_errors_fp(stderr);
 749       throw std::runtime_error("Failed to setup SSL/TLS for DoH listener, an error occurred while trying to load the DOH server certificate file: " + pair.first);
 750     }
 751     if (SSL_CTX_use_PrivateKey_file(ctx.get(), pair.second.c_str(), SSL_FILETYPE_PEM) != 1) {
 752       ERR_print_errors_fp(stderr);
 753       throw std::runtime_error("Failed to setup SSL/TLS for DoH listener, an error occurred while trying to load the DOH server private key file: " + pair.second);
 754     }
 755   }
 756
 757   if (SSL_CTX_set_cipher_list(ctx.get(), ciphers.empty() == false ? ciphers.c_str() : DOH_DEFAULT_CIPHERS) != 1) {
 758     throw std::runtime_error("Failed to setup SSL/TLS for DoH listener, DOH ciphers could not be set: " + ciphers);
 759   }
 760
 761 #ifdef HAVE_SSL_CTX_SET_CIPHERSUITES
 762   if (!ciphers13.empty() && SSL_CTX_set_ciphersuites(ctx.get(), ciphers13.c_str()) != 1) {
 763     throw std::runtime_error("Failed to setup SSL/TLS for DoH listener, DOH TLS 1.3 ciphers could not be set: " + ciphers13);
 764   }
 765 #endif /* HAVE_SSL_CTX_SET_CIPHERSUITES */
 766
 767   h2o_ssl_register_alpn_protocols(ctx.get(), h2o_http2_alpn_protocols);
 768
 769   return ctx;
 770 }
 771
 772 static void setupAcceptContext(DOHAcceptContext& ctx, DOHServerConfig& dsc, bool setupTLS)
 773 {
 774   auto nativeCtx = ctx.get();
 775   nativeCtx->ctx = &dsc.h2o_ctx;
 776   nativeCtx->hosts = dsc.h2o_config.hosts;
 777   if (setupTLS) {
 778     auto tlsCtx = getTLSContext(dsc.df->d_certKeyPairs,
 779                                 dsc.df->d_ciphers,
 780                                 dsc.df->d_ciphers13);
 781
 782     nativeCtx->ssl_ctx = tlsCtx.release();
 783   }
 784
 785   ctx.release();
 786 }
 787
 788 void DOHFrontend::reloadCertificates()
 789 {
 790   auto newAcceptContext = std::unique_ptr<DOHAcceptContext>(new DOHAcceptContext());
 791   setupAcceptContext(*newAcceptContext, *d_dsc, true);
 792   DOHAcceptContext* oldCtx = d_dsc->accept_ctx;
 793   d_dsc->accept_ctx = newAcceptContext.release();
 794   oldCtx->release();
 795 }
 796
 797 void DOHFrontend::setup()
 798 {
 799   registerOpenSSLUser();
 800
 801   d_dsc = std::make_shared<DOHServerConfig>(d_idleTimeout);
 802
 803   auto tlsCtx = getTLSContext(d_certKeyPairs,
 804                               d_ciphers,
 805                               d_ciphers13);
 806
 807   auto accept_ctx = d_dsc->accept_ctx->get();
 808   accept_ctx->ssl_ctx = tlsCtx.release();
 809   d_dsc->accept_ctx->release();
 810 }
 811
 812 // this is the entrypoint from dnsdist.cc
 813 void dohThread(ClientState* cs)
 814 try
 815 {
 816   std::shared_ptr<DOHFrontend>& df = cs->dohFrontend;
 817   auto& dsc = df->d_dsc;
 818   dsc->cs = cs;
 819   dsc->df = cs->dohFrontend;
 820   dsc->h2o_config.server_name = h2o_iovec_init(df->d_serverTokens.c_str(), df->d_serverTokens.size());
 821
 822
 823   std::thread dnsdistThread(dnsdistclient, dsc->dohquerypair[1], dsc->dohresponsepair[0]);
 824   dnsdistThread.detach(); // gets us better error reporting
 825
 826   setThreadName("dnsdist/doh");
 827   // I wonder if this registers an IP address.. I think it does
 828   // this may mean we need to actually register a site "name" here and not the IP address
 829   h2o_hostconf_t *hostconf = h2o_config_register_host(&dsc->h2o_config, h2o_iovec_init(df->d_local.toString().c_str(), df->d_local.toString().size()), 65535);
 830
 831   for(const auto& url : df->d_urls) {
 832     register_handler(hostconf, url.c_str(), doh_handler);
 833   }
 834
 835   h2o_context_init(&dsc->h2o_ctx, h2o_evloop_create(), &dsc->h2o_config);
 836
 837   // in this complicated way we insert the DOHServerConfig pointer in there
 838   h2o_vector_reserve(nullptr, &dsc->h2o_ctx.storage, 1);
 839   dsc->h2o_ctx.storage.entries[0].data = dsc.get();
 840   ++dsc->h2o_ctx.storage.size;
 841
 842   auto sock = h2o_evloop_socket_create(dsc->h2o_ctx.loop, dsc->dohresponsepair[1], H2O_SOCKET_FLAG_DONT_READ);
 843   sock->data = dsc.get();
 844
 845   // this listens to responses from dnsdist to turn into http responses
 846   h2o_socket_read_start(sock, on_dnsdist);
 847
 848   setupAcceptContext(*dsc->accept_ctx, *dsc, false);
 849
 850   if (create_listener(df->d_local, dsc, cs->tcpFD) != 0) {
 851     throw std::runtime_error("DOH server failed to listen on " + df->d_local.toStringWithPort() + ": " + strerror(errno));
 852   }
 853
 854   bool stop = false;
 855   do {
 856     int result = h2o_evloop_run(dsc->h2o_ctx.loop, INT32_MAX);
 857     if (result == -1) {
 858       if (errno != EINTR) {
 859         errlog("Error in the DoH event loop: %s", strerror(errno));
 860         stop = true;
 861       }
 862     }
 863   }
 864   while (stop == false);
 865
 866 }
 867 catch(const std::exception& e) {
 868   throw runtime_error("DOH thread failed to launch: " + std::string(e.what()));
 869 }
 870 catch(...) {
 871   throw runtime_error("DOH thread failed to launch");
 872 }
 873
 874 #else /* HAVE_DNS_OVER_HTTPS */
 875
 876 void handleDOHTimeout(DOHUnit* oldDU)
 877 {
 878 }
 879
 880 #endif /* HAVE_DNS_OVER_HTTPS */