src/tunnel.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 26    Secure Sockets Layer Proxy */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/CbcPointer.h"
  14 #include "CachePeer.h"
  15 #include "cbdata.h"
  16 #include "client_side.h"
  17 #include "client_side_request.h"
  18 #include "comm.h"
  19 #include "comm/Connection.h"
  20 #include "comm/ConnOpener.h"
  21 #include "comm/Read.h"
  22 #include "comm/Write.h"
  23 #include "errorpage.h"
  24 #include "fd.h"
  25 #include "fde.h"
  26 #include "FwdState.h"
  27 #include "globals.h"
  28 #include "http.h"
  29 #include "http/Stream.h"
  30 #include "HttpRequest.h"
  31 #include "ip/QosConfig.h"
  32 #include "LogTags.h"
  33 #include "MemBuf.h"
  34 #include "neighbors.h"
  35 #include "PeerSelectState.h"
  36 #include "sbuf/SBuf.h"
  37 #include "security/BlindPeerConnector.h"
  38 #include "SquidConfig.h"
  39 #include "SquidTime.h"
  40 #include "StatCounters.h"
  41 #if USE_OPENSSL
  42 #include "ssl/bio.h"
  43 #include "ssl/ServerBump.h"
  44 #endif
  45 #include "tools.h"
  46 #if USE_DELAY_POOLS
  47 #include "DelayId.h"
  48 #endif
  49
  50 #include <climits>
  51 #include <cerrno>
  52
  53 /**
  54  * TunnelStateData is the state engine performing the tasks for
  55  * setup of a TCP tunnel from an existing open client FD to a server
  56  * then shuffling binary data between the resulting FD pair.
  57  */
  58 /*
  59  * TODO 1: implement a read/write API on ConnStateData to send/receive blocks
  60  * of pre-formatted data. Then we can use that as the client side of the tunnel
  61  * instead of re-implementing it here and occasionally getting the ConnStateData
  62  * read/write state wrong.
  63  *
  64  * TODO 2: then convert this into a AsyncJob, possibly a child of 'Server'
  65  */
  66 class TunnelStateData: public PeerSelectionInitiator
  67 {
  68     CBDATA_CHILD(TunnelStateData);
  69
  70 public:
  71     TunnelStateData(ClientHttpRequest *);
  72     virtual ~TunnelStateData();
  73     TunnelStateData(const TunnelStateData &); // do not implement
  74     TunnelStateData &operator =(const TunnelStateData &); // do not implement
  75
  76     class Connection;
  77     static void ReadClient(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data);
  78     static void ReadServer(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data);
  79     static void WriteClientDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data);
  80     static void WriteServerDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data);
  81
  82     /// Starts reading peer response to our CONNECT request.
  83     void readConnectResponse();
  84
  85     /// Called when we may be done handling a CONNECT exchange with the peer.
  86     void connectExchangeCheckpoint();
  87
  88     bool noConnections() const;
  89     char *url;
  90     CbcPointer<ClientHttpRequest> http;
  91     HttpRequest::Pointer request;
  92     AccessLogEntryPointer al;
  93     Comm::ConnectionList serverDestinations;
  94
  95     const char * getHost() const {
  96         return (server.conn != NULL && server.conn->getPeer() ? server.conn->getPeer()->host : request->url.host());
  97     };
  98
  99     /// Whether we are writing a CONNECT request to a peer.
 100     bool waitingForConnectRequest() const { return connectReqWriting; }
 101     /// Whether we are reading a CONNECT response from a peer.
 102     bool waitingForConnectResponse() const { return connectRespBuf; }
 103     /// Whether we are waiting for the CONNECT request/response exchange with the peer.
 104     bool waitingForConnectExchange() const { return waitingForConnectRequest() || waitingForConnectResponse(); }
 105
 106     /// Whether the client sent a CONNECT request to us.
 107     bool clientExpectsConnectResponse() const {
 108         // If we are forcing a tunnel after receiving a client CONNECT, then we
 109         // have already responded to that CONNECT before tunnel.cc started.
 110         if (request && request->flags.forceTunnel)
 111             return false;
 112 #if USE_OPENSSL
 113         // We are bumping and we had already send "OK CONNECTED"
 114         if (http.valid() && http->getConn() && http->getConn()->serverBump() && http->getConn()->serverBump()->step > Ssl::bumpStep1)
 115             return false;
 116 #endif
 117         return !(request != NULL &&
 118                  (request->flags.interceptTproxy || request->flags.intercepted));
 119     }
 120
 121     /// Sends "502 Bad Gateway" error response to the client,
 122     /// if it is waiting for Squid CONNECT response, closing connections.
 123     void informUserOfPeerError(const char *errMsg, size_t);
 124
 125     /// starts connecting to the next hop, either for the first time or while
 126     /// recovering from the previous connect failure
 127     void startConnecting();
 128
 129     void noteConnectFailure(const Comm::ConnectionPointer &conn);
 130
 131     class Connection
 132     {
 133
 134     public:
 135         Connection() : len (0), buf ((char *)xmalloc(SQUID_TCP_SO_RCVBUF)), size_ptr(NULL), delayedLoops(0),
 136             readPending(NULL), readPendingFunc(NULL) {}
 137
 138         ~Connection();
 139
 140         int bytesWanted(int lower=0, int upper = INT_MAX) const;
 141         void bytesIn(int const &);
 142 #if USE_DELAY_POOLS
 143
 144         void setDelayId(DelayId const &);
 145 #endif
 146
 147         void error(int const xerrno);
 148         int debugLevelForError(int const xerrno) const;
 149         void closeIfOpen();
 150         void dataSent (size_t amount);
 151         /// writes 'b' buffer, setting the 'writer' member to 'callback'.
 152         void write(const char *b, int size, AsyncCall::Pointer &callback, FREE * free_func);
 153         int len;
 154         char *buf;
 155         AsyncCall::Pointer writer; ///< pending Comm::Write callback
 156         uint64_t *size_ptr;      /* pointer to size in an ConnStateData for logging */
 157
 158         Comm::ConnectionPointer conn;    ///< The currently connected connection.
 159         uint8_t delayedLoops; ///< how many times a read on this connection has been postponed.
 160
 161         // XXX: make these an AsyncCall when event API can handle them
 162         TunnelStateData *readPending;
 163         EVH *readPendingFunc;
 164     private:
 165 #if USE_DELAY_POOLS
 166
 167         DelayId delayId;
 168 #endif
 169
 170     };
 171
 172     Connection client, server;
 173     int *status_ptr;        ///< pointer for logging HTTP status
 174     LogTags *logTag_ptr;    ///< pointer for logging Squid processing code
 175     MemBuf *connectRespBuf; ///< accumulates peer CONNECT response when we need it
 176     bool connectReqWriting; ///< whether we are writing a CONNECT request to a peer
 177     SBuf preReadClientData;
 178     SBuf preReadServerData;
 179     time_t startTime; ///< object creation time, before any peer selection/connection attempts
 180
 181     void copyRead(Connection &from, IOCB *completion);
 182
 183     /// continue to set up connection to a peer, going async for SSL peers
 184     void connectToPeer();
 185
 186     /* PeerSelectionInitiator API */
 187     virtual void noteDestination(Comm::ConnectionPointer conn) override;
 188     virtual void noteDestinationsEnd(ErrorState *selectionError) override;
 189
 190     void saveError(ErrorState *finalError);
 191     void sendError(ErrorState *finalError, const char *reason);
 192
 193 private:
 194     /// Gives Security::PeerConnector access to Answer in the TunnelStateData callback dialer.
 195     class MyAnswerDialer: public CallDialer, public Security::PeerConnector::CbDialer
 196     {
 197     public:
 198         typedef void (TunnelStateData::*Method)(Security::EncryptorAnswer &);
 199
 200         MyAnswerDialer(Method method, TunnelStateData *tunnel):
 201             method_(method), tunnel_(tunnel), answer_() {}
 202
 203         /* CallDialer API */
 204         virtual bool canDial(AsyncCall &call) { return tunnel_.valid(); }
 205         void dial(AsyncCall &call) { ((&(*tunnel_))->*method_)(answer_); }
 206         virtual void print(std::ostream &os) const {
 207             os << '(' << tunnel_.get() << ", " << answer_ << ')';
 208         }
 209
 210         /* Security::PeerConnector::CbDialer API */
 211         virtual Security::EncryptorAnswer &answer() { return answer_; }
 212
 213     private:
 214         Method method_;
 215         CbcPointer<TunnelStateData> tunnel_;
 216         Security::EncryptorAnswer answer_;
 217     };
 218
 219     /// callback handler after connection setup (including any encryption)
 220     void connectedToPeer(Security::EncryptorAnswer &answer);
 221
 222     /// details of the "last tunneling attempt" failure (if it failed)
 223     ErrorState *savedError = nullptr;
 224
 225 public:
 226     bool keepGoingAfterRead(size_t len, Comm::Flag errcode, int xerrno, Connection &from, Connection &to);
 227     void copy(size_t len, Connection &from, Connection &to, IOCB *);
 228     void handleConnectResponse(const size_t chunkSize);
 229     void readServer(char *buf, size_t len, Comm::Flag errcode, int xerrno);
 230     void readClient(char *buf, size_t len, Comm::Flag errcode, int xerrno);
 231     void writeClientDone(char *buf, size_t len, Comm::Flag flag, int xerrno);
 232     void writeServerDone(char *buf, size_t len, Comm::Flag flag, int xerrno);
 233
 234     static void ReadConnectResponseDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data);
 235     void readConnectResponseDone(char *buf, size_t len, Comm::Flag errcode, int xerrno);
 236     void copyClientBytes();
 237     void copyServerBytes();
 238 };
 239
 240 static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
 241
 242 static CNCB tunnelConnectDone;
 243 static ERCB tunnelErrorComplete;
 244 static CLCB tunnelServerClosed;
 245 static CLCB tunnelClientClosed;
 246 static CTCB tunnelTimeout;
 247 static EVH tunnelDelayedClientRead;
 248 static EVH tunnelDelayedServerRead;
 249 static void tunnelConnected(const Comm::ConnectionPointer &server, void *);
 250 static void tunnelRelayConnectRequest(const Comm::ConnectionPointer &server, void *);
 251
 252 static void
 253 tunnelServerClosed(const CommCloseCbParams &params)
 254 {
 255     TunnelStateData *tunnelState = (TunnelStateData *)params.data;
 256     debugs(26, 3, HERE << tunnelState->server.conn);
 257     tunnelState->server.conn = NULL;
 258     tunnelState->server.writer = NULL;
 259
 260     if (tunnelState->request != NULL)
 261         tunnelState->request->hier.stopPeerClock(false);
 262
 263     if (tunnelState->noConnections()) {
 264         // ConnStateData pipeline should contain the CONNECT we are performing
 265         // but it may be invalid already (bug 4392)
 266         if (tunnelState->http.valid() && tunnelState->http->getConn()) {
 267             auto ctx = tunnelState->http->getConn()->pipeline.front();
 268             if (ctx != nullptr)
 269                 ctx->finished();
 270         }
 271         delete tunnelState;
 272         return;
 273     }
 274
 275     if (!tunnelState->client.writer) {
 276         tunnelState->client.conn->close();
 277         return;
 278     }
 279 }
 280
 281 static void
 282 tunnelClientClosed(const CommCloseCbParams &params)
 283 {
 284     TunnelStateData *tunnelState = (TunnelStateData *)params.data;
 285     debugs(26, 3, HERE << tunnelState->client.conn);
 286     tunnelState->client.conn = NULL;
 287     tunnelState->client.writer = NULL;
 288
 289     if (tunnelState->noConnections()) {
 290         // ConnStateData pipeline should contain the CONNECT we are performing
 291         // but it may be invalid already (bug 4392)
 292         if (tunnelState->http.valid() && tunnelState->http->getConn()) {
 293             auto ctx = tunnelState->http->getConn()->pipeline.front();
 294             if (ctx != nullptr)
 295                 ctx->finished();
 296         }
 297         delete tunnelState;
 298         return;
 299     }
 300
 301     if (!tunnelState->server.writer) {
 302         tunnelState->server.conn->close();
 303         return;
 304     }
 305 }
 306
 307 TunnelStateData::TunnelStateData(ClientHttpRequest *clientRequest) :
 308     connectRespBuf(NULL),
 309     connectReqWriting(false),
 310     startTime(squid_curtime)
 311 {
 312     debugs(26, 3, "TunnelStateData constructed this=" << this);
 313     client.readPendingFunc = &tunnelDelayedClientRead;
 314     server.readPendingFunc = &tunnelDelayedServerRead;
 315
 316     assert(clientRequest);
 317     url = xstrdup(clientRequest->uri);
 318     request = clientRequest->request;
 319     server.size_ptr = &clientRequest->out.size;
 320     client.size_ptr = &clientRequest->al->http.clientRequestSz.payloadData;
 321     status_ptr = &clientRequest->al->http.code;
 322     logTag_ptr = &clientRequest->logType;
 323     al = clientRequest->al;
 324     http = clientRequest;
 325
 326     client.conn = clientRequest->getConn()->clientConnection;
 327     comm_add_close_handler(client.conn->fd, tunnelClientClosed, this);
 328
 329     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 330                                      CommTimeoutCbPtrFun(tunnelTimeout, this));
 331     commSetConnTimeout(client.conn, Config.Timeout.lifetime, timeoutCall);
 332 }
 333
 334 TunnelStateData::~TunnelStateData()
 335 {
 336     debugs(26, 3, "TunnelStateData destructed this=" << this);
 337     assert(noConnections());
 338     xfree(url);
 339     serverDestinations.clear();
 340     delete connectRespBuf;
 341     delete savedError;
 342 }
 343
 344 TunnelStateData::Connection::~Connection()
 345 {
 346     if (readPending)
 347         eventDelete(readPendingFunc, readPending);
 348
 349     safe_free(buf);
 350 }
 351
 352 int
 353 TunnelStateData::Connection::bytesWanted(int lowerbound, int upperbound) const
 354 {
 355 #if USE_DELAY_POOLS
 356     return delayId.bytesWanted(lowerbound, upperbound);
 357 #else
 358
 359     return upperbound;
 360 #endif
 361 }
 362
 363 void
 364 TunnelStateData::Connection::bytesIn(int const &count)
 365 {
 366     debugs(26, 3, HERE << "len=" << len << " + count=" << count);
 367 #if USE_DELAY_POOLS
 368     delayId.bytesIn(count);
 369 #endif
 370
 371     len += count;
 372 }
 373
 374 int
 375 TunnelStateData::Connection::debugLevelForError(int const xerrno) const
 376 {
 377 #ifdef ECONNRESET
 378
 379     if (xerrno == ECONNRESET)
 380         return 2;
 381
 382 #endif
 383
 384     if (ignoreErrno(xerrno))
 385         return 3;
 386
 387     return 1;
 388 }
 389
 390 /* Read from server side and queue it for writing to the client */
 391 void
 392 TunnelStateData::ReadServer(const Comm::ConnectionPointer &c, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data)
 393 {
 394     TunnelStateData *tunnelState = (TunnelStateData *)data;
 395     assert(cbdataReferenceValid(tunnelState));
 396     debugs(26, 3, HERE << c);
 397
 398     tunnelState->readServer(buf, len, errcode, xerrno);
 399 }
 400
 401 void
 402 TunnelStateData::readServer(char *, size_t len, Comm::Flag errcode, int xerrno)
 403 {
 404     debugs(26, 3, HERE << server.conn << ", read " << len << " bytes, err=" << errcode);
 405     server.delayedLoops=0;
 406
 407     /*
 408      * Bail out early on Comm::ERR_CLOSING
 409      * - close handlers will tidy up for us
 410      */
 411
 412     if (errcode == Comm::ERR_CLOSING)
 413         return;
 414
 415     if (len > 0) {
 416         server.bytesIn(len);
 417         statCounter.server.all.kbytes_in += len;
 418         statCounter.server.other.kbytes_in += len;
 419     }
 420
 421     if (keepGoingAfterRead(len, errcode, xerrno, server, client))
 422         copy(len, server, client, WriteClientDone);
 423 }
 424
 425 /// Called when we read [a part of] CONNECT response from the peer
 426 void
 427 TunnelStateData::readConnectResponseDone(char *, size_t len, Comm::Flag errcode, int xerrno)
 428 {
 429     debugs(26, 3, server.conn << ", read " << len << " bytes, err=" << errcode);
 430     assert(waitingForConnectResponse());
 431
 432     if (errcode == Comm::ERR_CLOSING)
 433         return;
 434
 435     if (len > 0) {
 436         connectRespBuf->appended(len);
 437         server.bytesIn(len);
 438         statCounter.server.all.kbytes_in += len;
 439         statCounter.server.other.kbytes_in += len;
 440     }
 441
 442     if (keepGoingAfterRead(len, errcode, xerrno, server, client))
 443         handleConnectResponse(len);
 444 }
 445
 446 void
 447 TunnelStateData::informUserOfPeerError(const char *errMsg, const size_t sz)
 448 {
 449     server.len = 0;
 450
 451     if (logTag_ptr)
 452         *logTag_ptr = LOG_TCP_TUNNEL;
 453
 454     if (!clientExpectsConnectResponse()) {
 455         // closing the connection is the best we can do here
 456         debugs(50, 3, server.conn << " closing on error: " << errMsg);
 457         server.conn->close();
 458         return;
 459     }
 460
 461     // if we have no reply suitable to relay, use 502 Bad Gateway
 462     if (!sz || sz > static_cast<size_t>(connectRespBuf->contentSize()))
 463         return sendError(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw()),
 464                          "peer error without reply");
 465
 466     // if we need to send back the server response. write its headers to the client
 467     server.len = sz;
 468     memcpy(server.buf, connectRespBuf->content(), server.len);
 469     copy(server.len, server, client, TunnelStateData::WriteClientDone);
 470     // then close the server FD to prevent any relayed keep-alive causing CVE-2015-5400
 471     server.closeIfOpen();
 472 }
 473
 474 /* Read from client side and queue it for writing to the server */
 475 void
 476 TunnelStateData::ReadConnectResponseDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data)
 477 {
 478     TunnelStateData *tunnelState = (TunnelStateData *)data;
 479     assert (cbdataReferenceValid (tunnelState));
 480
 481     tunnelState->readConnectResponseDone(buf, len, errcode, xerrno);
 482 }
 483
 484 /// Parses [possibly incomplete] CONNECT response and reacts to it.
 485 /// If the tunnel is being closed or more response data is needed, returns false.
 486 /// Otherwise, the caller should handle the remaining read data, if any.
 487 void
 488 TunnelStateData::handleConnectResponse(const size_t chunkSize)
 489 {
 490     assert(waitingForConnectResponse());
 491
 492     // Ideally, client and server should use MemBuf or better, but current code
 493     // never accumulates more than one read when shoveling data (XXX) so it does
 494     // not need to deal with MemBuf complexity. To keep it simple, we use a
 495     // dedicated MemBuf for accumulating CONNECT responses. TODO: When shoveling
 496     // is optimized, reuse server.buf for CONNEC response accumulation instead.
 497
 498     /* mimic the basic parts of HttpStateData::processReplyHeader() */
 499     HttpReply rep;
 500     Http::StatusCode parseErr = Http::scNone;
 501     const bool eof = !chunkSize;
 502     connectRespBuf->terminate(); // Http::Message::parse requires terminated string
 503     const bool parsed = rep.parse(connectRespBuf->content(), connectRespBuf->contentSize(), eof, &parseErr);
 504     if (!parsed) {
 505         if (parseErr > 0) { // unrecoverable parsing error
 506             informUserOfPeerError("malformed CONNECT response from peer", 0);
 507             return;
 508         }
 509
 510         // need more data
 511         assert(!eof);
 512         assert(!parseErr);
 513
 514         if (!connectRespBuf->hasSpace()) {
 515             informUserOfPeerError("huge CONNECT response from peer", 0);
 516             return;
 517         }
 518
 519         // keep reading
 520         readConnectResponse();
 521         return;
 522     }
 523
 524     // CONNECT response was successfully parsed
 525     *status_ptr = rep.sline.status();
 526
 527     // we need to relay the 401/407 responses when login=PASS(THRU)
 528     const CachePeer *peer = server.conn->getPeer();
 529     const char *pwd = (peer ? peer->login : nullptr);
 530     const bool relay = pwd && (strcmp(pwd, "PASS") == 0 || strcmp(pwd, "PASSTHRU") == 0) &&
 531                        (*status_ptr == Http::scProxyAuthenticationRequired ||
 532                         *status_ptr == Http::scUnauthorized);
 533
 534     // bail if we did not get an HTTP 200 (Connection Established) response
 535     if (rep.sline.status() != Http::scOkay) {
 536         // if we ever decide to reuse the peer connection, we must extract the error response first
 537         informUserOfPeerError("unsupported CONNECT response status code", (relay ? rep.hdr_sz : 0));
 538         return;
 539     }
 540
 541     if (rep.hdr_sz < connectRespBuf->contentSize()) {
 542         // preserve bytes that the server already sent after the CONNECT response
 543         server.len = connectRespBuf->contentSize() - rep.hdr_sz;
 544         memcpy(server.buf, connectRespBuf->content()+rep.hdr_sz, server.len);
 545     } else {
 546         // reset; delay pools were using this field to throttle CONNECT response
 547         server.len = 0;
 548     }
 549
 550     delete connectRespBuf;
 551     connectRespBuf = NULL;
 552     connectExchangeCheckpoint();
 553 }
 554
 555 void
 556 TunnelStateData::Connection::error(int const xerrno)
 557 {
 558     debugs(50, debugLevelForError(xerrno), HERE << conn << ": read/write failure: " << xstrerr(xerrno));
 559
 560     if (!ignoreErrno(xerrno))
 561         conn->close();
 562 }
 563
 564 /* Read from client side and queue it for writing to the server */
 565 void
 566 TunnelStateData::ReadClient(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data)
 567 {
 568     TunnelStateData *tunnelState = (TunnelStateData *)data;
 569     assert (cbdataReferenceValid (tunnelState));
 570
 571     tunnelState->readClient(buf, len, errcode, xerrno);
 572 }
 573
 574 void
 575 TunnelStateData::readClient(char *, size_t len, Comm::Flag errcode, int xerrno)
 576 {
 577     debugs(26, 3, HERE << client.conn << ", read " << len << " bytes, err=" << errcode);
 578     client.delayedLoops=0;
 579
 580     /*
 581      * Bail out early on Comm::ERR_CLOSING
 582      * - close handlers will tidy up for us
 583      */
 584
 585     if (errcode == Comm::ERR_CLOSING)
 586         return;
 587
 588     if (len > 0) {
 589         client.bytesIn(len);
 590         statCounter.client_http.kbytes_in += len;
 591     }
 592
 593     if (keepGoingAfterRead(len, errcode, xerrno, client, server))
 594         copy(len, client, server, WriteServerDone);
 595 }
 596
 597 /// Updates state after reading from client or server.
 598 /// Returns whether the caller should use the data just read.
 599 bool
 600 TunnelStateData::keepGoingAfterRead(size_t len, Comm::Flag errcode, int xerrno, Connection &from, Connection &to)
 601 {
 602     debugs(26, 3, HERE << "from={" << from.conn << "}, to={" << to.conn << "}");
 603
 604     /* I think this is to prevent free-while-in-a-callback behaviour
 605      * - RBC 20030229
 606      * from.conn->close() / to.conn->close() done here trigger close callbacks which may free TunnelStateData
 607      */
 608     const CbcPointer<TunnelStateData> safetyLock(this);
 609
 610     /* Bump the source connection read timeout on any activity */
 611     if (Comm::IsConnOpen(from.conn)) {
 612         AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 613                                          CommTimeoutCbPtrFun(tunnelTimeout, this));
 614         commSetConnTimeout(from.conn, Config.Timeout.read, timeoutCall);
 615     }
 616
 617     /* Bump the dest connection read timeout on any activity */
 618     /* see Bug 3659: tunnels can be weird, with very long one-way transfers */
 619     if (Comm::IsConnOpen(to.conn)) {
 620         AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 621                                          CommTimeoutCbPtrFun(tunnelTimeout, this));
 622         commSetConnTimeout(to.conn, Config.Timeout.read, timeoutCall);
 623     }
 624
 625     if (errcode)
 626         from.error (xerrno);
 627     else if (len == 0 || !Comm::IsConnOpen(to.conn)) {
 628         debugs(26, 3, HERE << "Nothing to write or client gone. Terminate the tunnel.");
 629         from.conn->close();
 630
 631         /* Only close the remote end if we've finished queueing data to it */
 632         if (from.len == 0 && Comm::IsConnOpen(to.conn) ) {
 633             to.conn->close();
 634         }
 635     } else if (cbdataReferenceValid(this)) {
 636         return true;
 637     }
 638
 639     return false;
 640 }
 641
 642 void
 643 TunnelStateData::copy(size_t len, Connection &from, Connection &to, IOCB *completion)
 644 {
 645     debugs(26, 3, HERE << "Schedule Write");
 646     AsyncCall::Pointer call = commCbCall(5,5, "TunnelBlindCopyWriteHandler",
 647                                          CommIoCbPtrFun(completion, this));
 648     to.write(from.buf, len, call, NULL);
 649 }
 650
 651 /* Writes data from the client buffer to the server side */
 652 void
 653 TunnelStateData::WriteServerDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 654 {
 655     TunnelStateData *tunnelState = (TunnelStateData *)data;
 656     assert (cbdataReferenceValid (tunnelState));
 657     tunnelState->server.writer = NULL;
 658
 659     tunnelState->writeServerDone(buf, len, flag, xerrno);
 660 }
 661
 662 void
 663 TunnelStateData::writeServerDone(char *, size_t len, Comm::Flag flag, int xerrno)
 664 {
 665     debugs(26, 3, HERE  << server.conn << ", " << len << " bytes written, flag=" << flag);
 666
 667     /* Error? */
 668     if (flag != Comm::OK) {
 669         if (flag != Comm::ERR_CLOSING) {
 670             debugs(26, 4, HERE << "calling TunnelStateData::server.error(" << xerrno <<")");
 671             server.error(xerrno); // may call comm_close
 672         }
 673         return;
 674     }
 675
 676     /* EOF? */
 677     if (len == 0) {
 678         debugs(26, 4, HERE << "No read input. Closing server connection.");
 679         server.conn->close();
 680         return;
 681     }
 682
 683     /* Valid data */
 684     statCounter.server.all.kbytes_out += len;
 685     statCounter.server.other.kbytes_out += len;
 686     client.dataSent(len);
 687
 688     /* If the other end has closed, so should we */
 689     if (!Comm::IsConnOpen(client.conn)) {
 690         debugs(26, 4, HERE << "Client gone away. Shutting down server connection.");
 691         server.conn->close();
 692         return;
 693     }
 694
 695     const CbcPointer<TunnelStateData> safetyLock(this); /* ??? should be locked by the caller... */
 696
 697     if (cbdataReferenceValid(this))
 698         copyClientBytes();
 699 }
 700
 701 /* Writes data from the server buffer to the client side */
 702 void
 703 TunnelStateData::WriteClientDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 704 {
 705     TunnelStateData *tunnelState = (TunnelStateData *)data;
 706     assert (cbdataReferenceValid (tunnelState));
 707     tunnelState->client.writer = NULL;
 708
 709     tunnelState->writeClientDone(buf, len, flag, xerrno);
 710 }
 711
 712 void
 713 TunnelStateData::Connection::dataSent(size_t amount)
 714 {
 715     debugs(26, 3, HERE << "len=" << len << " - amount=" << amount);
 716     assert(amount == (size_t)len);
 717     len =0;
 718     /* increment total object size */
 719
 720     if (size_ptr)
 721         *size_ptr += amount;
 722 }
 723
 724 void
 725 TunnelStateData::Connection::write(const char *b, int size, AsyncCall::Pointer &callback, FREE * free_func)
 726 {
 727     writer = callback;
 728     Comm::Write(conn, b, size, callback, free_func);
 729 }
 730
 731 void
 732 TunnelStateData::writeClientDone(char *, size_t len, Comm::Flag flag, int xerrno)
 733 {
 734     debugs(26, 3, HERE << client.conn << ", " << len << " bytes written, flag=" << flag);
 735
 736     /* Error? */
 737     if (flag != Comm::OK) {
 738         if (flag != Comm::ERR_CLOSING) {
 739             debugs(26, 4, HERE << "Closing client connection due to comm flags.");
 740             client.error(xerrno); // may call comm_close
 741         }
 742         return;
 743     }
 744
 745     /* EOF? */
 746     if (len == 0) {
 747         debugs(26, 4, HERE << "Closing client connection due to 0 byte read.");
 748         client.conn->close();
 749         return;
 750     }
 751
 752     /* Valid data */
 753     statCounter.client_http.kbytes_out += len;
 754     server.dataSent(len);
 755
 756     /* If the other end has closed, so should we */
 757     if (!Comm::IsConnOpen(server.conn)) {
 758         debugs(26, 4, HERE << "Server has gone away. Terminating client connection.");
 759         client.conn->close();
 760         return;
 761     }
 762
 763     CbcPointer<TunnelStateData> safetyLock(this);   /* ??? should be locked by the caller... */
 764
 765     if (cbdataReferenceValid(this))
 766         copyServerBytes();
 767 }
 768
 769 static void
 770 tunnelTimeout(const CommTimeoutCbParams &io)
 771 {
 772     TunnelStateData *tunnelState = static_cast<TunnelStateData *>(io.data);
 773     debugs(26, 3, HERE << io.conn);
 774     /* Temporary lock to protect our own feets (comm_close -> tunnelClientClosed -> Free) */
 775     CbcPointer<TunnelStateData> safetyLock(tunnelState);
 776
 777     tunnelState->client.closeIfOpen();
 778     tunnelState->server.closeIfOpen();
 779 }
 780
 781 void
 782 TunnelStateData::Connection::closeIfOpen()
 783 {
 784     if (Comm::IsConnOpen(conn))
 785         conn->close();
 786 }
 787
 788 static void
 789 tunnelDelayedClientRead(void *data)
 790 {
 791     if (!data)
 792         return;
 793
 794     TunnelStateData *tunnel = static_cast<TunnelStateData*>(data);
 795     tunnel->client.readPending = NULL;
 796     static uint64_t counter=0;
 797     debugs(26, 7, "Client read(2) delayed " << ++counter << " times");
 798     tunnel->copyRead(tunnel->client, TunnelStateData::ReadClient);
 799 }
 800
 801 static void
 802 tunnelDelayedServerRead(void *data)
 803 {
 804     if (!data)
 805         return;
 806
 807     TunnelStateData *tunnel = static_cast<TunnelStateData*>(data);
 808     tunnel->server.readPending = NULL;
 809     static uint64_t counter=0;
 810     debugs(26, 7, "Server read(2) delayed " << ++counter << " times");
 811     tunnel->copyRead(tunnel->server, TunnelStateData::ReadServer);
 812 }
 813
 814 void
 815 TunnelStateData::copyRead(Connection &from, IOCB *completion)
 816 {
 817     assert(from.len == 0);
 818     // If only the minimum permitted read size is going to be attempted
 819     // then we schedule an event to try again in a few I/O cycles.
 820     // Allow at least 1 byte to be read every (0.3*10) seconds.
 821     int bw = from.bytesWanted(1, SQUID_TCP_SO_RCVBUF);
 822     if (bw == 1 && ++from.delayedLoops < 10) {
 823         from.readPending = this;
 824         eventAdd("tunnelDelayedServerRead", from.readPendingFunc, from.readPending, 0.3, true);
 825         return;
 826     }
 827
 828     AsyncCall::Pointer call = commCbCall(5,4, "TunnelBlindCopyReadHandler",
 829                                          CommIoCbPtrFun(completion, this));
 830     comm_read(from.conn, from.buf, bw, call);
 831 }
 832
 833 void
 834 TunnelStateData::readConnectResponse()
 835 {
 836     assert(waitingForConnectResponse());
 837
 838     AsyncCall::Pointer call = commCbCall(5,4, "readConnectResponseDone",
 839                                          CommIoCbPtrFun(ReadConnectResponseDone, this));
 840     comm_read(server.conn, connectRespBuf->space(),
 841               server.bytesWanted(1, connectRespBuf->spaceSize()), call);
 842 }
 843
 844 void
 845 TunnelStateData::copyClientBytes()
 846 {
 847     if (preReadClientData.length()) {
 848         size_t copyBytes = preReadClientData.length() > SQUID_TCP_SO_RCVBUF ? SQUID_TCP_SO_RCVBUF : preReadClientData.length();
 849         memcpy(client.buf, preReadClientData.rawContent(), copyBytes);
 850         preReadClientData.consume(copyBytes);
 851         client.bytesIn(copyBytes);
 852         if (keepGoingAfterRead(copyBytes, Comm::OK, 0, client, server))
 853             copy(copyBytes, client, server, TunnelStateData::WriteServerDone);
 854     } else
 855         copyRead(client, ReadClient);
 856 }
 857
 858 void
 859 TunnelStateData::copyServerBytes()
 860 {
 861     if (preReadServerData.length()) {
 862         size_t copyBytes = preReadServerData.length() > SQUID_TCP_SO_RCVBUF ? SQUID_TCP_SO_RCVBUF : preReadServerData.length();
 863         memcpy(server.buf, preReadServerData.rawContent(), copyBytes);
 864         preReadServerData.consume(copyBytes);
 865         server.bytesIn(copyBytes);
 866         if (keepGoingAfterRead(copyBytes, Comm::OK, 0, server, client))
 867             copy(copyBytes, server, client, TunnelStateData::WriteClientDone);
 868     } else
 869         copyRead(server, ReadServer);
 870 }
 871
 872 /**
 873  * Set the HTTP status for this request and sets the read handlers for client
 874  * and server side connections.
 875  */
 876 static void
 877 tunnelStartShoveling(TunnelStateData *tunnelState)
 878 {
 879     assert(!tunnelState->waitingForConnectExchange());
 880     *tunnelState->status_ptr = Http::scOkay;
 881     if (tunnelState->logTag_ptr)
 882         *tunnelState->logTag_ptr = LOG_TCP_TUNNEL;
 883     if (cbdataReferenceValid(tunnelState)) {
 884
 885         // Shovel any payload already pushed into reply buffer by the server response
 886         if (!tunnelState->server.len)
 887             tunnelState->copyServerBytes();
 888         else {
 889             debugs(26, DBG_DATA, "Tunnel server PUSH Payload: \n" << Raw("", tunnelState->server.buf, tunnelState->server.len) << "\n----------");
 890             tunnelState->copy(tunnelState->server.len, tunnelState->server, tunnelState->client, TunnelStateData::WriteClientDone);
 891         }
 892
 893         if (tunnelState->http.valid() && tunnelState->http->getConn() && !tunnelState->http->getConn()->inBuf.isEmpty()) {
 894             SBuf * const in = &tunnelState->http->getConn()->inBuf;
 895             debugs(26, DBG_DATA, "Tunnel client PUSH Payload: \n" << *in << "\n----------");
 896             tunnelState->preReadClientData.append(*in);
 897             in->consume(); // ConnStateData buffer accounting after the shuffle.
 898         }
 899         tunnelState->copyClientBytes();
 900     }
 901 }
 902
 903 /**
 904  * All the pieces we need to write to client and/or server connection
 905  * have been written.
 906  * Call the tunnelStartShoveling to start the blind pump.
 907  */
 908 static void
 909 tunnelConnectedWriteDone(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
 910 {
 911     TunnelStateData *tunnelState = (TunnelStateData *)data;
 912     debugs(26, 3, HERE << conn << ", flag=" << flag);
 913     tunnelState->client.writer = NULL;
 914
 915     if (flag != Comm::OK) {
 916         *tunnelState->status_ptr = Http::scInternalServerError;
 917         tunnelErrorComplete(conn->fd, data, 0);
 918         return;
 919     }
 920
 921     if (auto http = tunnelState->http.get()) {
 922         http->out.headers_sz += len;
 923         http->out.size += len;
 924     }
 925
 926     tunnelStartShoveling(tunnelState);
 927 }
 928
 929 /// Called when we are done writing CONNECT request to a peer.
 930 static void
 931 tunnelConnectReqWriteDone(const Comm::ConnectionPointer &conn, char *, size_t, Comm::Flag flag, int, void *data)
 932 {
 933     TunnelStateData *tunnelState = (TunnelStateData *)data;
 934     debugs(26, 3, conn << ", flag=" << flag);
 935     tunnelState->server.writer = NULL;
 936     assert(tunnelState->waitingForConnectRequest());
 937
 938     if (flag != Comm::OK) {
 939         *tunnelState->status_ptr = Http::scInternalServerError;
 940         tunnelErrorComplete(conn->fd, data, 0);
 941         return;
 942     }
 943
 944     tunnelState->connectReqWriting = false;
 945     tunnelState->connectExchangeCheckpoint();
 946 }
 947
 948 void
 949 TunnelStateData::connectExchangeCheckpoint()
 950 {
 951     if (waitingForConnectResponse()) {
 952         debugs(26, 5, "still reading CONNECT response on " << server.conn);
 953     } else if (waitingForConnectRequest()) {
 954         debugs(26, 5, "still writing CONNECT request on " << server.conn);
 955     } else {
 956         assert(!waitingForConnectExchange());
 957         debugs(26, 3, "done with CONNECT exchange on " << server.conn);
 958         tunnelConnected(server.conn, this);
 959     }
 960 }
 961
 962 /*
 963  * handle the write completion from a proxy request to an upstream origin
 964  */
 965 static void
 966 tunnelConnected(const Comm::ConnectionPointer &server, void *data)
 967 {
 968     TunnelStateData *tunnelState = (TunnelStateData *)data;
 969     debugs(26, 3, HERE << server << ", tunnelState=" << tunnelState);
 970
 971     if (!tunnelState->clientExpectsConnectResponse())
 972         tunnelStartShoveling(tunnelState); // ssl-bumped connection, be quiet
 973     else {
 974         AsyncCall::Pointer call = commCbCall(5,5, "tunnelConnectedWriteDone",
 975                                              CommIoCbPtrFun(tunnelConnectedWriteDone, tunnelState));
 976         tunnelState->client.write(conn_established, strlen(conn_established), call, NULL);
 977     }
 978 }
 979
 980 static void
 981 tunnelErrorComplete(int fd/*const Comm::ConnectionPointer &*/, void *data, size_t)
 982 {
 983     TunnelStateData *tunnelState = (TunnelStateData *)data;
 984     debugs(26, 3, HERE << "FD " << fd);
 985     assert(tunnelState != NULL);
 986     /* temporary lock to save our own feets (comm_close -> tunnelClientClosed -> Free) */
 987     CbcPointer<TunnelStateData> safetyLock(tunnelState);
 988
 989     if (Comm::IsConnOpen(tunnelState->client.conn))
 990         tunnelState->client.conn->close();
 991
 992     if (Comm::IsConnOpen(tunnelState->server.conn))
 993         tunnelState->server.conn->close();
 994 }
 995
 996 /// reacts to a failure to establish the given TCP connection
 997 void
 998 TunnelStateData::noteConnectFailure(const Comm::ConnectionPointer &conn)
 999 {
1000     debugs(26, 4, "removing the failed one from " << serverDestinations.size() <<
1001            " destinations: " << conn);
1002
1003     if (CachePeer *peer = conn->getPeer())
1004         peerConnectFailed(peer);
1005
1006     assert(!serverDestinations.empty());
1007     serverDestinations.erase(serverDestinations.begin());
1008
1009     // Since no TCP payload has been passed to client or server, we may
1010     // TCP-connect to other destinations (including alternate IPs).
1011
1012     if (!FwdState::EnoughTimeToReForward(startTime))
1013         return sendError(savedError, "forwarding timeout");
1014
1015     if (!serverDestinations.empty())
1016         return startConnecting();
1017
1018     if (!PeerSelectionInitiator::subscribed)
1019         return sendError(savedError, "tried all destinations");
1020
1021     debugs(26, 4, "wait for more destinations to try");
1022     // expect a noteDestination*() call
1023 }
1024
1025 static void
1026 tunnelConnectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno, void *data)
1027 {
1028     TunnelStateData *tunnelState = (TunnelStateData *)data;
1029
1030     if (status != Comm::OK) {
1031         ErrorState *err = new ErrorState(ERR_CONNECT_FAIL, Http::scServiceUnavailable, tunnelState->request.getRaw());
1032         err->xerrno = xerrno;
1033         // on timeout is this still:    err->xerrno = ETIMEDOUT;
1034         err->port = conn->remote.port();
1035         tunnelState->saveError(err);
1036         tunnelState->noteConnectFailure(conn);
1037         return;
1038     }
1039
1040 #if USE_DELAY_POOLS
1041     /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
1042     if (conn->getPeer() && conn->getPeer()->options.no_delay)
1043         tunnelState->server.setDelayId(DelayId());
1044 #endif
1045
1046     tunnelState->request->hier.note(conn, tunnelState->getHost());
1047
1048     tunnelState->server.conn = conn;
1049     tunnelState->request->peer_host = conn->getPeer() ? conn->getPeer()->host : NULL;
1050     comm_add_close_handler(conn->fd, tunnelServerClosed, tunnelState);
1051
1052     debugs(26, 4, HERE << "determine post-connect handling pathway.");
1053     if (conn->getPeer()) {
1054         tunnelState->request->peer_login = conn->getPeer()->login;
1055         tunnelState->request->peer_domain = conn->getPeer()->domain;
1056         tunnelState->request->flags.auth_no_keytab = conn->getPeer()->options.auth_no_keytab;
1057         tunnelState->request->flags.proxying = !(conn->getPeer()->options.originserver);
1058     } else {
1059         tunnelState->request->peer_login = NULL;
1060         tunnelState->request->peer_domain = NULL;
1061         tunnelState->request->flags.auth_no_keytab = false;
1062         tunnelState->request->flags.proxying = false;
1063     }
1064
1065     if (tunnelState->request->flags.proxying)
1066         tunnelState->connectToPeer();
1067     else {
1068         tunnelConnected(conn, tunnelState);
1069     }
1070
1071     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
1072                                      CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
1073     commSetConnTimeout(conn, Config.Timeout.read, timeoutCall);
1074 }
1075
1076 void
1077 tunnelStart(ClientHttpRequest * http)
1078 {
1079     debugs(26, 3, HERE);
1080     /* Create state structure. */
1081     TunnelStateData *tunnelState = NULL;
1082     ErrorState *err = NULL;
1083     HttpRequest *request = http->request;
1084     char *url = http->uri;
1085
1086     /*
1087      * client_addr.isNoAddr()  indicates this is an "internal" request
1088      * from peer_digest.c, asn.c, netdb.c, etc and should always
1089      * be allowed.  yuck, I know.
1090      */
1091
1092     if (Config.accessList.miss && !request->client_addr.isNoAddr()) {
1093         /*
1094          * Check if this host is allowed to fetch MISSES from us (miss_access)
1095          * default is to allow.
1096          */
1097         ACLFilledChecklist ch(Config.accessList.miss, request, NULL);
1098         ch.src_addr = request->client_addr;
1099         ch.my_addr = request->my_addr;
1100         if (ch.fastCheck() == ACCESS_DENIED) {
1101             debugs(26, 4, HERE << "MISS access forbidden.");
1102             err = new ErrorState(ERR_FORWARDING_DENIED, Http::scForbidden, request);
1103             http->al->http.code = Http::scForbidden;
1104             errorSend(http->getConn()->clientConnection, err);
1105             return;
1106         }
1107     }
1108
1109     debugs(26, 3, request->method << ' ' << url << ' ' << request->http_ver);
1110     ++statCounter.server.all.requests;
1111     ++statCounter.server.other.requests;
1112
1113     tunnelState = new TunnelStateData(http);
1114 #if USE_DELAY_POOLS
1115     //server.setDelayId called from tunnelConnectDone after server side connection established
1116 #endif
1117     tunnelState->startSelectingDestinations(request, http->al, nullptr);
1118 }
1119
1120 void
1121 TunnelStateData::connectToPeer()
1122 {
1123     if (CachePeer *p = server.conn->getPeer()) {
1124         if (p->secure.encryptTransport) {
1125             AsyncCall::Pointer callback = asyncCall(5,4,
1126                                                     "TunnelStateData::ConnectedToPeer",
1127                                                     MyAnswerDialer(&TunnelStateData::connectedToPeer, this));
1128             auto *connector = new Security::BlindPeerConnector(request, server.conn, callback, al);
1129             AsyncJob::Start(connector); // will call our callback
1130             return;
1131         }
1132     }
1133
1134     Security::EncryptorAnswer nil;
1135     connectedToPeer(nil);
1136 }
1137
1138 void
1139 TunnelStateData::connectedToPeer(Security::EncryptorAnswer &answer)
1140 {
1141     if (ErrorState *error = answer.error.get()) {
1142         answer.error.clear(); // sendError() will own the error
1143         sendError(error, "TLS peer connection error");
1144         return;
1145     }
1146
1147     tunnelRelayConnectRequest(server.conn, this);
1148 }
1149
1150 static void
1151 tunnelRelayConnectRequest(const Comm::ConnectionPointer &srv, void *data)
1152 {
1153     TunnelStateData *tunnelState = (TunnelStateData *)data;
1154     assert(!tunnelState->waitingForConnectExchange());
1155     HttpHeader hdr_out(hoRequest);
1156     Http::StateFlags flags;
1157     debugs(26, 3, HERE << srv << ", tunnelState=" << tunnelState);
1158     memset(&flags, '\0', sizeof(flags));
1159     flags.proxying = tunnelState->request->flags.proxying;
1160     MemBuf mb;
1161     mb.init();
1162     mb.appendf("CONNECT %s HTTP/1.1\r\n", tunnelState->url);
1163     HttpStateData::httpBuildRequestHeader(tunnelState->request.getRaw(),
1164                                           NULL,         /* StoreEntry */
1165                                           tunnelState->al,          /* AccessLogEntry */
1166                                           &hdr_out,
1167                                           flags);           /* flags */
1168     hdr_out.packInto(&mb);
1169     hdr_out.clean();
1170     mb.append("\r\n", 2);
1171
1172     debugs(11, 2, "Tunnel Server REQUEST: " << tunnelState->server.conn <<
1173            ":\n----------\n" << mb.buf << "\n----------");
1174
1175     AsyncCall::Pointer writeCall = commCbCall(5,5, "tunnelConnectReqWriteDone",
1176                                    CommIoCbPtrFun(tunnelConnectReqWriteDone,
1177                                            tunnelState));
1178
1179     tunnelState->server.write(mb.buf, mb.size, writeCall, mb.freeFunc());
1180     tunnelState->connectReqWriting = true;
1181
1182     tunnelState->connectRespBuf = new MemBuf;
1183     // SQUID_TCP_SO_RCVBUF: we should not accumulate more than regular I/O buffer
1184     // can hold since any CONNECT response leftovers have to fit into server.buf.
1185     // 2*SQUID_TCP_SO_RCVBUF: Http::Message::parse() zero-terminates, which uses space.
1186     tunnelState->connectRespBuf->init(SQUID_TCP_SO_RCVBUF, 2*SQUID_TCP_SO_RCVBUF);
1187     tunnelState->readConnectResponse();
1188
1189     assert(tunnelState->waitingForConnectExchange());
1190
1191     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
1192                                      CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
1193     commSetConnTimeout(srv, Config.Timeout.read, timeoutCall);
1194 }
1195
1196 static Comm::ConnectionPointer
1197 borrowPinnedConnection(HttpRequest *request, Comm::ConnectionPointer &serverDestination)
1198 {
1199     // pinned_connection may become nil after a pconn race
1200     if (ConnStateData *pinned_connection = request ? request->pinnedConnection() : nullptr) {
1201         Comm::ConnectionPointer serverConn = pinned_connection->borrowPinnedConnection(request, serverDestination->getPeer());
1202         return serverConn;
1203     }
1204
1205     return nullptr;
1206 }
1207
1208 void
1209 TunnelStateData::noteDestination(Comm::ConnectionPointer path)
1210 {
1211     const bool wasBlocked = serverDestinations.empty();
1212     serverDestinations.push_back(path);
1213     if (wasBlocked)
1214         startConnecting();
1215     // else continue to use one of the previously noted destinations;
1216     // if all of them fail, we may try this path
1217 }
1218
1219 void
1220 TunnelStateData::noteDestinationsEnd(ErrorState *selectionError)
1221 {
1222     PeerSelectionInitiator::subscribed = false;
1223     if (const bool wasBlocked = serverDestinations.empty()) {
1224
1225         if (selectionError)
1226             return sendError(selectionError, "path selection has failed");
1227
1228         if (savedError)
1229             return sendError(savedError, "all found paths have failed");
1230
1231         return sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw()),
1232                          "path selection found no paths");
1233     }
1234     // else continue to use one of the previously noted destinations;
1235     // if all of them fail, tunneling as whole will fail
1236     Must(!selectionError); // finding at least one path means selection succeeded
1237 }
1238
1239 /// remembers an error to be used if there will be no more connection attempts
1240 void
1241 TunnelStateData::saveError(ErrorState *error)
1242 {
1243     debugs(26, 4, savedError << " ? " << error);
1244     assert(error);
1245     delete savedError; // may be nil
1246     savedError = error;
1247 }
1248
1249 /// Starts sending the given error message to the client, leading to the
1250 /// eventual transaction termination. Call with savedError to send savedError.
1251 void
1252 TunnelStateData::sendError(ErrorState *finalError, const char *reason)
1253 {
1254     debugs(26, 3, "aborting transaction for " << reason);
1255
1256     if (request)
1257         request->hier.stopPeerClock(false);
1258
1259     assert(finalError);
1260
1261     // get rid of any cached error unless that is what the caller is sending
1262     if (savedError != finalError)
1263         delete savedError; // may be nil
1264     savedError = nullptr;
1265
1266     // we cannot try other destinations after responding with an error
1267     PeerSelectionInitiator::subscribed = false; // may already be false
1268
1269     *status_ptr = finalError->httpStatus;
1270     finalError->callback = tunnelErrorComplete;
1271     finalError->callback_data = this;
1272     errorSend(client.conn, finalError);
1273 }
1274
1275 void
1276 TunnelStateData::startConnecting()
1277 {
1278     if (request)
1279         request->hier.startPeerClock();
1280
1281     assert(!serverDestinations.empty());
1282     Comm::ConnectionPointer &dest = serverDestinations.front();
1283     debugs(26, 3, "to " << dest);
1284
1285     if (dest->peerType == PINNED) {
1286         Comm::ConnectionPointer serverConn = borrowPinnedConnection(request.getRaw(), dest);
1287         debugs(26,7, "pinned peer connection: " << serverConn);
1288         if (Comm::IsConnOpen(serverConn)) {
1289             tunnelConnectDone(serverConn, Comm::OK, 0, (void *)this);
1290             return;
1291         }
1292         // a PINNED path failure is fatal; do not wait for more paths
1293         sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw()),
1294                   "pinned path failure");
1295         return;
1296     }
1297
1298     GetMarkingsToServer(request.getRaw(), *dest);
1299
1300     const time_t connectTimeout = dest->connectTimeout(startTime);
1301     AsyncCall::Pointer call = commCbCall(26,3, "tunnelConnectDone", CommConnectCbPtrFun(tunnelConnectDone, this));
1302     Comm::ConnOpener *cs = new Comm::ConnOpener(dest, call, connectTimeout);
1303     cs->setHost(url);
1304     AsyncJob::Start(cs);
1305 }
1306
1307 CBDATA_CLASS_INIT(TunnelStateData);
1308
1309 bool
1310 TunnelStateData::noConnections() const
1311 {
1312     return !Comm::IsConnOpen(server.conn) && !Comm::IsConnOpen(client.conn);
1313 }
1314
1315 #if USE_DELAY_POOLS
1316 void
1317 TunnelStateData::Connection::setDelayId(DelayId const &newDelay)
1318 {
1319     delayId = newDelay;
1320 }
1321
1322 #endif
1323
1324 #if USE_OPENSSL
1325 void
1326 switchToTunnel(HttpRequest *request, Comm::ConnectionPointer &clientConn, Comm::ConnectionPointer &srvConn)
1327 {
1328     debugs(26,5, "Revert to tunnel FD " << clientConn->fd << " with FD " << srvConn->fd);
1329
1330     /* Create state structure. */
1331     ++statCounter.server.all.requests;
1332     ++statCounter.server.other.requests;
1333
1334     auto conn = request->clientConnectionManager.get();
1335     Must(conn);
1336     Http::StreamPointer context = conn->pipeline.front();
1337     Must(context && context->http);
1338
1339     debugs(26, 3, request->method << " " << context->http->uri << " " << request->http_ver);
1340
1341     TunnelStateData *tunnelState = new TunnelStateData(context->http);
1342
1343     fd_table[clientConn->fd].read_method = &default_read_method;
1344     fd_table[clientConn->fd].write_method = &default_write_method;
1345
1346     request->hier.note(srvConn, tunnelState->getHost());
1347
1348     tunnelState->server.conn = srvConn;
1349
1350 #if USE_DELAY_POOLS
1351     /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
1352     if (srvConn->getPeer() && srvConn->getPeer()->options.no_delay)
1353         tunnelState->server.setDelayId(DelayId::DelayClient(context->http));
1354 #endif
1355
1356     request->peer_host = srvConn->getPeer() ? srvConn->getPeer()->host : nullptr;
1357     comm_add_close_handler(srvConn->fd, tunnelServerClosed, tunnelState);
1358
1359     debugs(26, 4, "determine post-connect handling pathway.");
1360     if (srvConn->getPeer()) {
1361         request->peer_login = srvConn->getPeer()->login;
1362         request->peer_domain = srvConn->getPeer()->domain;
1363         request->flags.auth_no_keytab = srvConn->getPeer()->options.auth_no_keytab;
1364         request->flags.proxying = !(srvConn->getPeer()->options.originserver);
1365     } else {
1366         request->peer_login = nullptr;
1367         request->peer_domain = nullptr;
1368         request->flags.auth_no_keytab = false;
1369         request->flags.proxying = false;
1370     }
1371
1372     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
1373                                      CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
1374     commSetConnTimeout(srvConn, Config.Timeout.read, timeoutCall);
1375     fd_table[srvConn->fd].read_method = &default_read_method;
1376     fd_table[srvConn->fd].write_method = &default_write_method;
1377
1378     auto ssl = fd_table[srvConn->fd].ssl.get();
1379     assert(ssl);
1380     BIO *b = SSL_get_rbio(ssl);
1381     Ssl::ServerBio *srvBio = static_cast<Ssl::ServerBio *>(BIO_get_data(b));
1382     tunnelState->preReadServerData = srvBio->rBufData();
1383     tunnelStartShoveling(tunnelState);
1384 }
1385 #endif //USE_OPENSSL
1386