src/tunnel.cc

   1 /*
   2  * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 26    Secure Sockets Layer Proxy */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/AsyncCallbacks.h"
  14 #include "base/CbcPointer.h"
  15 #include "base/JobWait.h"
  16 #include "base/Raw.h"
  17 #include "CachePeer.h"
  18 #include "cbdata.h"
  19 #include "client_side.h"
  20 #include "client_side_request.h"
  21 #include "clients/HttpTunneler.h"
  22 #include "comm.h"
  23 #include "comm/Connection.h"
  24 #include "comm/ConnOpener.h"
  25 #include "comm/Read.h"
  26 #include "comm/Write.h"
  27 #include "errorpage.h"
  28 #include "fd.h"
  29 #include "fde.h"
  30 #include "FwdState.h"
  31 #include "globals.h"
  32 #include "HappyConnOpener.h"
  33 #include "http.h"
  34 #include "http/StatusCode.h"
  35 #include "http/Stream.h"
  36 #include "HttpRequest.h"
  37 #include "icmp/net_db.h"
  38 #include "ip/QosConfig.h"
  39 #include "LogTags.h"
  40 #include "MemBuf.h"
  41 #include "neighbors.h"
  42 #include "PeerSelectState.h"
  43 #include "ResolvedPeers.h"
  44 #include "sbuf/SBuf.h"
  45 #include "security/BlindPeerConnector.h"
  46 #include "SquidConfig.h"
  47 #include "StatCounters.h"
  48 #if USE_OPENSSL
  49 #include "ssl/bio.h"
  50 #include "ssl/ServerBump.h"
  51 #endif
  52 #include "tools.h"
  53 #include "tunnel.h"
  54 #if USE_DELAY_POOLS
  55 #include "DelayId.h"
  56 #endif
  57
  58 #include <climits>
  59 #include <cerrno>
  60
  61 /**
  62  * TunnelStateData is the state engine performing the tasks for
  63  * setup of a TCP tunnel from an existing open client FD to a server
  64  * then shuffling binary data between the resulting FD pair.
  65  */
  66 /*
  67  * TODO 1: implement a read/write API on ConnStateData to send/receive blocks
  68  * of pre-formatted data. Then we can use that as the client side of the tunnel
  69  * instead of re-implementing it here and occasionally getting the ConnStateData
  70  * read/write state wrong.
  71  *
  72  * TODO 2: then convert this into a AsyncJob, possibly a child of 'Server'
  73  */
  74 class TunnelStateData: public PeerSelectionInitiator
  75 {
  76     CBDATA_CHILD(TunnelStateData);
  77
  78 public:
  79     TunnelStateData(ClientHttpRequest *);
  80     ~TunnelStateData() override;
  81     TunnelStateData(const TunnelStateData &); // do not implement
  82     TunnelStateData &operator =(const TunnelStateData &); // do not implement
  83
  84     class Connection;
  85     static void ReadClient(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data);
  86     static void ReadServer(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data);
  87     static void WriteClientDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data);
  88     static void WriteServerDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data);
  89
  90     bool noConnections() const;
  91     /// closes both client and server connections
  92     void closeConnections();
  93
  94     char *url;
  95     CbcPointer<ClientHttpRequest> http;
  96     HttpRequest::Pointer request;
  97     AccessLogEntryPointer al;
  98
  99     const char * getHost() const {
 100         return (server.conn != nullptr && server.conn->getPeer() ? server.conn->getPeer()->host : request->url.host());
 101     };
 102
 103     /// store the given to-server connection; prohibit retries and do not look
 104     /// for any other destinations
 105     void commitToServer(const Comm::ConnectionPointer &);
 106
 107     /// Whether the client sent a CONNECT request to us.
 108     bool clientExpectsConnectResponse() const {
 109         // If we are forcing a tunnel after receiving a client CONNECT, then we
 110         // have already responded to that CONNECT before tunnel.cc started.
 111         if (request && request->flags.forceTunnel)
 112             return false;
 113 #if USE_OPENSSL
 114         // We are bumping and we had already send "OK CONNECTED"
 115         if (http.valid() && http->getConn() && http->getConn()->serverBump() && http->getConn()->serverBump()->at(XactionStep::tlsBump2, XactionStep::tlsBump3))
 116             return false;
 117 #endif
 118         return !(request != nullptr &&
 119                  (request->flags.interceptTproxy || request->flags.intercepted));
 120     }
 121
 122     /// starts connecting to the next hop, either for the first time or while
 123     /// recovering from the previous connect failure
 124     void startConnecting();
 125     void closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason);
 126
 127     /// called when negotiations with the peer have been successfully completed
 128     void notePeerReadyToShovel(const Comm::ConnectionPointer &);
 129
 130     class Connection
 131     {
 132
 133     public:
 134         Connection() : len (0), buf ((char *)xmalloc(SQUID_TCP_SO_RCVBUF)), size_ptr(nullptr), delayedLoops(0),
 135             dirty(false),
 136             readPending(nullptr), readPendingFunc(nullptr) {}
 137
 138         ~Connection();
 139
 140         /// initiates Comm::Connection ownership, including closure monitoring
 141         template <typename Method>
 142         void initConnection(const Comm::ConnectionPointer &aConn, Method method, const char *name, TunnelStateData *tunnelState);
 143
 144         /// reacts to the external closure of our connection
 145         void noteClosure();
 146
 147         int bytesWanted(int lower=0, int upper = INT_MAX) const;
 148         void bytesIn(int const &);
 149 #if USE_DELAY_POOLS
 150
 151         void setDelayId(DelayId const &);
 152 #endif
 153
 154         void error(int const xerrno);
 155         int debugLevelForError(int const xerrno) const;
 156
 157         void dataSent (size_t amount);
 158         /// writes 'b' buffer, setting the 'writer' member to 'callback'.
 159         void write(const char *b, int size, AsyncCall::Pointer &callback, FREE * free_func);
 160         int len;
 161         char *buf;
 162         AsyncCall::Pointer writer; ///< pending Comm::Write callback
 163         uint64_t *size_ptr;      /* pointer to size in an ConnStateData for logging */
 164
 165         Comm::ConnectionPointer conn;    ///< The currently connected connection.
 166         uint8_t delayedLoops; ///< how many times a read on this connection has been postponed.
 167
 168         bool dirty; ///< whether write() has been called (at least once)
 169
 170         // XXX: make these an AsyncCall when event API can handle them
 171         TunnelStateData *readPending;
 172         EVH *readPendingFunc;
 173
 174 #if USE_DELAY_POOLS
 175
 176         DelayId delayId;
 177 #endif
 178
 179     private:
 180         /// the registered close handler for the connection
 181         AsyncCall::Pointer closer;
 182     };
 183
 184     Connection client, server;
 185     int *status_ptr;        ///< pointer for logging HTTP status
 186
 187     SBuf preReadClientData;
 188     SBuf preReadServerData;
 189     time_t startTime; ///< object creation time, before any peer selection/connection attempts
 190     ResolvedPeersPointer destinations; ///< paths for forwarding the request
 191     bool destinationsFound; ///< At least one candidate path found
 192
 193     /// whether the decision to tunnel to a particular destination was final
 194     bool committedToServer;
 195
 196     int n_tries; ///< the number of forwarding attempts so far
 197
 198     /// a reason to ban reforwarding attempts (or nil)
 199     const char *banRetries;
 200
 201     // TODO: remove after fixing deferred reads in TunnelStateData::copyRead()
 202     CodeContext::Pointer codeContext; ///< our creator context
 203
 204     /// waits for a transport connection to the peer to be established/opened
 205     JobWait<HappyConnOpener> transportWait;
 206
 207     /// waits for the established transport connection to be secured/encrypted
 208     JobWait<Security::PeerConnector> encryptionWait;
 209
 210     /// waits for an HTTP CONNECT tunnel through a cache_peer to be negotiated
 211     /// over the (encrypted, if needed) transport connection to that cache_peer
 212     JobWait<Http::Tunneler> peerWait;
 213
 214     void copyRead(Connection &from, IOCB *completion);
 215
 216     /// continue to set up connection to a peer, going async for SSL peers
 217     void connectToPeer(const Comm::ConnectionPointer &);
 218     void secureConnectionToPeer(const Comm::ConnectionPointer &);
 219
 220     /* PeerSelectionInitiator API */
 221     void noteDestination(Comm::ConnectionPointer conn) override;
 222     void noteDestinationsEnd(ErrorState *selectionError) override;
 223
 224     void syncHierNote(const Comm::ConnectionPointer &server, const char *origin);
 225
 226     /// called when a connection has been successfully established or
 227     /// when all candidate destinations have been tried and all have failed
 228     void noteConnection(HappyConnOpenerAnswer &);
 229
 230     /// Start using an established connection
 231     void connectDone(const Comm::ConnectionPointer &conn, const char *origin, const bool reused);
 232
 233     void notifyConnOpener();
 234
 235     void saveError(ErrorState *finalError);
 236     void sendError(ErrorState *finalError, const char *reason);
 237
 238 private:
 239     void usePinned();
 240
 241     /// callback handler for the Security::PeerConnector encryptor
 242     void noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &);
 243
 244     /// called after connection setup (including any encryption)
 245     void connectedToPeer(const Comm::ConnectionPointer &);
 246     void establishTunnelThruProxy(const Comm::ConnectionPointer &);
 247
 248     template <typename StepStart>
 249     void advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep);
 250
 251     /// \returns whether the request should be retried (nil) or the description why it should not
 252     const char *checkRetry();
 253
 254     bool transporting() const;
 255
 256     // TODO: convert to unique_ptr
 257     /// details of the "last tunneling attempt" failure (if it failed)
 258     ErrorState *savedError = nullptr;
 259
 260     /// resumes operations after the (possibly failed) HTTP CONNECT exchange
 261     void tunnelEstablishmentDone(Http::TunnelerAnswer &answer);
 262
 263     void deleteThis();
 264
 265     void cancelStep(const char *reason);
 266
 267     bool exhaustedTries() const;
 268     void updateAttempts(int);
 269
 270 public:
 271     bool keepGoingAfterRead(size_t len, Comm::Flag errcode, int xerrno, Connection &from, Connection &to);
 272     void copy(size_t len, Connection &from, Connection &to, IOCB *);
 273     void readServer(char *buf, size_t len, Comm::Flag errcode, int xerrno);
 274     void readClient(char *buf, size_t len, Comm::Flag errcode, int xerrno);
 275     void writeClientDone(char *buf, size_t len, Comm::Flag flag, int xerrno);
 276     void writeServerDone(char *buf, size_t len, Comm::Flag flag, int xerrno);
 277
 278     void copyClientBytes();
 279     void copyServerBytes();
 280
 281     /// handles client-to-Squid connection closure; may destroy us
 282     void clientClosed();
 283
 284     /// handles Squid-to-server connection closure; may destroy us
 285     void serverClosed();
 286
 287     /// tries connecting to another destination, if available,
 288     /// otherwise, initiates the transaction termination
 289     void retryOrBail(const char *context);
 290 };
 291
 292 static ERCB tunnelErrorComplete;
 293 static CLCB tunnelServerClosed;
 294 static CLCB tunnelClientClosed;
 295 static CTCB tunnelTimeout;
 296 static EVH tunnelDelayedClientRead;
 297 static EVH tunnelDelayedServerRead;
 298
 299 /// TunnelStateData::serverClosed() wrapper
 300 static void
 301 tunnelServerClosed(const CommCloseCbParams &params)
 302 {
 303     const auto tunnelState = reinterpret_cast<TunnelStateData *>(params.data);
 304     tunnelState->serverClosed();
 305 }
 306
 307 void
 308 TunnelStateData::serverClosed()
 309 {
 310     server.noteClosure();
 311
 312     retryOrBail(__FUNCTION__);
 313 }
 314
 315 /// TunnelStateData::clientClosed() wrapper
 316 static void
 317 tunnelClientClosed(const CommCloseCbParams &params)
 318 {
 319     const auto tunnelState = reinterpret_cast<TunnelStateData *>(params.data);
 320     tunnelState->clientClosed();
 321 }
 322
 323 void
 324 TunnelStateData::clientClosed()
 325 {
 326     client.noteClosure();
 327
 328     if (noConnections())
 329         return deleteThis();
 330
 331     if (!server.writer)
 332         server.conn->close();
 333 }
 334
 335 /// destroys the tunnel (after performing potentially-throwing cleanup)
 336 void
 337 TunnelStateData::deleteThis()
 338 {
 339     assert(noConnections());
 340     // ConnStateData pipeline should contain the CONNECT we are performing
 341     // but it may be invalid already (bug 4392)
 342     if (const auto h = http.valid()) {
 343         if (const auto c = h->getConn())
 344             if (const auto ctx = c->pipeline.front())
 345                 ctx->finished();
 346     }
 347     delete this;
 348 }
 349
 350 TunnelStateData::TunnelStateData(ClientHttpRequest *clientRequest) :
 351     startTime(squid_curtime),
 352     destinations(new ResolvedPeers()),
 353     destinationsFound(false),
 354     committedToServer(false),
 355     n_tries(0),
 356     banRetries(nullptr),
 357     codeContext(CodeContext::Current())
 358 {
 359     debugs(26, 3, "TunnelStateData constructed this=" << this);
 360     client.readPendingFunc = &tunnelDelayedClientRead;
 361     server.readPendingFunc = &tunnelDelayedServerRead;
 362
 363     assert(clientRequest);
 364     url = xstrdup(clientRequest->uri);
 365     request = clientRequest->request;
 366     Must(request);
 367     server.size_ptr = &clientRequest->out.size;
 368     client.size_ptr = &clientRequest->al->http.clientRequestSz.payloadData;
 369     status_ptr = &clientRequest->al->http.code;
 370     al = clientRequest->al;
 371     http = clientRequest;
 372
 373     client.initConnection(clientRequest->getConn()->clientConnection, tunnelClientClosed, "tunnelClientClosed", this);
 374
 375     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 376                                      CommTimeoutCbPtrFun(tunnelTimeout, this));
 377     commSetConnTimeout(client.conn, Config.Timeout.lifetime, timeoutCall);
 378 }
 379
 380 TunnelStateData::~TunnelStateData()
 381 {
 382     debugs(26, 3, "TunnelStateData destructed this=" << this);
 383     assert(noConnections());
 384     xfree(url);
 385     cancelStep("~TunnelStateData");
 386     delete savedError;
 387 }
 388
 389 TunnelStateData::Connection::~Connection()
 390 {
 391     if (readPending)
 392         eventDelete(readPendingFunc, readPending);
 393
 394     safe_free(buf);
 395 }
 396
 397 const char *
 398 TunnelStateData::checkRetry()
 399 {
 400     if (shutting_down)
 401         return "shutting down";
 402     if (exhaustedTries())
 403         return "exhausted tries";
 404     if (!FwdState::EnoughTimeToReForward(startTime))
 405         return "forwarding timeout";
 406     if (banRetries)
 407         return banRetries;
 408     if (noConnections())
 409         return "no connections";
 410
 411     // TODO: Use std::optional for peer_reply_status to avoid treating zero value specially.
 412     if (request->hier.peer_reply_status != Http::scNone && !Http::IsReforwardableStatus(request->hier.peer_reply_status))
 413         return "received HTTP status code is not reforwardable";
 414
 415     // TODO: check pinned connections; see FwdState::pinnedCanRetry()
 416     return nullptr;
 417 }
 418
 419 void
 420 TunnelStateData::retryOrBail(const char *context)
 421 {
 422     assert(!server.conn);
 423
 424     const auto *bailDescription = checkRetry();
 425     if (!bailDescription) {
 426         if (!destinations->empty())
 427             return startConnecting(); // try connecting to another destination
 428
 429         if (subscribed) {
 430             debugs(26, 4, "wait for more destinations to try");
 431             return; // expect a noteDestination*() call
 432         }
 433
 434         // fall through to bail
 435     }
 436
 437     /* bail */
 438
 439     if (request)
 440         request->hier.stopPeerClock(false);
 441
 442     // TODO: Add sendSavedErrorOr(err_type type, Http::StatusCode, context).
 443     // Then, the remaining method code (below) should become the common part of
 444     // sendNewError() and sendSavedErrorOr(), used in "error detected" cases.
 445     if (!savedError)
 446         saveError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al));
 447     const auto canSendError = Comm::IsConnOpen(client.conn) && !client.dirty &&
 448                               clientExpectsConnectResponse();
 449     if (canSendError)
 450         return sendError(savedError, bailDescription ? bailDescription : context);
 451     *status_ptr = savedError->httpStatus;
 452
 453     if (noConnections())
 454         return deleteThis();
 455
 456     // This is a "Comm::IsConnOpen(client.conn) but !canSendError" case.
 457     // Closing the connection (after finishing writing) is the best we can do.
 458     if (!client.writer)
 459         client.conn->close();
 460     // else writeClientDone() must notice a closed server and close the client
 461 }
 462
 463 int
 464 TunnelStateData::Connection::bytesWanted(int lowerbound, int upperbound) const
 465 {
 466 #if USE_DELAY_POOLS
 467     return delayId.bytesWanted(lowerbound, upperbound);
 468 #else
 469     (void)lowerbound;
 470     return upperbound;
 471 #endif
 472 }
 473
 474 void
 475 TunnelStateData::Connection::bytesIn(int const &count)
 476 {
 477     debugs(26, 3, "len=" << len << " + count=" << count);
 478 #if USE_DELAY_POOLS
 479     delayId.bytesIn(count);
 480 #endif
 481
 482     len += count;
 483 }
 484
 485 /// update "hierarchy" annotations with a new (possibly failed) destination
 486 /// \param origin the name of the origin server we were trying to reach
 487 void
 488 TunnelStateData::syncHierNote(const Comm::ConnectionPointer &conn, const char *origin)
 489 {
 490     request->hier.resetPeerNotes(conn, origin);
 491     al->hier.resetPeerNotes(conn, origin);
 492 }
 493
 494 /// sets n_tries to the given value (while keeping ALE in sync)
 495 void
 496 TunnelStateData::updateAttempts(const int newValue)
 497 {
 498     Assure(n_tries <= newValue); // n_tries cannot decrease
 499
 500     // Squid probably creates at most one FwdState/TunnelStateData object per
 501     // ALE, but, unlike an assignment would, this increment logic works even if
 502     // Squid uses multiple such objects for a given ALE in some esoteric cases.
 503     al->requestAttempts += (newValue - n_tries);
 504
 505     n_tries = newValue;
 506     debugs(26, 5, n_tries);
 507 }
 508
 509 int
 510 TunnelStateData::Connection::debugLevelForError(int const xerrno) const
 511 {
 512 #ifdef ECONNRESET
 513
 514     if (xerrno == ECONNRESET)
 515         return 2;
 516
 517 #endif
 518
 519     if (ignoreErrno(xerrno))
 520         return 3;
 521
 522     return 1;
 523 }
 524
 525 /* Read from server side and queue it for writing to the client */
 526 void
 527 TunnelStateData::ReadServer(const Comm::ConnectionPointer &c, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data)
 528 {
 529     TunnelStateData *tunnelState = (TunnelStateData *)data;
 530     assert(cbdataReferenceValid(tunnelState));
 531     debugs(26, 3, c);
 532
 533     tunnelState->readServer(buf, len, errcode, xerrno);
 534 }
 535
 536 void
 537 TunnelStateData::readServer(char *, size_t len, Comm::Flag errcode, int xerrno)
 538 {
 539     debugs(26, 3, server.conn << ", read " << len << " bytes, err=" << errcode);
 540     server.delayedLoops=0;
 541
 542     /*
 543      * Bail out early on Comm::ERR_CLOSING
 544      * - close handlers will tidy up for us
 545      */
 546
 547     if (errcode == Comm::ERR_CLOSING)
 548         return;
 549
 550     if (len > 0) {
 551         server.bytesIn(len);
 552         statCounter.server.all.kbytes_in += len;
 553         statCounter.server.other.kbytes_in += len;
 554         request->hier.notePeerRead();
 555     }
 556
 557     if (keepGoingAfterRead(len, errcode, xerrno, server, client))
 558         copy(len, server, client, WriteClientDone);
 559 }
 560
 561 void
 562 TunnelStateData::Connection::error(int const xerrno)
 563 {
 564     debugs(50, debugLevelForError(xerrno), conn << ": read/write failure: " << xstrerr(xerrno));
 565
 566     if (!ignoreErrno(xerrno))
 567         conn->close();
 568 }
 569
 570 /* Read from client side and queue it for writing to the server */
 571 void
 572 TunnelStateData::ReadClient(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data)
 573 {
 574     TunnelStateData *tunnelState = (TunnelStateData *)data;
 575     assert (cbdataReferenceValid (tunnelState));
 576
 577     tunnelState->readClient(buf, len, errcode, xerrno);
 578 }
 579
 580 void
 581 TunnelStateData::readClient(char *, size_t len, Comm::Flag errcode, int xerrno)
 582 {
 583     debugs(26, 3, client.conn << ", read " << len << " bytes, err=" << errcode);
 584     client.delayedLoops=0;
 585
 586     /*
 587      * Bail out early on Comm::ERR_CLOSING
 588      * - close handlers will tidy up for us
 589      */
 590
 591     if (errcode == Comm::ERR_CLOSING)
 592         return;
 593
 594     if (len > 0) {
 595         client.bytesIn(len);
 596         statCounter.client_http.kbytes_in += len;
 597     }
 598
 599     if (keepGoingAfterRead(len, errcode, xerrno, client, server))
 600         copy(len, client, server, WriteServerDone);
 601 }
 602
 603 /// Updates state after reading from client or server.
 604 /// Returns whether the caller should use the data just read.
 605 bool
 606 TunnelStateData::keepGoingAfterRead(size_t len, Comm::Flag errcode, int xerrno, Connection &from, Connection &to)
 607 {
 608     debugs(26, 3, "from={" << from.conn << "}, to={" << to.conn << "}");
 609
 610     /* I think this is to prevent free-while-in-a-callback behaviour
 611      * - RBC 20030229
 612      * from.conn->close() / to.conn->close() done here trigger close callbacks which may free TunnelStateData
 613      */
 614     const CbcPointer<TunnelStateData> safetyLock(this);
 615
 616     /* Bump the source connection read timeout on any activity */
 617     if (Comm::IsConnOpen(from.conn)) {
 618         AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 619                                          CommTimeoutCbPtrFun(tunnelTimeout, this));
 620         commSetConnTimeout(from.conn, Config.Timeout.read, timeoutCall);
 621     }
 622
 623     /* Bump the dest connection read timeout on any activity */
 624     /* see Bug 3659: tunnels can be weird, with very long one-way transfers */
 625     if (Comm::IsConnOpen(to.conn)) {
 626         AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 627                                          CommTimeoutCbPtrFun(tunnelTimeout, this));
 628         commSetConnTimeout(to.conn, Config.Timeout.read, timeoutCall);
 629     }
 630
 631     if (errcode)
 632         from.error (xerrno);
 633     else if (len == 0 || !Comm::IsConnOpen(to.conn)) {
 634         debugs(26, 3, "Nothing to write or client gone. Terminate the tunnel.");
 635         from.conn->close();
 636
 637         /* Only close the remote end if we've finished queueing data to it */
 638         if (from.len == 0 && Comm::IsConnOpen(to.conn) ) {
 639             to.conn->close();
 640         }
 641     } else if (cbdataReferenceValid(this)) {
 642         return true;
 643     }
 644
 645     return false;
 646 }
 647
 648 void
 649 TunnelStateData::copy(size_t len, Connection &from, Connection &to, IOCB *completion)
 650 {
 651     debugs(26, 3, "Schedule Write");
 652     AsyncCall::Pointer call = commCbCall(5,5, "TunnelBlindCopyWriteHandler",
 653                                          CommIoCbPtrFun(completion, this));
 654     to.write(from.buf, len, call, nullptr);
 655 }
 656
 657 /* Writes data from the client buffer to the server side */
 658 void
 659 TunnelStateData::WriteServerDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 660 {
 661     TunnelStateData *tunnelState = (TunnelStateData *)data;
 662     assert (cbdataReferenceValid (tunnelState));
 663     tunnelState->server.writer = nullptr;
 664
 665     tunnelState->writeServerDone(buf, len, flag, xerrno);
 666 }
 667
 668 void
 669 TunnelStateData::writeServerDone(char *, size_t len, Comm::Flag flag, int xerrno)
 670 {
 671     debugs(26, 3, server.conn << ", " << len << " bytes written, flag=" << flag);
 672
 673     if (flag == Comm::ERR_CLOSING)
 674         return;
 675
 676     request->hier.notePeerWrite();
 677
 678     /* Error? */
 679     if (flag != Comm::OK) {
 680         debugs(26, 4, "to-server write failed: " << xerrno);
 681         server.error(xerrno); // may call comm_close
 682         return;
 683     }
 684
 685     /* EOF? */
 686     if (len == 0) {
 687         debugs(26, 4, "No read input. Closing server connection.");
 688         server.conn->close();
 689         return;
 690     }
 691
 692     /* Valid data */
 693     statCounter.server.all.kbytes_out += len;
 694     statCounter.server.other.kbytes_out += len;
 695     client.dataSent(len);
 696
 697     /* If the other end has closed, so should we */
 698     if (!Comm::IsConnOpen(client.conn)) {
 699         debugs(26, 4, "Client gone away. Shutting down server connection.");
 700         server.conn->close();
 701         return;
 702     }
 703
 704     const CbcPointer<TunnelStateData> safetyLock(this); /* ??? should be locked by the caller... */
 705
 706     if (cbdataReferenceValid(this))
 707         copyClientBytes();
 708 }
 709
 710 /* Writes data from the server buffer to the client side */
 711 void
 712 TunnelStateData::WriteClientDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 713 {
 714     TunnelStateData *tunnelState = (TunnelStateData *)data;
 715     assert (cbdataReferenceValid (tunnelState));
 716     tunnelState->client.writer = nullptr;
 717
 718     tunnelState->writeClientDone(buf, len, flag, xerrno);
 719 }
 720
 721 void
 722 TunnelStateData::Connection::dataSent(size_t amount)
 723 {
 724     debugs(26, 3, "len=" << len << " - amount=" << amount);
 725     assert(amount == (size_t)len);
 726     len =0;
 727     /* increment total object size */
 728
 729     if (size_ptr)
 730         *size_ptr += amount;
 731
 732 }
 733
 734 void
 735 TunnelStateData::Connection::write(const char *b, int size, AsyncCall::Pointer &callback, FREE * free_func)
 736 {
 737     writer = callback;
 738     dirty = true;
 739     Comm::Write(conn, b, size, callback, free_func);
 740 }
 741
 742 template <typename Method>
 743 void
 744 TunnelStateData::Connection::initConnection(const Comm::ConnectionPointer &aConn, Method method, const char *name, TunnelStateData *tunnelState)
 745 {
 746     Must(!Comm::IsConnOpen(conn));
 747     Must(!closer);
 748     Must(Comm::IsConnOpen(aConn));
 749     conn = aConn;
 750     closer = commCbCall(5, 4, name, CommCloseCbPtrFun(method, tunnelState));
 751     comm_add_close_handler(conn->fd, closer);
 752 }
 753
 754 void
 755 TunnelStateData::Connection::noteClosure()
 756 {
 757     debugs(26, 3, conn);
 758     conn = nullptr;
 759     closer = nullptr;
 760     writer = nullptr; // may already be nil
 761 }
 762
 763 void
 764 TunnelStateData::writeClientDone(char *, size_t len, Comm::Flag flag, int xerrno)
 765 {
 766     debugs(26, 3, client.conn << ", " << len << " bytes written, flag=" << flag);
 767
 768     if (flag == Comm::ERR_CLOSING)
 769         return;
 770
 771     /* Error? */
 772     if (flag != Comm::OK) {
 773         debugs(26, 4, "from-client read failed: " << xerrno);
 774         client.error(xerrno); // may call comm_close
 775         return;
 776     }
 777
 778     /* EOF? */
 779     if (len == 0) {
 780         debugs(26, 4, "Closing client connection due to 0 byte read.");
 781         client.conn->close();
 782         return;
 783     }
 784
 785     /* Valid data */
 786     statCounter.client_http.kbytes_out += len;
 787     server.dataSent(len);
 788
 789     /* If the other end has closed, so should we */
 790     if (!Comm::IsConnOpen(server.conn)) {
 791         debugs(26, 4, "Server has gone away. Terminating client connection.");
 792         client.conn->close();
 793         return;
 794     }
 795
 796     CbcPointer<TunnelStateData> safetyLock(this);   /* ??? should be locked by the caller... */
 797
 798     if (cbdataReferenceValid(this))
 799         copyServerBytes();
 800 }
 801
 802 static void
 803 tunnelTimeout(const CommTimeoutCbParams &io)
 804 {
 805     TunnelStateData *tunnelState = static_cast<TunnelStateData *>(io.data);
 806     debugs(26, 3, io.conn);
 807     /* Temporary lock to protect our own feets (comm_close -> tunnelClientClosed -> Free) */
 808     CbcPointer<TunnelStateData> safetyLock(tunnelState);
 809
 810     tunnelState->closeConnections();
 811 }
 812
 813 void
 814 TunnelStateData::closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason)
 815 {
 816     debugs(26, 3, "because " << reason << "; " << conn);
 817     assert(!server.conn);
 818     if (IsConnOpen(conn))
 819         conn->close();
 820 }
 821
 822 void
 823 TunnelStateData::closeConnections()
 824 {
 825     if (Comm::IsConnOpen(server.conn))
 826         server.conn->close();
 827     if (Comm::IsConnOpen(client.conn))
 828         client.conn->close();
 829 }
 830
 831 static void
 832 tunnelDelayedClientRead(void *data)
 833 {
 834     if (!data)
 835         return;
 836
 837     TunnelStateData *tunnel = static_cast<TunnelStateData*>(data);
 838     const auto savedContext = CodeContext::Current();
 839     CodeContext::Reset(tunnel->codeContext);
 840     tunnel->client.readPending = nullptr;
 841     static uint64_t counter=0;
 842     debugs(26, 7, "Client read(2) delayed " << ++counter << " times");
 843     tunnel->copyRead(tunnel->client, TunnelStateData::ReadClient);
 844     CodeContext::Reset(savedContext);
 845 }
 846
 847 static void
 848 tunnelDelayedServerRead(void *data)
 849 {
 850     if (!data)
 851         return;
 852
 853     TunnelStateData *tunnel = static_cast<TunnelStateData*>(data);
 854     const auto savedContext = CodeContext::Current();
 855     CodeContext::Reset(tunnel->codeContext);
 856     tunnel->server.readPending = nullptr;
 857     static uint64_t counter=0;
 858     debugs(26, 7, "Server read(2) delayed " << ++counter << " times");
 859     tunnel->copyRead(tunnel->server, TunnelStateData::ReadServer);
 860     CodeContext::Reset(savedContext);
 861 }
 862
 863 void
 864 TunnelStateData::copyRead(Connection &from, IOCB *completion)
 865 {
 866     assert(from.len == 0);
 867     // If only the minimum permitted read size is going to be attempted
 868     // then we schedule an event to try again in a few I/O cycles.
 869     // Allow at least 1 byte to be read every (0.3*10) seconds.
 870     int bw = from.bytesWanted(1, SQUID_TCP_SO_RCVBUF);
 871     // XXX: Delay pools must not delay client-to-Squid traffic (i.e. when
 872     // from.readPendingFunc is tunnelDelayedClientRead()).
 873     // XXX: Bug #4913: For delay pools, use delayRead() API instead.
 874     if (bw == 1 && ++from.delayedLoops < 10) {
 875         from.readPending = this;
 876         eventAdd("tunnelDelayedServerRead", from.readPendingFunc, from.readPending, 0.3, true);
 877         return;
 878     }
 879
 880     AsyncCall::Pointer call = commCbCall(5,4, "TunnelBlindCopyReadHandler",
 881                                          CommIoCbPtrFun(completion, this));
 882     comm_read(from.conn, from.buf, bw, call);
 883 }
 884
 885 void
 886 TunnelStateData::copyClientBytes()
 887 {
 888     if (preReadClientData.length()) {
 889         size_t copyBytes = preReadClientData.length() > SQUID_TCP_SO_RCVBUF ? SQUID_TCP_SO_RCVBUF : preReadClientData.length();
 890         memcpy(client.buf, preReadClientData.rawContent(), copyBytes);
 891         preReadClientData.consume(copyBytes);
 892         client.bytesIn(copyBytes);
 893         if (keepGoingAfterRead(copyBytes, Comm::OK, 0, client, server))
 894             copy(copyBytes, client, server, TunnelStateData::WriteServerDone);
 895     } else
 896         copyRead(client, ReadClient);
 897 }
 898
 899 void
 900 TunnelStateData::copyServerBytes()
 901 {
 902     if (preReadServerData.length()) {
 903         size_t copyBytes = preReadServerData.length() > SQUID_TCP_SO_RCVBUF ? SQUID_TCP_SO_RCVBUF : preReadServerData.length();
 904         memcpy(server.buf, preReadServerData.rawContent(), copyBytes);
 905         preReadServerData.consume(copyBytes);
 906         server.bytesIn(copyBytes);
 907         if (keepGoingAfterRead(copyBytes, Comm::OK, 0, server, client))
 908             copy(copyBytes, server, client, TunnelStateData::WriteClientDone);
 909     } else
 910         copyRead(server, ReadServer);
 911 }
 912
 913 /**
 914  * Set the HTTP status for this request and sets the read handlers for client
 915  * and server side connections.
 916  */
 917 static void
 918 tunnelStartShoveling(TunnelStateData *tunnelState)
 919 {
 920     assert(!tunnelState->transportWait);
 921     assert(!tunnelState->encryptionWait);
 922     assert(!tunnelState->peerWait);
 923
 924     assert(tunnelState->server.conn);
 925     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 926                                      CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
 927     commSetConnTimeout(tunnelState->server.conn, Config.Timeout.read, timeoutCall);
 928
 929     *tunnelState->status_ptr = Http::scOkay;
 930     tunnelState->al->cache.code.update(LOG_TCP_TUNNEL);
 931     if (cbdataReferenceValid(tunnelState)) {
 932
 933         // Shovel any payload already pushed into reply buffer by the server response
 934         if (!tunnelState->server.len)
 935             tunnelState->copyServerBytes();
 936         else {
 937             debugs(26, DBG_DATA, "Tunnel server PUSH Payload: \n" << Raw("", tunnelState->server.buf, tunnelState->server.len) << "\n----------");
 938             tunnelState->copy(tunnelState->server.len, tunnelState->server, tunnelState->client, TunnelStateData::WriteClientDone);
 939         }
 940
 941         if (tunnelState->http.valid() && tunnelState->http->getConn() && !tunnelState->http->getConn()->inBuf.isEmpty()) {
 942             SBuf * const in = &tunnelState->http->getConn()->inBuf;
 943             debugs(26, DBG_DATA, "Tunnel client PUSH Payload: \n" << *in << "\n----------");
 944             tunnelState->preReadClientData.append(*in);
 945             in->consume(); // ConnStateData buffer accounting after the shuffle.
 946         }
 947         tunnelState->copyClientBytes();
 948     }
 949 }
 950
 951 /**
 952  * All the pieces we need to write to client and/or server connection
 953  * have been written.
 954  * Call the tunnelStartShoveling to start the blind pump.
 955  */
 956 static void
 957 tunnelConnectedWriteDone(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
 958 {
 959     TunnelStateData *tunnelState = (TunnelStateData *)data;
 960     debugs(26, 3, conn << ", flag=" << flag);
 961     tunnelState->client.writer = nullptr;
 962
 963     if (flag != Comm::OK) {
 964         *tunnelState->status_ptr = Http::scInternalServerError;
 965         tunnelErrorComplete(conn->fd, data, 0);
 966         return;
 967     }
 968
 969     if (auto http = tunnelState->http.get()) {
 970         http->out.headers_sz += len;
 971         http->out.size += len;
 972     }
 973
 974     tunnelStartShoveling(tunnelState);
 975 }
 976
 977 void
 978 TunnelStateData::tunnelEstablishmentDone(Http::TunnelerAnswer &answer)
 979 {
 980     peerWait.finish();
 981     server.len = 0;
 982
 983     al->cache.code.update(LOG_TCP_TUNNEL);
 984
 985     // XXX: al->http.code (i.e. *status_ptr) should not be (re)set
 986     // until we actually start responding to the client. Right here/now, we only
 987     // know how this cache_peer has responded to us.
 988     if (answer.peerResponseStatus != Http::scNone)
 989         *status_ptr = answer.peerResponseStatus;
 990
 991     auto sawProblem = false;
 992
 993     if (!answer.positive()) {
 994         sawProblem = true;
 995         assert(!answer.conn);
 996     } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
 997         sawProblem = true;
 998         closePendingConnection(answer.conn, "conn was closed while waiting for tunnelEstablishmentDone");
 999     }
1000
1001     if (!sawProblem) {
1002         assert(answer.positive()); // paranoid
1003         // copy any post-200 OK bytes to our buffer
1004         preReadServerData = answer.leftovers;
1005         notePeerReadyToShovel(answer.conn);
1006         return;
1007     }
1008
1009     ErrorState *error = nullptr;
1010     if (answer.positive()) {
1011         error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
1012     } else {
1013         error = answer.squidError.get();
1014         Must(error);
1015         answer.squidError.clear(); // preserve error for errorSendComplete()
1016     }
1017     assert(error);
1018     saveError(error);
1019     retryOrBail("tunneler error");
1020 }
1021
1022 void
1023 TunnelStateData::notePeerReadyToShovel(const Comm::ConnectionPointer &conn)
1024 {
1025     assert(!client.dirty);
1026     commitToServer(conn);
1027
1028     if (!clientExpectsConnectResponse())
1029         tunnelStartShoveling(this); // ssl-bumped connection, be quiet
1030     else {
1031         *status_ptr = Http::scOkay;
1032         AsyncCall::Pointer call = commCbCall(5,5, "tunnelConnectedWriteDone",
1033                                              CommIoCbPtrFun(tunnelConnectedWriteDone, this));
1034         al->reply = HttpReply::MakeConnectionEstablished();
1035         const auto mb = al->reply->pack();
1036         client.write(mb->content(), mb->contentSize(), call, mb->freeFunc());
1037         delete mb;
1038     }
1039 }
1040
1041 void
1042 TunnelStateData::commitToServer(const Comm::ConnectionPointer &conn)
1043 {
1044     committedToServer = true;
1045     banRetries = "committed to server";
1046     PeerSelectionInitiator::subscribed = false; // may already be false
1047     server.initConnection(conn, tunnelServerClosed, "tunnelServerClosed", this);
1048 }
1049
1050 static void
1051 tunnelErrorComplete(int fd/*const Comm::ConnectionPointer &*/, void *data, size_t)
1052 {
1053     TunnelStateData *tunnelState = (TunnelStateData *)data;
1054     debugs(26, 3, "FD " << fd);
1055     assert(tunnelState != nullptr);
1056     /* temporary lock to save our own feets (comm_close -> tunnelClientClosed -> Free) */
1057     CbcPointer<TunnelStateData> safetyLock(tunnelState);
1058
1059     if (Comm::IsConnOpen(tunnelState->client.conn))
1060         tunnelState->client.conn->close();
1061
1062     if (Comm::IsConnOpen(tunnelState->server.conn))
1063         tunnelState->server.conn->close();
1064 }
1065
1066 void
1067 TunnelStateData::noteConnection(HappyConnOpener::Answer &answer)
1068 {
1069     transportWait.finish();
1070
1071     updateAttempts(answer.n_tries);
1072
1073     ErrorState *error = nullptr;
1074     if ((error = answer.error.get())) {
1075         banRetries = "HappyConnOpener gave up";
1076         Must(!Comm::IsConnOpen(answer.conn));
1077         syncHierNote(answer.conn, request->url.host());
1078         answer.error.clear();
1079     } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
1080         error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
1081         closePendingConnection(answer.conn, "conn was closed while waiting for  noteConnection");
1082     }
1083
1084     if (error) {
1085         saveError(error);
1086         retryOrBail("tried all destinations");
1087         return;
1088     }
1089
1090     connectDone(answer.conn, request->url.host(), answer.reused);
1091 }
1092
1093 void
1094 TunnelStateData::connectDone(const Comm::ConnectionPointer &conn, const char *origin, const bool reused)
1095 {
1096     Must(Comm::IsConnOpen(conn));
1097
1098     if (reused)
1099         ResetMarkingsToServer(request.getRaw(), *conn);
1100     // else Comm::ConnOpener already applied proper/current markings
1101
1102     // TODO: add pconn race state tracking
1103
1104     syncHierNote(conn, origin);
1105
1106 #if USE_DELAY_POOLS
1107     /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
1108     if (conn->getPeer() && conn->getPeer()->options.no_delay)
1109         server.setDelayId(DelayId());
1110 #endif
1111
1112     netdbPingSite(request->url.host());
1113
1114     request->peer_host = conn->getPeer() ? conn->getPeer()->host : nullptr;
1115
1116     bool toOrigin = false; // same semantics as StateFlags::toOrigin
1117     if (const auto * const peer = conn->getPeer()) {
1118         request->prepForPeering(*peer);
1119         toOrigin = peer->options.originserver;
1120     } else {
1121         request->prepForDirect();
1122         toOrigin = true;
1123     }
1124
1125     if (!toOrigin)
1126         connectToPeer(conn);
1127     else {
1128         notePeerReadyToShovel(conn);
1129     }
1130 }
1131
1132 /// whether we have used up all permitted forwarding attempts
1133 bool
1134 TunnelStateData::exhaustedTries() const
1135 {
1136     return n_tries >= Config.forward_max_tries;
1137 }
1138
1139 void
1140 tunnelStart(ClientHttpRequest * http)
1141 {
1142     debugs(26, 3, MYNAME);
1143     /* Create state structure. */
1144     TunnelStateData *tunnelState = nullptr;
1145     ErrorState *err = nullptr;
1146     HttpRequest *request = http->request;
1147     char *url = http->uri;
1148
1149     /*
1150      * client_addr.isNoAddr()  indicates this is an "internal" request
1151      * from peer_digest.c, asn.c, netdb.c, etc and should always
1152      * be allowed.  yuck, I know.
1153      */
1154
1155     if (Config.accessList.miss && !request->client_addr.isNoAddr()) {
1156         /*
1157          * Check if this host is allowed to fetch MISSES from us (miss_access)
1158          * default is to allow.
1159          */
1160         ACLFilledChecklist ch(Config.accessList.miss, request, nullptr);
1161         ch.al = http->al;
1162         ch.src_addr = request->client_addr;
1163         ch.my_addr = request->my_addr;
1164         ch.syncAle(request, http->log_uri);
1165         if (ch.fastCheck().denied()) {
1166             debugs(26, 4, "MISS access forbidden.");
1167             err = new ErrorState(ERR_FORWARDING_DENIED, Http::scForbidden, request, http->al);
1168             http->al->http.code = Http::scForbidden;
1169             errorSend(http->getConn()->clientConnection, err);
1170             return;
1171         }
1172     }
1173
1174     debugs(26, 3, request->method << ' ' << url << ' ' << request->http_ver);
1175     ++statCounter.server.all.requests;
1176     ++statCounter.server.other.requests;
1177
1178     tunnelState = new TunnelStateData(http);
1179 #if USE_DELAY_POOLS
1180     tunnelState->server.setDelayId(DelayId::DelayClient(http));
1181 #endif
1182     tunnelState->startSelectingDestinations(request, http->al, nullptr);
1183 }
1184
1185 void
1186 TunnelStateData::connectToPeer(const Comm::ConnectionPointer &conn)
1187 {
1188     if (const auto p = conn->getPeer()) {
1189         if (p->secure.encryptTransport)
1190             return advanceDestination("secure connection to peer", conn, [this,&conn] {
1191             secureConnectionToPeer(conn);
1192         });
1193     }
1194
1195     connectedToPeer(conn);
1196 }
1197
1198 /// encrypts an established TCP connection to peer
1199 void
1200 TunnelStateData::secureConnectionToPeer(const Comm::ConnectionPointer &conn)
1201 {
1202     const auto callback = asyncCallback(5, 4, TunnelStateData::noteSecurityPeerConnectorAnswer, this);
1203     const auto connector = new Security::BlindPeerConnector(request, conn, callback, al);
1204     encryptionWait.start(connector, callback);
1205 }
1206
1207 /// starts a preparation step for an established connection; retries on failures
1208 template <typename StepStart>
1209 void
1210 TunnelStateData::advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep)
1211 {
1212     // TODO: Extract destination-specific handling from TunnelStateData so that
1213     // all the awkward, limited-scope advanceDestination() calls can be replaced
1214     // with a single simple try/catch,retry block.
1215     try {
1216         startStep();
1217         // now wait for the step callback
1218     } catch (...) {
1219         debugs (26, 2, "exception while trying to " << stepDescription << ": " << CurrentException);
1220         closePendingConnection(conn, "connection preparation exception");
1221         if (!savedError)
1222             saveError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al));
1223         retryOrBail(stepDescription);
1224     }
1225 }
1226
1227 /// callback handler for the connection encryptor
1228 void
1229 TunnelStateData::noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &answer)
1230 {
1231     encryptionWait.finish();
1232
1233     ErrorState *error = nullptr;
1234     assert(!answer.tunneled);
1235     if ((error = answer.error.get())) {
1236         assert(!answer.conn);
1237         answer.error.clear();
1238     } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
1239         error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
1240         closePendingConnection(answer.conn, "conn was closed while waiting for noteSecurityPeerConnectorAnswer");
1241     }
1242
1243     if (error) {
1244         saveError(error);
1245         retryOrBail("TLS peer connection error");
1246         return;
1247     }
1248
1249     connectedToPeer(answer.conn);
1250 }
1251
1252 void
1253 TunnelStateData::connectedToPeer(const Comm::ConnectionPointer &conn)
1254 {
1255     advanceDestination("establish tunnel through proxy", conn, [this,&conn] {
1256         establishTunnelThruProxy(conn);
1257     });
1258 }
1259
1260 void
1261 TunnelStateData::establishTunnelThruProxy(const Comm::ConnectionPointer &conn)
1262 {
1263     const auto callback = asyncCallback(5, 4, TunnelStateData::tunnelEstablishmentDone, this);
1264     const auto tunneler = new Http::Tunneler(conn, request, callback, Config.Timeout.lifetime, al);
1265 #if USE_DELAY_POOLS
1266     tunneler->setDelayId(server.delayId);
1267 #endif
1268     peerWait.start(tunneler, callback);
1269 }
1270
1271 void
1272 TunnelStateData::noteDestination(Comm::ConnectionPointer path)
1273 {
1274     destinationsFound = true;
1275
1276     if (!path) { // decided to use a pinned connection
1277         // We can call usePinned() without fear of clashing with an earlier
1278         // forwarding attempt because PINNED must be the first destination.
1279         assert(destinations->empty());
1280         usePinned();
1281         return;
1282     }
1283
1284     destinations->addPath(path);
1285
1286     if (transportWait) {
1287         assert(!transporting());
1288         notifyConnOpener();
1289         return; // and continue to wait for tunnelConnectDone() callback
1290     }
1291
1292     if (transporting())
1293         return; // and continue to receive destinations for backup
1294
1295     startConnecting();
1296 }
1297
1298 void
1299 TunnelStateData::noteDestinationsEnd(ErrorState *selectionError)
1300 {
1301     PeerSelectionInitiator::subscribed = false;
1302     destinations->destinationsFinalized = true;
1303     if (!destinationsFound) {
1304
1305         // XXX: Honor clientExpectsConnectResponse() before replying.
1306
1307         if (selectionError)
1308             return sendError(selectionError, "path selection has failed");
1309
1310         // TODO: Merge with FwdState and remove this likely unnecessary check.
1311         if (savedError)
1312             return sendError(savedError, "path selection found no paths (with an impossible early error)");
1313
1314         return sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al),
1315                          "path selection found no paths");
1316     }
1317     // else continue to use one of the previously noted destinations;
1318     // if all of them fail, tunneling as whole will fail
1319     Must(!selectionError); // finding at least one path means selection succeeded
1320
1321     if (transportWait) {
1322         assert(!transporting());
1323         notifyConnOpener();
1324         return; // and continue to wait for the noteConnection() callback
1325     }
1326
1327     if (transporting()) {
1328         // We are already using a previously opened connection (but were also
1329         // receiving more destinations in case we need to re-forward).
1330         debugs(17, 7, "keep transporting");
1331         return;
1332     }
1333
1334     // destinationsFound, but none of them worked, and we were waiting for more
1335     assert(savedError);
1336     // XXX: Honor clientExpectsConnectResponse() before replying.
1337     sendError(savedError, "all found paths have failed");
1338 }
1339
1340 /// Whether a tunneling attempt to some selected destination X is in progress
1341 /// (after successfully opening/reusing a transport connection to X).
1342 /// \sa transportWait
1343 bool
1344 TunnelStateData::transporting() const
1345 {
1346     return encryptionWait || peerWait || committedToServer;
1347 }
1348
1349 /// remembers an error to be used if there will be no more connection attempts
1350 void
1351 TunnelStateData::saveError(ErrorState *error)
1352 {
1353     debugs(26, 4, savedError << " ? " << error);
1354     assert(error);
1355     delete savedError; // may be nil
1356     savedError = error;
1357 }
1358
1359 /// Starts sending the given error message to the client, leading to the
1360 /// eventual transaction termination. Call with savedError to send savedError.
1361 void
1362 TunnelStateData::sendError(ErrorState *finalError, const char *reason)
1363 {
1364     debugs(26, 3, "aborting transaction for " << reason);
1365
1366     if (request)
1367         request->hier.stopPeerClock(false);
1368
1369     cancelStep(reason);
1370
1371     assert(finalError);
1372
1373     // get rid of any cached error unless that is what the caller is sending
1374     if (savedError != finalError)
1375         delete savedError; // may be nil
1376     savedError = nullptr;
1377
1378     // we cannot try other destinations after responding with an error
1379     PeerSelectionInitiator::subscribed = false; // may already be false
1380
1381     *status_ptr = finalError->httpStatus;
1382     finalError->callback = tunnelErrorComplete;
1383     finalError->callback_data = this;
1384     errorSend(client.conn, finalError);
1385 }
1386
1387 /// Notify a pending subtask, if any, that we no longer need its help. We do not
1388 /// have to do this -- the subtask job will eventually end -- but ending it
1389 /// earlier reduces waste and may reduce DoS attack surface.
1390 void
1391 TunnelStateData::cancelStep(const char *reason)
1392 {
1393     transportWait.cancel(reason);
1394     encryptionWait.cancel(reason);
1395     peerWait.cancel(reason);
1396 }
1397
1398 void
1399 TunnelStateData::startConnecting()
1400 {
1401     if (request)
1402         request->hier.startPeerClock();
1403
1404     assert(!destinations->empty());
1405     assert(!transporting());
1406
1407     delete savedError; // may still be nil
1408     savedError = nullptr;
1409     request->hier.peer_reply_status = Http::scNone; // TODO: Move to startPeerClock()?
1410
1411     const auto callback = asyncCallback(17, 5, TunnelStateData::noteConnection, this);
1412     const auto cs = new HappyConnOpener(destinations, callback, request, startTime, n_tries, al);
1413     cs->setHost(request->url.host());
1414     cs->setRetriable(false);
1415     cs->allowPersistent(false);
1416     destinations->notificationPending = true; // start() is async
1417     transportWait.start(cs, callback);
1418 }
1419
1420 /// send request on an existing connection dedicated to the requesting client
1421 void
1422 TunnelStateData::usePinned()
1423 {
1424     Must(request);
1425     const auto connManager = request->pinnedConnection();
1426     try {
1427         const auto serverConn = ConnStateData::BorrowPinnedConnection(request.getRaw(), al);
1428         debugs(26, 7, "pinned peer connection: " << serverConn);
1429
1430         updateAttempts(n_tries + 1);
1431
1432         // Set HttpRequest pinned related flags for consistency even if
1433         // they are not really used by tunnel.cc code.
1434         request->flags.pinned = true;
1435         if (connManager->pinnedAuth())
1436             request->flags.auth = true;
1437
1438         // the server may close the pinned connection before this request
1439         const auto reused = true;
1440         connectDone(serverConn, connManager->pinning.host, reused);
1441     } catch (ErrorState * const error) {
1442         syncHierNote(nullptr, connManager ? connManager->pinning.host : request->url.host());
1443         // XXX: Honor clientExpectsConnectResponse() before replying.
1444         // a PINNED path failure is fatal; do not wait for more paths
1445         sendError(error, "pinned path failure");
1446         return;
1447     }
1448
1449 }
1450
1451 CBDATA_CLASS_INIT(TunnelStateData);
1452
1453 bool
1454 TunnelStateData::noConnections() const
1455 {
1456     return !Comm::IsConnOpen(server.conn) && !Comm::IsConnOpen(client.conn);
1457 }
1458
1459 #if USE_DELAY_POOLS
1460 void
1461 TunnelStateData::Connection::setDelayId(DelayId const &newDelay)
1462 {
1463     delayId = newDelay;
1464 }
1465
1466 #endif
1467
1468 /// makes sure connection opener knows that the destinations have changed
1469 void
1470 TunnelStateData::notifyConnOpener()
1471 {
1472     if (destinations->notificationPending) {
1473         debugs(17, 7, "reusing pending notification");
1474     } else {
1475         destinations->notificationPending = true;
1476         CallJobHere(17, 5, transportWait.job(), HappyConnOpener, noteCandidatesChange);
1477     }
1478 }
1479
1480 /**
1481  * Sets up a TCP tunnel through Squid and starts shoveling traffic.
1482  * \param request the request that initiated/caused this tunnel
1483  * \param clientConn the already accepted client-to-Squid TCP connection
1484  * \param srvConn the already established Squid-to-server TCP connection
1485  * \param preReadServerData server-sent bytes to be forwarded to the client
1486  */
1487 void
1488 switchToTunnel(HttpRequest *request, const Comm::ConnectionPointer &clientConn, const Comm::ConnectionPointer &srvConn, const SBuf &preReadServerData)
1489 {
1490     Must(Comm::IsConnOpen(clientConn));
1491     Must(Comm::IsConnOpen(srvConn));
1492
1493     debugs(26,5, "Revert to tunnel FD " << clientConn->fd << " with FD " << srvConn->fd);
1494
1495     /* Create state structure. */
1496     ++statCounter.server.all.requests;
1497     ++statCounter.server.other.requests;
1498
1499     auto conn = request->clientConnectionManager.get();
1500     Must(conn);
1501     Http::StreamPointer context = conn->pipeline.front();
1502     Must(context && context->http);
1503
1504     debugs(26, 3, request->method << " " << context->http->uri << " " << request->http_ver);
1505
1506     TunnelStateData *tunnelState = new TunnelStateData(context->http);
1507     tunnelState->commitToServer(srvConn);
1508
1509     request->hier.resetPeerNotes(srvConn, tunnelState->getHost());
1510
1511 #if USE_DELAY_POOLS
1512     /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
1513     if (!srvConn->getPeer() || !srvConn->getPeer()->options.no_delay)
1514         tunnelState->server.setDelayId(DelayId::DelayClient(context->http));
1515 #endif
1516
1517     request->peer_host = srvConn->getPeer() ? srvConn->getPeer()->host : nullptr;
1518
1519     debugs(26, 4, "determine post-connect handling pathway.");
1520     if (const auto peer = srvConn->getPeer())
1521         request->prepForPeering(*peer);
1522     else
1523         request->prepForDirect();
1524
1525     tunnelState->preReadServerData = preReadServerData;
1526
1527     tunnelStartShoveling(tunnelState);
1528 }
1529