src/tunnel.cc

   1 /*
   2  * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 26    Secure Sockets Layer Proxy */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/AsyncCallbacks.h"
  14 #include "base/CbcPointer.h"
  15 #include "base/JobWait.h"
  16 #include "base/Raw.h"
  17 #include "CachePeer.h"
  18 #include "cbdata.h"
  19 #include "client_side.h"
  20 #include "client_side_request.h"
  21 #include "clients/HttpTunneler.h"
  22 #include "comm.h"
  23 #include "comm/Connection.h"
  24 #include "comm/ConnOpener.h"
  25 #include "comm/Read.h"
  26 #include "comm/Write.h"
  27 #include "errorpage.h"
  28 #include "fd.h"
  29 #include "fde.h"
  30 #include "FwdState.h"
  31 #include "globals.h"
  32 #include "HappyConnOpener.h"
  33 #include "http.h"
  34 #include "http/StatusCode.h"
  35 #include "http/Stream.h"
  36 #include "HttpRequest.h"
  37 #include "icmp/net_db.h"
  38 #include "ip/QosConfig.h"
  39 #include "LogTags.h"
  40 #include "MemBuf.h"
  41 #include "neighbors.h"
  42 #include "PeerSelectState.h"
  43 #include "ResolvedPeers.h"
  44 #include "sbuf/SBuf.h"
  45 #include "security/BlindPeerConnector.h"
  46 #include "SquidConfig.h"
  47 #include "StatCounters.h"
  48 #if USE_OPENSSL
  49 #include "ssl/bio.h"
  50 #include "ssl/ServerBump.h"
  51 #endif
  52 #include "tools.h"
  53 #include "tunnel.h"
  54 #if USE_DELAY_POOLS
  55 #include "DelayId.h"
  56 #endif
  57
  58 #include <climits>
  59 #include <cerrno>
  60
  61 /**
  62  * TunnelStateData is the state engine performing the tasks for
  63  * setup of a TCP tunnel from an existing open client FD to a server
  64  * then shuffling binary data between the resulting FD pair.
  65  */
  66 /*
  67  * TODO 1: implement a read/write API on ConnStateData to send/receive blocks
  68  * of pre-formatted data. Then we can use that as the client side of the tunnel
  69  * instead of re-implementing it here and occasionally getting the ConnStateData
  70  * read/write state wrong.
  71  *
  72  * TODO 2: then convert this into a AsyncJob, possibly a child of 'Server'
  73  */
  74 class TunnelStateData: public PeerSelectionInitiator
  75 {
  76     CBDATA_CHILD(TunnelStateData);
  77
  78 public:
  79     TunnelStateData(ClientHttpRequest *);
  80     ~TunnelStateData() override;
  81     TunnelStateData(const TunnelStateData &); // do not implement
  82     TunnelStateData &operator =(const TunnelStateData &); // do not implement
  83
  84     class Connection;
  85     static void ReadClient(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data);
  86     static void ReadServer(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data);
  87     static void WriteClientDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data);
  88     static void WriteServerDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data);
  89
  90     bool noConnections() const;
  91     /// closes both client and server connections
  92     void closeConnections();
  93
  94     char *url;
  95     CbcPointer<ClientHttpRequest> http;
  96     HttpRequest::Pointer request;
  97     AccessLogEntryPointer al;
  98
  99     const char * getHost() const {
 100         return (server.conn != nullptr && server.conn->getPeer() ? server.conn->getPeer()->host : request->url.host());
 101     };
 102
 103     /// store the given to-server connection; prohibit retries and do not look
 104     /// for any other destinations
 105     void commitToServer(const Comm::ConnectionPointer &);
 106
 107     /// Whether the client sent a CONNECT request to us.
 108     bool clientExpectsConnectResponse() const {
 109         // If we are forcing a tunnel after receiving a client CONNECT, then we
 110         // have already responded to that CONNECT before tunnel.cc started.
 111         if (request && request->flags.forceTunnel)
 112             return false;
 113 #if USE_OPENSSL
 114         // We are bumping and we had already send "OK CONNECTED"
 115         if (http.valid() && http->getConn() && http->getConn()->serverBump() && http->getConn()->serverBump()->at(XactionStep::tlsBump2, XactionStep::tlsBump3))
 116             return false;
 117 #endif
 118         return !(request != nullptr &&
 119                  (request->flags.interceptTproxy || request->flags.intercepted));
 120     }
 121
 122     /// starts connecting to the next hop, either for the first time or while
 123     /// recovering from the previous connect failure
 124     void startConnecting();
 125     void closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason);
 126
 127     /// called when negotiations with the peer have been successfully completed
 128     void notePeerReadyToShovel(const Comm::ConnectionPointer &);
 129
 130     class Connection
 131     {
 132
 133     public:
 134         Connection() : len (0), buf ((char *)xmalloc(SQUID_TCP_SO_RCVBUF)), size_ptr(nullptr), delayedLoops(0),
 135             dirty(false),
 136             readPending(nullptr), readPendingFunc(nullptr) {}
 137
 138         ~Connection();
 139
 140         /// initiates Comm::Connection ownership, including closure monitoring
 141         template <typename Method>
 142         void initConnection(const Comm::ConnectionPointer &aConn, Method method, const char *name, TunnelStateData *tunnelState);
 143
 144         /// reacts to the external closure of our connection
 145         void noteClosure();
 146
 147         int bytesWanted(int lower=0, int upper = INT_MAX) const;
 148         void bytesIn(int const &);
 149 #if USE_DELAY_POOLS
 150
 151         void setDelayId(DelayId const &);
 152 #endif
 153
 154         void error(int const xerrno);
 155         int debugLevelForError(int const xerrno) const;
 156
 157         void dataSent (size_t amount);
 158         /// writes 'b' buffer, setting the 'writer' member to 'callback'.
 159         void write(const char *b, int size, AsyncCall::Pointer &callback, FREE * free_func);
 160         int len;
 161         char *buf;
 162         AsyncCall::Pointer writer; ///< pending Comm::Write callback
 163         uint64_t *size_ptr;      /* pointer to size in an ConnStateData for logging */
 164
 165         Comm::ConnectionPointer conn;    ///< The currently connected connection.
 166         uint8_t delayedLoops; ///< how many times a read on this connection has been postponed.
 167
 168         bool dirty; ///< whether write() has been called (at least once)
 169
 170         // XXX: make these an AsyncCall when event API can handle them
 171         TunnelStateData *readPending;
 172         EVH *readPendingFunc;
 173
 174 #if USE_DELAY_POOLS
 175
 176         DelayId delayId;
 177 #endif
 178
 179     private:
 180         /// the registered close handler for the connection
 181         AsyncCall::Pointer closer;
 182     };
 183
 184     Connection client, server;
 185     int *status_ptr;        ///< pointer for logging HTTP status
 186
 187     SBuf preReadClientData;
 188     SBuf preReadServerData;
 189     time_t startTime; ///< object creation time, before any peer selection/connection attempts
 190     ResolvedPeersPointer destinations; ///< paths for forwarding the request
 191     bool destinationsFound; ///< At least one candidate path found
 192
 193     /// whether the decision to tunnel to a particular destination was final
 194     bool committedToServer;
 195
 196     int n_tries; ///< the number of forwarding attempts so far
 197
 198     /// a reason to ban reforwarding attempts (or nil)
 199     const char *banRetries;
 200
 201     // TODO: remove after fixing deferred reads in TunnelStateData::copyRead()
 202     CodeContext::Pointer codeContext; ///< our creator context
 203
 204     /// waits for a transport connection to the peer to be established/opened
 205     JobWait<HappyConnOpener> transportWait;
 206
 207     /// waits for the established transport connection to be secured/encrypted
 208     JobWait<Security::PeerConnector> encryptionWait;
 209
 210     /// waits for an HTTP CONNECT tunnel through a cache_peer to be negotiated
 211     /// over the (encrypted, if needed) transport connection to that cache_peer
 212     JobWait<Http::Tunneler> peerWait;
 213
 214     void copyRead(Connection &from, IOCB *completion);
 215
 216     /// continue to set up connection to a peer, going async for SSL peers
 217     void connectToPeer(const Comm::ConnectionPointer &);
 218     void secureConnectionToPeer(const Comm::ConnectionPointer &);
 219
 220     /* PeerSelectionInitiator API */
 221     void noteDestination(Comm::ConnectionPointer conn) override;
 222     void noteDestinationsEnd(ErrorState *selectionError) override;
 223
 224     void syncHierNote(const Comm::ConnectionPointer &server, const char *origin);
 225
 226     /// called when a connection has been successfully established or
 227     /// when all candidate destinations have been tried and all have failed
 228     void noteConnection(HappyConnOpenerAnswer &);
 229
 230     /// Start using an established connection
 231     void connectDone(const Comm::ConnectionPointer &conn, const char *origin, const bool reused);
 232
 233     void notifyConnOpener();
 234
 235     void saveError(ErrorState *finalError);
 236     void sendError(ErrorState *finalError, const char *reason);
 237
 238 private:
 239     void usePinned();
 240
 241     /// callback handler for the Security::PeerConnector encryptor
 242     void noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &);
 243
 244     /// called after connection setup (including any encryption)
 245     void connectedToPeer(const Comm::ConnectionPointer &);
 246     void establishTunnelThruProxy(const Comm::ConnectionPointer &);
 247
 248     template <typename StepStart>
 249     void advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep);
 250
 251     /// \returns whether the request should be retried (nil) or the description why it should not
 252     const char *checkRetry();
 253
 254     bool transporting() const;
 255
 256     // TODO: convert to unique_ptr
 257     /// details of the "last tunneling attempt" failure (if it failed)
 258     ErrorState *savedError = nullptr;
 259
 260     /// resumes operations after the (possibly failed) HTTP CONNECT exchange
 261     void tunnelEstablishmentDone(Http::TunnelerAnswer &answer);
 262
 263     void deleteThis();
 264
 265     void cancelStep(const char *reason);
 266
 267     bool exhaustedTries() const;
 268     void updateAttempts(int);
 269
 270 public:
 271     bool keepGoingAfterRead(size_t len, Comm::Flag errcode, int xerrno, Connection &from, Connection &to);
 272     void copy(size_t len, Connection &from, Connection &to, IOCB *);
 273     void readServer(char *buf, size_t len, Comm::Flag errcode, int xerrno);
 274     void readClient(char *buf, size_t len, Comm::Flag errcode, int xerrno);
 275     void writeClientDone(char *buf, size_t len, Comm::Flag flag, int xerrno);
 276     void writeServerDone(char *buf, size_t len, Comm::Flag flag, int xerrno);
 277
 278     void copyClientBytes();
 279     void copyServerBytes();
 280
 281     /// handles client-to-Squid connection closure; may destroy us
 282     void clientClosed();
 283
 284     /// handles Squid-to-server connection closure; may destroy us
 285     void serverClosed();
 286
 287     /// tries connecting to another destination, if available,
 288     /// otherwise, initiates the transaction termination
 289     void retryOrBail(const char *context);
 290 };
 291
 292 static ERCB tunnelErrorComplete;
 293 static CLCB tunnelServerClosed;
 294 static CLCB tunnelClientClosed;
 295 static CTCB tunnelTimeout;
 296 static EVH tunnelDelayedClientRead;
 297 static EVH tunnelDelayedServerRead;
 298
 299 /// TunnelStateData::serverClosed() wrapper
 300 static void
 301 tunnelServerClosed(const CommCloseCbParams &params)
 302 {
 303     const auto tunnelState = reinterpret_cast<TunnelStateData *>(params.data);
 304     tunnelState->serverClosed();
 305 }
 306
 307 void
 308 TunnelStateData::serverClosed()
 309 {
 310     server.noteClosure();
 311
 312     retryOrBail(__FUNCTION__);
 313 }
 314
 315 /// TunnelStateData::clientClosed() wrapper
 316 static void
 317 tunnelClientClosed(const CommCloseCbParams &params)
 318 {
 319     const auto tunnelState = reinterpret_cast<TunnelStateData *>(params.data);
 320     tunnelState->clientClosed();
 321 }
 322
 323 void
 324 TunnelStateData::clientClosed()
 325 {
 326     client.noteClosure();
 327
 328     if (noConnections())
 329         return deleteThis();
 330
 331     if (!server.writer)
 332         server.conn->close();
 333 }
 334
 335 /// destroys the tunnel (after performing potentially-throwing cleanup)
 336 void
 337 TunnelStateData::deleteThis()
 338 {
 339     assert(noConnections());
 340     // ConnStateData pipeline should contain the CONNECT we are performing
 341     // but it may be invalid already (bug 4392)
 342     if (const auto h = http.valid()) {
 343         if (const auto c = h->getConn())
 344             if (const auto ctx = c->pipeline.front())
 345                 ctx->finished();
 346     }
 347     delete this;
 348 }
 349
 350 TunnelStateData::TunnelStateData(ClientHttpRequest *clientRequest) :
 351     startTime(squid_curtime),
 352     destinations(new ResolvedPeers()),
 353     destinationsFound(false),
 354     committedToServer(false),
 355     n_tries(0),
 356     banRetries(nullptr),
 357     codeContext(CodeContext::Current())
 358 {
 359     debugs(26, 3, "TunnelStateData constructed this=" << this);
 360     client.readPendingFunc = &tunnelDelayedClientRead;
 361     server.readPendingFunc = &tunnelDelayedServerRead;
 362
 363     assert(clientRequest);
 364     url = xstrdup(clientRequest->uri);
 365     request = clientRequest->request;
 366     Must(request);
 367     server.size_ptr = &clientRequest->out.size;
 368     client.size_ptr = &clientRequest->al->http.clientRequestSz.payloadData;
 369     status_ptr = &clientRequest->al->http.code;
 370     al = clientRequest->al;
 371     http = clientRequest;
 372
 373     al->cache.code.update(LOG_TCP_TUNNEL);
 374
 375     client.initConnection(clientRequest->getConn()->clientConnection, tunnelClientClosed, "tunnelClientClosed", this);
 376
 377     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 378                                      CommTimeoutCbPtrFun(tunnelTimeout, this));
 379     commSetConnTimeout(client.conn, Config.Timeout.lifetime, timeoutCall);
 380 }
 381
 382 TunnelStateData::~TunnelStateData()
 383 {
 384     debugs(26, 3, "TunnelStateData destructed this=" << this);
 385     assert(noConnections());
 386     xfree(url);
 387     cancelStep("~TunnelStateData");
 388     delete savedError;
 389 }
 390
 391 TunnelStateData::Connection::~Connection()
 392 {
 393     if (readPending)
 394         eventDelete(readPendingFunc, readPending);
 395
 396     safe_free(buf);
 397 }
 398
 399 const char *
 400 TunnelStateData::checkRetry()
 401 {
 402     if (shutting_down)
 403         return "shutting down";
 404     if (exhaustedTries())
 405         return "exhausted tries";
 406     if (!FwdState::EnoughTimeToReForward(startTime))
 407         return "forwarding timeout";
 408     if (banRetries)
 409         return banRetries;
 410     if (noConnections())
 411         return "no connections";
 412
 413     // TODO: Use std::optional for peer_reply_status to avoid treating zero value specially.
 414     if (request->hier.peer_reply_status != Http::scNone && !Http::IsReforwardableStatus(request->hier.peer_reply_status))
 415         return "received HTTP status code is not reforwardable";
 416
 417     // TODO: check pinned connections; see FwdState::pinnedCanRetry()
 418     return nullptr;
 419 }
 420
 421 void
 422 TunnelStateData::retryOrBail(const char *context)
 423 {
 424     assert(!server.conn);
 425
 426     const auto *bailDescription = checkRetry();
 427     if (!bailDescription) {
 428         if (!destinations->empty())
 429             return startConnecting(); // try connecting to another destination
 430
 431         if (subscribed) {
 432             debugs(26, 4, "wait for more destinations to try");
 433             return; // expect a noteDestination*() call
 434         }
 435
 436         // fall through to bail
 437     }
 438
 439     /* bail */
 440
 441     if (request)
 442         request->hier.stopPeerClock(false);
 443
 444     // TODO: Add sendSavedErrorOr(err_type type, Http::StatusCode, context).
 445     // Then, the remaining method code (below) should become the common part of
 446     // sendNewError() and sendSavedErrorOr(), used in "error detected" cases.
 447     if (!savedError)
 448         saveError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al));
 449     const auto canSendError = Comm::IsConnOpen(client.conn) && !client.dirty &&
 450                               clientExpectsConnectResponse();
 451     if (canSendError)
 452         return sendError(savedError, bailDescription ? bailDescription : context);
 453     *status_ptr = savedError->httpStatus;
 454
 455     if (noConnections())
 456         return deleteThis();
 457
 458     // This is a "Comm::IsConnOpen(client.conn) but !canSendError" case.
 459     // Closing the connection (after finishing writing) is the best we can do.
 460     if (!client.writer)
 461         client.conn->close();
 462     // else writeClientDone() must notice a closed server and close the client
 463 }
 464
 465 int
 466 TunnelStateData::Connection::bytesWanted(int lowerbound, int upperbound) const
 467 {
 468 #if USE_DELAY_POOLS
 469     return delayId.bytesWanted(lowerbound, upperbound);
 470 #else
 471     (void)lowerbound;
 472     return upperbound;
 473 #endif
 474 }
 475
 476 void
 477 TunnelStateData::Connection::bytesIn(int const &count)
 478 {
 479     debugs(26, 3, "len=" << len << " + count=" << count);
 480 #if USE_DELAY_POOLS
 481     delayId.bytesIn(count);
 482 #endif
 483
 484     len += count;
 485 }
 486
 487 /// update "hierarchy" annotations with a new (possibly failed) destination
 488 /// \param origin the name of the origin server we were trying to reach
 489 void
 490 TunnelStateData::syncHierNote(const Comm::ConnectionPointer &conn, const char *origin)
 491 {
 492     request->hier.resetPeerNotes(conn, origin);
 493     al->hier.resetPeerNotes(conn, origin);
 494 }
 495
 496 /// sets n_tries to the given value (while keeping ALE in sync)
 497 void
 498 TunnelStateData::updateAttempts(const int newValue)
 499 {
 500     Assure(n_tries <= newValue); // n_tries cannot decrease
 501
 502     // Squid probably creates at most one FwdState/TunnelStateData object per
 503     // ALE, but, unlike an assignment would, this increment logic works even if
 504     // Squid uses multiple such objects for a given ALE in some esoteric cases.
 505     al->requestAttempts += (newValue - n_tries);
 506
 507     n_tries = newValue;
 508     debugs(26, 5, n_tries);
 509 }
 510
 511 int
 512 TunnelStateData::Connection::debugLevelForError(int const xerrno) const
 513 {
 514 #ifdef ECONNRESET
 515
 516     if (xerrno == ECONNRESET)
 517         return 2;
 518
 519 #endif
 520
 521     if (ignoreErrno(xerrno))
 522         return 3;
 523
 524     return 1;
 525 }
 526
 527 /* Read from server side and queue it for writing to the client */
 528 void
 529 TunnelStateData::ReadServer(const Comm::ConnectionPointer &c, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data)
 530 {
 531     TunnelStateData *tunnelState = (TunnelStateData *)data;
 532     assert(cbdataReferenceValid(tunnelState));
 533     debugs(26, 3, c);
 534
 535     tunnelState->readServer(buf, len, errcode, xerrno);
 536 }
 537
 538 void
 539 TunnelStateData::readServer(char *, size_t len, Comm::Flag errcode, int xerrno)
 540 {
 541     debugs(26, 3, server.conn << ", read " << len << " bytes, err=" << errcode);
 542     server.delayedLoops=0;
 543
 544     /*
 545      * Bail out early on Comm::ERR_CLOSING
 546      * - close handlers will tidy up for us
 547      */
 548
 549     if (errcode == Comm::ERR_CLOSING)
 550         return;
 551
 552     if (len > 0) {
 553         server.bytesIn(len);
 554         statCounter.server.all.kbytes_in += len;
 555         statCounter.server.other.kbytes_in += len;
 556         request->hier.notePeerRead();
 557     }
 558
 559     if (keepGoingAfterRead(len, errcode, xerrno, server, client))
 560         copy(len, server, client, WriteClientDone);
 561 }
 562
 563 void
 564 TunnelStateData::Connection::error(int const xerrno)
 565 {
 566     debugs(50, debugLevelForError(xerrno), conn << ": read/write failure: " << xstrerr(xerrno));
 567
 568     if (!ignoreErrno(xerrno))
 569         conn->close();
 570 }
 571
 572 /* Read from client side and queue it for writing to the server */
 573 void
 574 TunnelStateData::ReadClient(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag errcode, int xerrno, void *data)
 575 {
 576     TunnelStateData *tunnelState = (TunnelStateData *)data;
 577     assert (cbdataReferenceValid (tunnelState));
 578
 579     tunnelState->readClient(buf, len, errcode, xerrno);
 580 }
 581
 582 void
 583 TunnelStateData::readClient(char *, size_t len, Comm::Flag errcode, int xerrno)
 584 {
 585     debugs(26, 3, client.conn << ", read " << len << " bytes, err=" << errcode);
 586     client.delayedLoops=0;
 587
 588     /*
 589      * Bail out early on Comm::ERR_CLOSING
 590      * - close handlers will tidy up for us
 591      */
 592
 593     if (errcode == Comm::ERR_CLOSING)
 594         return;
 595
 596     if (len > 0) {
 597         client.bytesIn(len);
 598         statCounter.client_http.kbytes_in += len;
 599     }
 600
 601     if (keepGoingAfterRead(len, errcode, xerrno, client, server))
 602         copy(len, client, server, WriteServerDone);
 603 }
 604
 605 /// Updates state after reading from client or server.
 606 /// Returns whether the caller should use the data just read.
 607 bool
 608 TunnelStateData::keepGoingAfterRead(size_t len, Comm::Flag errcode, int xerrno, Connection &from, Connection &to)
 609 {
 610     debugs(26, 3, "from={" << from.conn << "}, to={" << to.conn << "}");
 611
 612     /* I think this is to prevent free-while-in-a-callback behaviour
 613      * - RBC 20030229
 614      * from.conn->close() / to.conn->close() done here trigger close callbacks which may free TunnelStateData
 615      */
 616     const CbcPointer<TunnelStateData> safetyLock(this);
 617
 618     /* Bump the source connection read timeout on any activity */
 619     if (Comm::IsConnOpen(from.conn)) {
 620         AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 621                                          CommTimeoutCbPtrFun(tunnelTimeout, this));
 622         commSetConnTimeout(from.conn, Config.Timeout.read, timeoutCall);
 623     }
 624
 625     /* Bump the dest connection read timeout on any activity */
 626     /* see Bug 3659: tunnels can be weird, with very long one-way transfers */
 627     if (Comm::IsConnOpen(to.conn)) {
 628         AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 629                                          CommTimeoutCbPtrFun(tunnelTimeout, this));
 630         commSetConnTimeout(to.conn, Config.Timeout.read, timeoutCall);
 631     }
 632
 633     if (errcode)
 634         from.error (xerrno);
 635     else if (len == 0 || !Comm::IsConnOpen(to.conn)) {
 636         debugs(26, 3, "Nothing to write or client gone. Terminate the tunnel.");
 637         from.conn->close();
 638
 639         /* Only close the remote end if we've finished queueing data to it */
 640         if (from.len == 0 && Comm::IsConnOpen(to.conn) ) {
 641             to.conn->close();
 642         }
 643     } else if (cbdataReferenceValid(this)) {
 644         return true;
 645     }
 646
 647     return false;
 648 }
 649
 650 void
 651 TunnelStateData::copy(size_t len, Connection &from, Connection &to, IOCB *completion)
 652 {
 653     debugs(26, 3, "Schedule Write");
 654     AsyncCall::Pointer call = commCbCall(5,5, "TunnelBlindCopyWriteHandler",
 655                                          CommIoCbPtrFun(completion, this));
 656     to.write(from.buf, len, call, nullptr);
 657 }
 658
 659 /* Writes data from the client buffer to the server side */
 660 void
 661 TunnelStateData::WriteServerDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 662 {
 663     TunnelStateData *tunnelState = (TunnelStateData *)data;
 664     assert (cbdataReferenceValid (tunnelState));
 665     tunnelState->server.writer = nullptr;
 666
 667     tunnelState->writeServerDone(buf, len, flag, xerrno);
 668 }
 669
 670 void
 671 TunnelStateData::writeServerDone(char *, size_t len, Comm::Flag flag, int xerrno)
 672 {
 673     debugs(26, 3, server.conn << ", " << len << " bytes written, flag=" << flag);
 674
 675     if (flag == Comm::ERR_CLOSING)
 676         return;
 677
 678     request->hier.notePeerWrite();
 679
 680     /* Error? */
 681     if (flag != Comm::OK) {
 682         debugs(26, 4, "to-server write failed: " << xerrno);
 683         server.error(xerrno); // may call comm_close
 684         return;
 685     }
 686
 687     /* EOF? */
 688     if (len == 0) {
 689         debugs(26, 4, "No read input. Closing server connection.");
 690         server.conn->close();
 691         return;
 692     }
 693
 694     /* Valid data */
 695     statCounter.server.all.kbytes_out += len;
 696     statCounter.server.other.kbytes_out += len;
 697     client.dataSent(len);
 698
 699     /* If the other end has closed, so should we */
 700     if (!Comm::IsConnOpen(client.conn)) {
 701         debugs(26, 4, "Client gone away. Shutting down server connection.");
 702         server.conn->close();
 703         return;
 704     }
 705
 706     const CbcPointer<TunnelStateData> safetyLock(this); /* ??? should be locked by the caller... */
 707
 708     if (cbdataReferenceValid(this))
 709         copyClientBytes();
 710 }
 711
 712 /* Writes data from the server buffer to the client side */
 713 void
 714 TunnelStateData::WriteClientDone(const Comm::ConnectionPointer &, char *buf, size_t len, Comm::Flag flag, int xerrno, void *data)
 715 {
 716     TunnelStateData *tunnelState = (TunnelStateData *)data;
 717     assert (cbdataReferenceValid (tunnelState));
 718     tunnelState->client.writer = nullptr;
 719
 720     tunnelState->writeClientDone(buf, len, flag, xerrno);
 721 }
 722
 723 void
 724 TunnelStateData::Connection::dataSent(size_t amount)
 725 {
 726     debugs(26, 3, "len=" << len << " - amount=" << amount);
 727     assert(amount == (size_t)len);
 728     len =0;
 729     /* increment total object size */
 730
 731     if (size_ptr)
 732         *size_ptr += amount;
 733
 734 }
 735
 736 void
 737 TunnelStateData::Connection::write(const char *b, int size, AsyncCall::Pointer &callback, FREE * free_func)
 738 {
 739     writer = callback;
 740     dirty = true;
 741     Comm::Write(conn, b, size, callback, free_func);
 742 }
 743
 744 template <typename Method>
 745 void
 746 TunnelStateData::Connection::initConnection(const Comm::ConnectionPointer &aConn, Method method, const char *name, TunnelStateData *tunnelState)
 747 {
 748     Must(!Comm::IsConnOpen(conn));
 749     Must(!closer);
 750     Must(Comm::IsConnOpen(aConn));
 751     conn = aConn;
 752     closer = commCbCall(5, 4, name, CommCloseCbPtrFun(method, tunnelState));
 753     comm_add_close_handler(conn->fd, closer);
 754 }
 755
 756 void
 757 TunnelStateData::Connection::noteClosure()
 758 {
 759     debugs(26, 3, conn);
 760     conn = nullptr;
 761     closer = nullptr;
 762     writer = nullptr; // may already be nil
 763 }
 764
 765 void
 766 TunnelStateData::writeClientDone(char *, size_t len, Comm::Flag flag, int xerrno)
 767 {
 768     debugs(26, 3, client.conn << ", " << len << " bytes written, flag=" << flag);
 769
 770     if (flag == Comm::ERR_CLOSING)
 771         return;
 772
 773     /* Error? */
 774     if (flag != Comm::OK) {
 775         debugs(26, 4, "from-client read failed: " << xerrno);
 776         client.error(xerrno); // may call comm_close
 777         return;
 778     }
 779
 780     /* EOF? */
 781     if (len == 0) {
 782         debugs(26, 4, "Closing client connection due to 0 byte read.");
 783         client.conn->close();
 784         return;
 785     }
 786
 787     /* Valid data */
 788     statCounter.client_http.kbytes_out += len;
 789     server.dataSent(len);
 790
 791     /* If the other end has closed, so should we */
 792     if (!Comm::IsConnOpen(server.conn)) {
 793         debugs(26, 4, "Server has gone away. Terminating client connection.");
 794         client.conn->close();
 795         return;
 796     }
 797
 798     CbcPointer<TunnelStateData> safetyLock(this);   /* ??? should be locked by the caller... */
 799
 800     if (cbdataReferenceValid(this))
 801         copyServerBytes();
 802 }
 803
 804 static void
 805 tunnelTimeout(const CommTimeoutCbParams &io)
 806 {
 807     TunnelStateData *tunnelState = static_cast<TunnelStateData *>(io.data);
 808     debugs(26, 3, io.conn);
 809     /* Temporary lock to protect our own feets (comm_close -> tunnelClientClosed -> Free) */
 810     CbcPointer<TunnelStateData> safetyLock(tunnelState);
 811
 812     tunnelState->closeConnections();
 813 }
 814
 815 void
 816 TunnelStateData::closePendingConnection(const Comm::ConnectionPointer &conn, const char *reason)
 817 {
 818     debugs(26, 3, "because " << reason << "; " << conn);
 819     assert(!server.conn);
 820     if (IsConnOpen(conn))
 821         conn->close();
 822 }
 823
 824 void
 825 TunnelStateData::closeConnections()
 826 {
 827     if (Comm::IsConnOpen(server.conn))
 828         server.conn->close();
 829     if (Comm::IsConnOpen(client.conn))
 830         client.conn->close();
 831 }
 832
 833 static void
 834 tunnelDelayedClientRead(void *data)
 835 {
 836     if (!data)
 837         return;
 838
 839     TunnelStateData *tunnel = static_cast<TunnelStateData*>(data);
 840     const auto savedContext = CodeContext::Current();
 841     CodeContext::Reset(tunnel->codeContext);
 842     tunnel->client.readPending = nullptr;
 843     static uint64_t counter=0;
 844     debugs(26, 7, "Client read(2) delayed " << ++counter << " times");
 845     tunnel->copyRead(tunnel->client, TunnelStateData::ReadClient);
 846     CodeContext::Reset(savedContext);
 847 }
 848
 849 static void
 850 tunnelDelayedServerRead(void *data)
 851 {
 852     if (!data)
 853         return;
 854
 855     TunnelStateData *tunnel = static_cast<TunnelStateData*>(data);
 856     const auto savedContext = CodeContext::Current();
 857     CodeContext::Reset(tunnel->codeContext);
 858     tunnel->server.readPending = nullptr;
 859     static uint64_t counter=0;
 860     debugs(26, 7, "Server read(2) delayed " << ++counter << " times");
 861     tunnel->copyRead(tunnel->server, TunnelStateData::ReadServer);
 862     CodeContext::Reset(savedContext);
 863 }
 864
 865 void
 866 TunnelStateData::copyRead(Connection &from, IOCB *completion)
 867 {
 868     assert(from.len == 0);
 869     // If only the minimum permitted read size is going to be attempted
 870     // then we schedule an event to try again in a few I/O cycles.
 871     // Allow at least 1 byte to be read every (0.3*10) seconds.
 872     int bw = from.bytesWanted(1, SQUID_TCP_SO_RCVBUF);
 873     // XXX: Delay pools must not delay client-to-Squid traffic (i.e. when
 874     // from.readPendingFunc is tunnelDelayedClientRead()).
 875     // XXX: Bug #4913: For delay pools, use delayRead() API instead.
 876     if (bw == 1 && ++from.delayedLoops < 10) {
 877         from.readPending = this;
 878         eventAdd("tunnelDelayedServerRead", from.readPendingFunc, from.readPending, 0.3, true);
 879         return;
 880     }
 881
 882     AsyncCall::Pointer call = commCbCall(5,4, "TunnelBlindCopyReadHandler",
 883                                          CommIoCbPtrFun(completion, this));
 884     comm_read(from.conn, from.buf, bw, call);
 885 }
 886
 887 void
 888 TunnelStateData::copyClientBytes()
 889 {
 890     if (preReadClientData.length()) {
 891         size_t copyBytes = preReadClientData.length() > SQUID_TCP_SO_RCVBUF ? SQUID_TCP_SO_RCVBUF : preReadClientData.length();
 892         memcpy(client.buf, preReadClientData.rawContent(), copyBytes);
 893         preReadClientData.consume(copyBytes);
 894         client.bytesIn(copyBytes);
 895         if (keepGoingAfterRead(copyBytes, Comm::OK, 0, client, server))
 896             copy(copyBytes, client, server, TunnelStateData::WriteServerDone);
 897     } else
 898         copyRead(client, ReadClient);
 899 }
 900
 901 void
 902 TunnelStateData::copyServerBytes()
 903 {
 904     if (preReadServerData.length()) {
 905         size_t copyBytes = preReadServerData.length() > SQUID_TCP_SO_RCVBUF ? SQUID_TCP_SO_RCVBUF : preReadServerData.length();
 906         memcpy(server.buf, preReadServerData.rawContent(), copyBytes);
 907         preReadServerData.consume(copyBytes);
 908         server.bytesIn(copyBytes);
 909         if (keepGoingAfterRead(copyBytes, Comm::OK, 0, server, client))
 910             copy(copyBytes, server, client, TunnelStateData::WriteClientDone);
 911     } else
 912         copyRead(server, ReadServer);
 913 }
 914
 915 /**
 916  * Set the HTTP status for this request and sets the read handlers for client
 917  * and server side connections.
 918  */
 919 static void
 920 tunnelStartShoveling(TunnelStateData *tunnelState)
 921 {
 922     assert(!tunnelState->transportWait);
 923     assert(!tunnelState->encryptionWait);
 924     assert(!tunnelState->peerWait);
 925
 926     assert(tunnelState->server.conn);
 927     AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
 928                                      CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
 929     commSetConnTimeout(tunnelState->server.conn, Config.Timeout.read, timeoutCall);
 930
 931     *tunnelState->status_ptr = Http::scOkay;
 932     if (cbdataReferenceValid(tunnelState)) {
 933
 934         // Shovel any payload already pushed into reply buffer by the server response
 935         if (!tunnelState->server.len)
 936             tunnelState->copyServerBytes();
 937         else {
 938             debugs(26, DBG_DATA, "Tunnel server PUSH Payload: \n" << Raw("", tunnelState->server.buf, tunnelState->server.len) << "\n----------");
 939             tunnelState->copy(tunnelState->server.len, tunnelState->server, tunnelState->client, TunnelStateData::WriteClientDone);
 940         }
 941
 942         if (tunnelState->http.valid() && tunnelState->http->getConn() && !tunnelState->http->getConn()->inBuf.isEmpty()) {
 943             SBuf * const in = &tunnelState->http->getConn()->inBuf;
 944             debugs(26, DBG_DATA, "Tunnel client PUSH Payload: \n" << *in << "\n----------");
 945             tunnelState->preReadClientData.append(*in);
 946             in->consume(); // ConnStateData buffer accounting after the shuffle.
 947         }
 948         tunnelState->copyClientBytes();
 949     }
 950 }
 951
 952 /**
 953  * All the pieces we need to write to client and/or server connection
 954  * have been written.
 955  * Call the tunnelStartShoveling to start the blind pump.
 956  */
 957 static void
 958 tunnelConnectedWriteDone(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
 959 {
 960     TunnelStateData *tunnelState = (TunnelStateData *)data;
 961     debugs(26, 3, conn << ", flag=" << flag);
 962     tunnelState->client.writer = nullptr;
 963
 964     if (flag != Comm::OK) {
 965         *tunnelState->status_ptr = Http::scInternalServerError;
 966         tunnelErrorComplete(conn->fd, data, 0);
 967         return;
 968     }
 969
 970     if (auto http = tunnelState->http.get()) {
 971         http->out.headers_sz += len;
 972         http->out.size += len;
 973     }
 974
 975     tunnelStartShoveling(tunnelState);
 976 }
 977
 978 void
 979 TunnelStateData::tunnelEstablishmentDone(Http::TunnelerAnswer &answer)
 980 {
 981     peerWait.finish();
 982     server.len = 0;
 983
 984     // XXX: al->http.code (i.e. *status_ptr) should not be (re)set
 985     // until we actually start responding to the client. Right here/now, we only
 986     // know how this cache_peer has responded to us.
 987     if (answer.peerResponseStatus != Http::scNone)
 988         *status_ptr = answer.peerResponseStatus;
 989
 990     auto sawProblem = false;
 991
 992     if (!answer.positive()) {
 993         sawProblem = true;
 994         assert(!answer.conn);
 995     } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
 996         sawProblem = true;
 997         closePendingConnection(answer.conn, "conn was closed while waiting for tunnelEstablishmentDone");
 998     }
 999
1000     if (!sawProblem) {
1001         assert(answer.positive()); // paranoid
1002         // copy any post-200 OK bytes to our buffer
1003         preReadServerData = answer.leftovers;
1004         notePeerReadyToShovel(answer.conn);
1005         return;
1006     }
1007
1008     ErrorState *error = nullptr;
1009     if (answer.positive()) {
1010         error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
1011     } else {
1012         error = answer.squidError.get();
1013         Must(error);
1014         answer.squidError.clear(); // preserve error for errorSendComplete()
1015     }
1016     assert(error);
1017     saveError(error);
1018     retryOrBail("tunneler error");
1019 }
1020
1021 void
1022 TunnelStateData::notePeerReadyToShovel(const Comm::ConnectionPointer &conn)
1023 {
1024     assert(!client.dirty);
1025     commitToServer(conn);
1026
1027     if (!clientExpectsConnectResponse())
1028         tunnelStartShoveling(this); // ssl-bumped connection, be quiet
1029     else {
1030         *status_ptr = Http::scOkay;
1031         AsyncCall::Pointer call = commCbCall(5,5, "tunnelConnectedWriteDone",
1032                                              CommIoCbPtrFun(tunnelConnectedWriteDone, this));
1033         al->reply = HttpReply::MakeConnectionEstablished();
1034         const auto mb = al->reply->pack();
1035         client.write(mb->content(), mb->contentSize(), call, mb->freeFunc());
1036         delete mb;
1037     }
1038 }
1039
1040 void
1041 TunnelStateData::commitToServer(const Comm::ConnectionPointer &conn)
1042 {
1043     committedToServer = true;
1044     banRetries = "committed to server";
1045     PeerSelectionInitiator::subscribed = false; // may already be false
1046     server.initConnection(conn, tunnelServerClosed, "tunnelServerClosed", this);
1047 }
1048
1049 static void
1050 tunnelErrorComplete(int fd/*const Comm::ConnectionPointer &*/, void *data, size_t)
1051 {
1052     TunnelStateData *tunnelState = (TunnelStateData *)data;
1053     debugs(26, 3, "FD " << fd);
1054     assert(tunnelState != nullptr);
1055     /* temporary lock to save our own feets (comm_close -> tunnelClientClosed -> Free) */
1056     CbcPointer<TunnelStateData> safetyLock(tunnelState);
1057
1058     if (Comm::IsConnOpen(tunnelState->client.conn))
1059         tunnelState->client.conn->close();
1060
1061     if (Comm::IsConnOpen(tunnelState->server.conn))
1062         tunnelState->server.conn->close();
1063 }
1064
1065 void
1066 TunnelStateData::noteConnection(HappyConnOpener::Answer &answer)
1067 {
1068     transportWait.finish();
1069
1070     updateAttempts(answer.n_tries);
1071
1072     ErrorState *error = nullptr;
1073     if ((error = answer.error.get())) {
1074         banRetries = "HappyConnOpener gave up";
1075         Must(!Comm::IsConnOpen(answer.conn));
1076         syncHierNote(answer.conn, request->url.host());
1077         answer.error.clear();
1078     } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
1079         error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
1080         closePendingConnection(answer.conn, "conn was closed while waiting for  noteConnection");
1081     }
1082
1083     if (error) {
1084         saveError(error);
1085         retryOrBail("tried all destinations");
1086         return;
1087     }
1088
1089     connectDone(answer.conn, request->url.host(), answer.reused);
1090 }
1091
1092 void
1093 TunnelStateData::connectDone(const Comm::ConnectionPointer &conn, const char *origin, const bool reused)
1094 {
1095     Must(Comm::IsConnOpen(conn));
1096
1097     if (reused)
1098         ResetMarkingsToServer(request.getRaw(), *conn);
1099     // else Comm::ConnOpener already applied proper/current markings
1100
1101     // TODO: add pconn race state tracking
1102
1103     syncHierNote(conn, origin);
1104
1105 #if USE_DELAY_POOLS
1106     /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
1107     if (conn->getPeer() && conn->getPeer()->options.no_delay)
1108         server.setDelayId(DelayId());
1109 #endif
1110
1111     netdbPingSite(request->url.host());
1112
1113     request->peer_host = conn->getPeer() ? conn->getPeer()->host : nullptr;
1114
1115     bool toOrigin = false; // same semantics as StateFlags::toOrigin
1116     if (const auto * const peer = conn->getPeer()) {
1117         request->prepForPeering(*peer);
1118         toOrigin = peer->options.originserver;
1119     } else {
1120         request->prepForDirect();
1121         toOrigin = true;
1122     }
1123
1124     if (!toOrigin)
1125         connectToPeer(conn);
1126     else {
1127         notePeerReadyToShovel(conn);
1128     }
1129 }
1130
1131 /// whether we have used up all permitted forwarding attempts
1132 bool
1133 TunnelStateData::exhaustedTries() const
1134 {
1135     return n_tries >= Config.forward_max_tries;
1136 }
1137
1138 void
1139 tunnelStart(ClientHttpRequest * http)
1140 {
1141     debugs(26, 3, MYNAME);
1142     /* Create state structure. */
1143     TunnelStateData *tunnelState = nullptr;
1144     ErrorState *err = nullptr;
1145     HttpRequest *request = http->request;
1146     char *url = http->uri;
1147
1148     /*
1149      * client_addr.isNoAddr()  indicates this is an "internal" request
1150      * from peer_digest.c, asn.c, netdb.c, etc and should always
1151      * be allowed.  yuck, I know.
1152      */
1153
1154     if (Config.accessList.miss && !request->client_addr.isNoAddr()) {
1155         /*
1156          * Check if this host is allowed to fetch MISSES from us (miss_access)
1157          * default is to allow.
1158          */
1159         ACLFilledChecklist ch(Config.accessList.miss, request, nullptr);
1160         ch.al = http->al;
1161         ch.src_addr = request->client_addr;
1162         ch.my_addr = request->my_addr;
1163         ch.syncAle(request, http->log_uri);
1164         if (ch.fastCheck().denied()) {
1165             debugs(26, 4, "MISS access forbidden.");
1166             http->updateLoggingTags(LOG_TCP_TUNNEL);
1167             err = new ErrorState(ERR_FORWARDING_DENIED, Http::scForbidden, request, http->al);
1168             http->al->http.code = Http::scForbidden;
1169             errorSend(http->getConn()->clientConnection, err);
1170             return;
1171         }
1172     }
1173
1174     debugs(26, 3, request->method << ' ' << url << ' ' << request->http_ver);
1175     ++statCounter.server.all.requests;
1176     ++statCounter.server.other.requests;
1177
1178     tunnelState = new TunnelStateData(http);
1179 #if USE_DELAY_POOLS
1180     tunnelState->server.setDelayId(DelayId::DelayClient(http));
1181 #endif
1182     tunnelState->startSelectingDestinations(request, http->al, nullptr);
1183 }
1184
1185 void
1186 TunnelStateData::connectToPeer(const Comm::ConnectionPointer &conn)
1187 {
1188     if (const auto p = conn->getPeer()) {
1189         if (p->secure.encryptTransport)
1190             return advanceDestination("secure connection to peer", conn, [this,&conn] {
1191             secureConnectionToPeer(conn);
1192         });
1193     }
1194
1195     connectedToPeer(conn);
1196 }
1197
1198 /// encrypts an established TCP connection to peer
1199 void
1200 TunnelStateData::secureConnectionToPeer(const Comm::ConnectionPointer &conn)
1201 {
1202     const auto callback = asyncCallback(5, 4, TunnelStateData::noteSecurityPeerConnectorAnswer, this);
1203     const auto connector = new Security::BlindPeerConnector(request, conn, callback, al);
1204     encryptionWait.start(connector, callback);
1205 }
1206
1207 /// starts a preparation step for an established connection; retries on failures
1208 template <typename StepStart>
1209 void
1210 TunnelStateData::advanceDestination(const char *stepDescription, const Comm::ConnectionPointer &conn, const StepStart &startStep)
1211 {
1212     // TODO: Extract destination-specific handling from TunnelStateData so that
1213     // all the awkward, limited-scope advanceDestination() calls can be replaced
1214     // with a single simple try/catch,retry block.
1215     try {
1216         startStep();
1217         // now wait for the step callback
1218     } catch (...) {
1219         debugs (26, 2, "exception while trying to " << stepDescription << ": " << CurrentException);
1220         closePendingConnection(conn, "connection preparation exception");
1221         if (!savedError)
1222             saveError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al));
1223         retryOrBail(stepDescription);
1224     }
1225 }
1226
1227 /// callback handler for the connection encryptor
1228 void
1229 TunnelStateData::noteSecurityPeerConnectorAnswer(Security::EncryptorAnswer &answer)
1230 {
1231     encryptionWait.finish();
1232
1233     ErrorState *error = nullptr;
1234     assert(!answer.tunneled);
1235     if ((error = answer.error.get())) {
1236         assert(!answer.conn);
1237         answer.error.clear();
1238     } else if (!Comm::IsConnOpen(answer.conn) || fd_table[answer.conn->fd].closing()) {
1239         error = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw(), al);
1240         closePendingConnection(answer.conn, "conn was closed while waiting for noteSecurityPeerConnectorAnswer");
1241     }
1242
1243     if (error) {
1244         saveError(error);
1245         retryOrBail("TLS peer connection error");
1246         return;
1247     }
1248
1249     connectedToPeer(answer.conn);
1250 }
1251
1252 void
1253 TunnelStateData::connectedToPeer(const Comm::ConnectionPointer &conn)
1254 {
1255     advanceDestination("establish tunnel through proxy", conn, [this,&conn] {
1256         establishTunnelThruProxy(conn);
1257     });
1258 }
1259
1260 void
1261 TunnelStateData::establishTunnelThruProxy(const Comm::ConnectionPointer &conn)
1262 {
1263     const auto callback = asyncCallback(5, 4, TunnelStateData::tunnelEstablishmentDone, this);
1264     const auto tunneler = new Http::Tunneler(conn, request, callback, Config.Timeout.lifetime, al);
1265 #if USE_DELAY_POOLS
1266     tunneler->setDelayId(server.delayId);
1267 #endif
1268     peerWait.start(tunneler, callback);
1269 }
1270
1271 void
1272 TunnelStateData::noteDestination(Comm::ConnectionPointer path)
1273 {
1274     destinationsFound = true;
1275
1276     if (!path) { // decided to use a pinned connection
1277         // We can call usePinned() without fear of clashing with an earlier
1278         // forwarding attempt because PINNED must be the first destination.
1279         assert(destinations->empty());
1280         usePinned();
1281         return;
1282     }
1283
1284     destinations->addPath(path);
1285
1286     if (transportWait) {
1287         assert(!transporting());
1288         notifyConnOpener();
1289         return; // and continue to wait for tunnelConnectDone() callback
1290     }
1291
1292     if (transporting())
1293         return; // and continue to receive destinations for backup
1294
1295     startConnecting();
1296 }
1297
1298 void
1299 TunnelStateData::noteDestinationsEnd(ErrorState *selectionError)
1300 {
1301     PeerSelectionInitiator::subscribed = false;
1302     destinations->destinationsFinalized = true;
1303     if (!destinationsFound) {
1304
1305         // XXX: Honor clientExpectsConnectResponse() before replying.
1306
1307         if (selectionError)
1308             return sendError(selectionError, "path selection has failed");
1309
1310         // TODO: Merge with FwdState and remove this likely unnecessary check.
1311         if (savedError)
1312             return sendError(savedError, "path selection found no paths (with an impossible early error)");
1313
1314         return sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request.getRaw(), al),
1315                          "path selection found no paths");
1316     }
1317     // else continue to use one of the previously noted destinations;
1318     // if all of them fail, tunneling as whole will fail
1319     Must(!selectionError); // finding at least one path means selection succeeded
1320
1321     if (transportWait) {
1322         assert(!transporting());
1323         notifyConnOpener();
1324         return; // and continue to wait for the noteConnection() callback
1325     }
1326
1327     if (transporting()) {
1328         // We are already using a previously opened connection (but were also
1329         // receiving more destinations in case we need to re-forward).
1330         debugs(17, 7, "keep transporting");
1331         return;
1332     }
1333
1334     // destinationsFound, but none of them worked, and we were waiting for more
1335     assert(savedError);
1336     // XXX: Honor clientExpectsConnectResponse() before replying.
1337     sendError(savedError, "all found paths have failed");
1338 }
1339
1340 /// Whether a tunneling attempt to some selected destination X is in progress
1341 /// (after successfully opening/reusing a transport connection to X).
1342 /// \sa transportWait
1343 bool
1344 TunnelStateData::transporting() const
1345 {
1346     return encryptionWait || peerWait || committedToServer;
1347 }
1348
1349 /// remembers an error to be used if there will be no more connection attempts
1350 void
1351 TunnelStateData::saveError(ErrorState *error)
1352 {
1353     debugs(26, 4, savedError << " ? " << error);
1354     assert(error);
1355     delete savedError; // may be nil
1356     savedError = error;
1357 }
1358
1359 /// Starts sending the given error message to the client, leading to the
1360 /// eventual transaction termination. Call with savedError to send savedError.
1361 void
1362 TunnelStateData::sendError(ErrorState *finalError, const char *reason)
1363 {
1364     debugs(26, 3, "aborting transaction for " << reason);
1365
1366     if (request)
1367         request->hier.stopPeerClock(false);
1368
1369     cancelStep(reason);
1370
1371     assert(finalError);
1372
1373     // get rid of any cached error unless that is what the caller is sending
1374     if (savedError != finalError)
1375         delete savedError; // may be nil
1376     savedError = nullptr;
1377
1378     // we cannot try other destinations after responding with an error
1379     PeerSelectionInitiator::subscribed = false; // may already be false
1380
1381     *status_ptr = finalError->httpStatus;
1382     finalError->callback = tunnelErrorComplete;
1383     finalError->callback_data = this;
1384     errorSend(client.conn, finalError);
1385 }
1386
1387 /// Notify a pending subtask, if any, that we no longer need its help. We do not
1388 /// have to do this -- the subtask job will eventually end -- but ending it
1389 /// earlier reduces waste and may reduce DoS attack surface.
1390 void
1391 TunnelStateData::cancelStep(const char *reason)
1392 {
1393     transportWait.cancel(reason);
1394     encryptionWait.cancel(reason);
1395     peerWait.cancel(reason);
1396 }
1397
1398 void
1399 TunnelStateData::startConnecting()
1400 {
1401     if (request)
1402         request->hier.startPeerClock();
1403
1404     assert(!destinations->empty());
1405     assert(!transporting());
1406
1407     delete savedError; // may still be nil
1408     savedError = nullptr;
1409     request->hier.peer_reply_status = Http::scNone; // TODO: Move to startPeerClock()?
1410
1411     const auto callback = asyncCallback(17, 5, TunnelStateData::noteConnection, this);
1412     const auto cs = new HappyConnOpener(destinations, callback, request, startTime, n_tries, al);
1413     cs->setHost(request->url.host());
1414     cs->setRetriable(false);
1415     cs->allowPersistent(false);
1416     destinations->notificationPending = true; // start() is async
1417     transportWait.start(cs, callback);
1418 }
1419
1420 /// send request on an existing connection dedicated to the requesting client
1421 void
1422 TunnelStateData::usePinned()
1423 {
1424     Must(request);
1425     const auto connManager = request->pinnedConnection();
1426     try {
1427         const auto serverConn = ConnStateData::BorrowPinnedConnection(request.getRaw(), al);
1428         debugs(26, 7, "pinned peer connection: " << serverConn);
1429
1430         updateAttempts(n_tries + 1);
1431
1432         // Set HttpRequest pinned related flags for consistency even if
1433         // they are not really used by tunnel.cc code.
1434         request->flags.pinned = true;
1435         if (connManager->pinnedAuth())
1436             request->flags.auth = true;
1437
1438         // the server may close the pinned connection before this request
1439         const auto reused = true;
1440         connectDone(serverConn, connManager->pinning.host, reused);
1441     } catch (ErrorState * const error) {
1442         syncHierNote(nullptr, connManager ? connManager->pinning.host : request->url.host());
1443         // XXX: Honor clientExpectsConnectResponse() before replying.
1444         // a PINNED path failure is fatal; do not wait for more paths
1445         sendError(error, "pinned path failure");
1446         return;
1447     }
1448
1449 }
1450
1451 CBDATA_CLASS_INIT(TunnelStateData);
1452
1453 bool
1454 TunnelStateData::noConnections() const
1455 {
1456     return !Comm::IsConnOpen(server.conn) && !Comm::IsConnOpen(client.conn);
1457 }
1458
1459 #if USE_DELAY_POOLS
1460 void
1461 TunnelStateData::Connection::setDelayId(DelayId const &newDelay)
1462 {
1463     delayId = newDelay;
1464 }
1465
1466 #endif
1467
1468 /// makes sure connection opener knows that the destinations have changed
1469 void
1470 TunnelStateData::notifyConnOpener()
1471 {
1472     if (destinations->notificationPending) {
1473         debugs(17, 7, "reusing pending notification");
1474     } else {
1475         destinations->notificationPending = true;
1476         CallJobHere(17, 5, transportWait.job(), HappyConnOpener, noteCandidatesChange);
1477     }
1478 }
1479
1480 /**
1481  * Sets up a TCP tunnel through Squid and starts shoveling traffic.
1482  * \param request the request that initiated/caused this tunnel
1483  * \param clientConn the already accepted client-to-Squid TCP connection
1484  * \param srvConn the already established Squid-to-server TCP connection
1485  * \param preReadServerData server-sent bytes to be forwarded to the client
1486  */
1487 void
1488 switchToTunnel(HttpRequest *request, const Comm::ConnectionPointer &clientConn, const Comm::ConnectionPointer &srvConn, const SBuf &preReadServerData)
1489 {
1490     Must(Comm::IsConnOpen(clientConn));
1491     Must(Comm::IsConnOpen(srvConn));
1492
1493     debugs(26,5, "Revert to tunnel FD " << clientConn->fd << " with FD " << srvConn->fd);
1494
1495     /* Create state structure. */
1496     ++statCounter.server.all.requests;
1497     ++statCounter.server.other.requests;
1498
1499     auto conn = request->clientConnectionManager.get();
1500     Must(conn);
1501     Http::StreamPointer context = conn->pipeline.front();
1502     Must(context && context->http);
1503
1504     debugs(26, 3, request->method << " " << context->http->uri << " " << request->http_ver);
1505
1506     TunnelStateData *tunnelState = new TunnelStateData(context->http);
1507     tunnelState->commitToServer(srvConn);
1508
1509     request->hier.resetPeerNotes(srvConn, tunnelState->getHost());
1510
1511 #if USE_DELAY_POOLS
1512     /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
1513     if (!srvConn->getPeer() || !srvConn->getPeer()->options.no_delay)
1514         tunnelState->server.setDelayId(DelayId::DelayClient(context->http));
1515 #endif
1516
1517     request->peer_host = srvConn->getPeer() ? srvConn->getPeer()->host : nullptr;
1518
1519     debugs(26, 4, "determine post-connect handling pathway.");
1520     if (const auto peer = srvConn->getPeer())
1521         request->prepForPeering(*peer);
1522     else
1523         request->prepForDirect();
1524
1525     tunnelState->preReadServerData = preReadServerData;
1526
1527     tunnelStartShoveling(tunnelState);
1528 }
1529