]> git.ipfire.org Git - thirdparty/squid.git/blobdiff - src/tunnel.cc
Supply AccessLogEntry (ALE) for more fast ACL checks. (#182)
[thirdparty/squid.git] / src / tunnel.cc
index a0f98770e447a2ba5a67f25c09d3265f450cb512..28bfb4b74563659d428e54c1b8cb80b3c76aa78c 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
+ * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
  *
  * Squid software is distributed under GPLv2+ license and includes
  * contributions from numerous individuals and organizations.
 #include "http.h"
 #include "http/Stream.h"
 #include "HttpRequest.h"
-#include "HttpStateFlags.h"
 #include "ip/QosConfig.h"
 #include "LogTags.h"
 #include "MemBuf.h"
+#include "neighbors.h"
 #include "PeerSelectState.h"
 #include "sbuf/SBuf.h"
+#include "security/BlindPeerConnector.h"
 #include "SquidConfig.h"
 #include "SquidTime.h"
-#include "ssl/BlindPeerConnector.h"
 #include "StatCounters.h"
 #if USE_OPENSSL
 #include "ssl/bio.h"
  *
  * TODO 2: then convert this into a AsyncJob, possibly a child of 'Server'
  */
-class TunnelStateData
+class TunnelStateData: public PeerSelectionInitiator
 {
-    CBDATA_CLASS(TunnelStateData);
+    CBDATA_CHILD(TunnelStateData);
 
 public:
-    TunnelStateData();
-    ~TunnelStateData();
+    TunnelStateData(ClientHttpRequest *);
+    virtual ~TunnelStateData();
     TunnelStateData(const TunnelStateData &); // do not implement
     TunnelStateData &operator =(const TunnelStateData &); // do not implement
 
@@ -105,6 +105,10 @@ public:
 
     /// Whether the client sent a CONNECT request to us.
     bool clientExpectsConnectResponse() const {
+        // If we are forcing a tunnel after receiving a client CONNECT, then we
+        // have already responded to that CONNECT before tunnel.cc started.
+        if (request && request->flags.forceTunnel)
+            return false;
 #if USE_OPENSSL
         // We are bumping and we had already send "OK CONNECTED"
         if (http.valid() && http->getConn() && http->getConn()->serverBump() && http->getConn()->serverBump()->step > Ssl::bumpStep1)
@@ -118,6 +122,12 @@ public:
     /// if it is waiting for Squid CONNECT response, closing connections.
     void informUserOfPeerError(const char *errMsg, size_t);
 
+    /// starts connecting to the next hop, either for the first time or while
+    /// recovering from the previous connect failure
+    void startConnecting();
+
+    void noteConnectFailure(const Comm::ConnectionPointer &conn);
+
     class Connection
     {
 
@@ -166,17 +176,23 @@ public:
     bool connectReqWriting; ///< whether we are writing a CONNECT request to a peer
     SBuf preReadClientData;
     SBuf preReadServerData;
-    time_t started;         ///< when this tunnel was initiated.
+    time_t startTime; ///< object creation time, before any peer selection/connection attempts
 
     void copyRead(Connection &from, IOCB *completion);
 
     /// continue to set up connection to a peer, going async for SSL peers
     void connectToPeer();
 
+    /* PeerSelectionInitiator API */
+    virtual void noteDestination(Comm::ConnectionPointer conn) override;
+    virtual void noteDestinationsEnd(ErrorState *selectionError) override;
+
+    void saveError(ErrorState *finalError);
+    void sendError(ErrorState *finalError, const char *reason);
+
 private:
-#if USE_OPENSSL
-    /// Gives PeerConnector access to Answer in the TunnelStateData callback dialer.
-    class MyAnswerDialer: public CallDialer, public Ssl::PeerConnector::CbDialer
+    /// Gives Security::PeerConnector access to Answer in the TunnelStateData callback dialer.
+    class MyAnswerDialer: public CallDialer, public Security::PeerConnector::CbDialer
     {
     public:
         typedef void (TunnelStateData::*Method)(Security::EncryptorAnswer &);
@@ -191,7 +207,7 @@ private:
             os << '(' << tunnel_.get() << ", " << answer_ << ')';
         }
 
-        /* Ssl::PeerConnector::CbDialer API */
+        /* Security::PeerConnector::CbDialer API */
         virtual Security::EncryptorAnswer &answer() { return answer_; }
 
     private:
@@ -199,11 +215,13 @@ private:
         CbcPointer<TunnelStateData> tunnel_;
         Security::EncryptorAnswer answer_;
     };
-#endif
 
     /// callback handler after connection setup (including any encryption)
     void connectedToPeer(Security::EncryptorAnswer &answer);
 
+    /// details of the "last tunneling attempt" failure (if it failed)
+    ErrorState *savedError = nullptr;
+
 public:
     bool keepGoingAfterRead(size_t len, Comm::Flag errcode, int xerrno, Connection &from, Connection &to);
     void copy(size_t len, Connection &from, Connection &to, IOCB *);
@@ -226,7 +244,6 @@ static ERCB tunnelErrorComplete;
 static CLCB tunnelServerClosed;
 static CLCB tunnelClientClosed;
 static CTCB tunnelTimeout;
-static PSC tunnelPeerSelectComplete;
 static EVH tunnelDelayedClientRead;
 static EVH tunnelDelayedServerRead;
 static void tunnelConnected(const Comm::ConnectionPointer &server, void *);
@@ -287,19 +304,31 @@ tunnelClientClosed(const CommCloseCbParams &params)
     }
 }
 
-TunnelStateData::TunnelStateData() :
-    url(NULL),
-    http(),
-    request(NULL),
-    status_ptr(NULL),
-    logTag_ptr(NULL),
+TunnelStateData::TunnelStateData(ClientHttpRequest *clientRequest) :
     connectRespBuf(NULL),
     connectReqWriting(false),
-    started(squid_curtime)
+    startTime(squid_curtime)
 {
     debugs(26, 3, "TunnelStateData constructed this=" << this);
     client.readPendingFunc = &tunnelDelayedClientRead;
     server.readPendingFunc = &tunnelDelayedServerRead;
+
+    assert(clientRequest);
+    url = xstrdup(clientRequest->uri);
+    request = clientRequest->request;
+    server.size_ptr = &clientRequest->out.size;
+    client.size_ptr = &clientRequest->al->http.clientRequestSz.payloadData;
+    status_ptr = &clientRequest->al->http.code;
+    logTag_ptr = &clientRequest->logType;
+    al = clientRequest->al;
+    http = clientRequest;
+
+    client.conn = clientRequest->getConn()->clientConnection;
+    comm_add_close_handler(client.conn->fd, tunnelClientClosed, this);
+
+    AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
+                                     CommTimeoutCbPtrFun(tunnelTimeout, this));
+    commSetConnTimeout(client.conn, Config.Timeout.lifetime, timeoutCall);
 }
 
 TunnelStateData::~TunnelStateData()
@@ -309,6 +338,7 @@ TunnelStateData::~TunnelStateData()
     xfree(url);
     serverDestinations.clear();
     delete connectRespBuf;
+    delete savedError;
 }
 
 TunnelStateData::Connection::~Connection()
@@ -386,6 +416,7 @@ TunnelStateData::readServer(char *, size_t len, Comm::Flag errcode, int xerrno)
         server.bytesIn(len);
         statCounter.server.all.kbytes_in += len;
         statCounter.server.other.kbytes_in += len;
+        request->hier.notePeerRead();
     }
 
     if (keepGoingAfterRead(len, errcode, xerrno, server, client))
@@ -407,6 +438,7 @@ TunnelStateData::readConnectResponseDone(char *, size_t len, Comm::Flag errcode,
         server.bytesIn(len);
         statCounter.server.all.kbytes_in += len;
         statCounter.server.other.kbytes_in += len;
+        request->hier.notePeerRead();
     }
 
     if (keepGoingAfterRead(len, errcode, xerrno, server, client))
@@ -429,14 +461,9 @@ TunnelStateData::informUserOfPeerError(const char *errMsg, const size_t sz)
     }
 
     // if we have no reply suitable to relay, use 502 Bad Gateway
-    if (!sz || sz > static_cast<size_t>(connectRespBuf->contentSize())) {
-        ErrorState *err = new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw());
-        *status_ptr = Http::scBadGateway;
-        err->callback = tunnelErrorComplete;
-        err->callback_data = this;
-        errorSend(http->getConn()->clientConnection, err);
-        return;
-    }
+    if (!sz || sz > static_cast<size_t>(connectRespBuf->contentSize()))
+        return sendError(new ErrorState(ERR_CONNECT_FAIL, Http::scBadGateway, request.getRaw()),
+                         "peer error without reply");
 
     // if we need to send back the server response. write its headers to the client
     server.len = sz;
@@ -474,7 +501,7 @@ TunnelStateData::handleConnectResponse(const size_t chunkSize)
     HttpReply rep;
     Http::StatusCode parseErr = Http::scNone;
     const bool eof = !chunkSize;
-    connectRespBuf->terminate(); // HttpMsg::parse requires terminated string
+    connectRespBuf->terminate(); // Http::Message::parse requires terminated string
     const bool parsed = rep.parse(connectRespBuf->content(), connectRespBuf->contentSize(), eof, &parseErr);
     if (!parsed) {
         if (parseErr > 0) { // unrecoverable parsing error
@@ -497,18 +524,13 @@ TunnelStateData::handleConnectResponse(const size_t chunkSize)
     }
 
     // CONNECT response was successfully parsed
-    *status_ptr = rep.sline.status();
-
-    // we need to relay the 401/407 responses when login=PASS(THRU)
-    const char *pwd = server.conn->getPeer()->login;
-    const bool relay = pwd && (strcmp(pwd, "PASS") != 0 || strcmp(pwd, "PASSTHRU") != 0) &&
-                       (*status_ptr == Http::scProxyAuthenticationRequired ||
-                        *status_ptr == Http::scUnauthorized);
+    request->hier.peer_reply_status = rep.sline.status();
 
     // bail if we did not get an HTTP 200 (Connection Established) response
     if (rep.sline.status() != Http::scOkay) {
         // if we ever decide to reuse the peer connection, we must extract the error response first
-        informUserOfPeerError("unsupported CONNECT response status code", (relay ? rep.hdr_sz : 0));
+        *status_ptr = rep.sline.status(); // we are relaying peer response
+        informUserOfPeerError("unsupported CONNECT response status code", rep.hdr_sz);
         return;
     }
 
@@ -529,10 +551,7 @@ TunnelStateData::handleConnectResponse(const size_t chunkSize)
 void
 TunnelStateData::Connection::error(int const xerrno)
 {
-    /* XXX fixme xstrerror and xerrno... */
-    errno = xerrno;
-
-    debugs(50, debugLevelForError(xerrno), HERE << conn << ": read/write failure: " << xstrerror());
+    debugs(50, debugLevelForError(xerrno), HERE << conn << ": read/write failure: " << xstrerr(xerrno));
 
     if (!ignoreErrno(xerrno))
         conn->close();
@@ -641,12 +660,15 @@ TunnelStateData::writeServerDone(char *, size_t len, Comm::Flag flag, int xerrno
 {
     debugs(26, 3, HERE  << server.conn << ", " << len << " bytes written, flag=" << flag);
 
+    if (flag == Comm::ERR_CLOSING)
+        return;
+
+    request->hier.notePeerWrite();
+
     /* Error? */
     if (flag != Comm::OK) {
-        if (flag != Comm::ERR_CLOSING) {
-            debugs(26, 4, HERE << "calling TunnelStateData::server.error(" << xerrno <<")");
-            server.error(xerrno); // may call comm_close
-        }
+        debugs(26, 4, "to-server write failed: " << xerrno);
+        server.error(xerrno); // may call comm_close
         return;
     }
 
@@ -696,6 +718,7 @@ TunnelStateData::Connection::dataSent(size_t amount)
 
     if (size_ptr)
         *size_ptr += amount;
+
 }
 
 void
@@ -710,12 +733,13 @@ TunnelStateData::writeClientDone(char *, size_t len, Comm::Flag flag, int xerrno
 {
     debugs(26, 3, HERE << client.conn << ", " << len << " bytes written, flag=" << flag);
 
+    if (flag == Comm::ERR_CLOSING)
+        return;
+
     /* Error? */
     if (flag != Comm::OK) {
-        if (flag != Comm::ERR_CLOSING) {
-            debugs(26, 4, HERE << "Closing client connection due to comm flags.");
-            client.error(xerrno); // may call comm_close
-        }
+        debugs(26, 4, "from-client read failed: " << xerrno);
+        client.error(xerrno); // may call comm_close
         return;
     }
 
@@ -883,7 +907,7 @@ tunnelStartShoveling(TunnelStateData *tunnelState)
  * Call the tunnelStartShoveling to start the blind pump.
  */
 static void
-tunnelConnectedWriteDone(const Comm::ConnectionPointer &conn, char *, size_t, Comm::Flag flag, int, void *data)
+tunnelConnectedWriteDone(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
 {
     TunnelStateData *tunnelState = (TunnelStateData *)data;
     debugs(26, 3, HERE << conn << ", flag=" << flag);
@@ -895,24 +919,34 @@ tunnelConnectedWriteDone(const Comm::ConnectionPointer &conn, char *, size_t, Co
         return;
     }
 
+    if (auto http = tunnelState->http.get()) {
+        http->out.headers_sz += len;
+        http->out.size += len;
+    }
+
     tunnelStartShoveling(tunnelState);
 }
 
 /// Called when we are done writing CONNECT request to a peer.
 static void
-tunnelConnectReqWriteDone(const Comm::ConnectionPointer &conn, char *, size_t, Comm::Flag flag, int, void *data)
+tunnelConnectReqWriteDone(const Comm::ConnectionPointer &conn, char *, size_t ioSize, Comm::Flag flag, int, void *data)
 {
     TunnelStateData *tunnelState = (TunnelStateData *)data;
     debugs(26, 3, conn << ", flag=" << flag);
     tunnelState->server.writer = NULL;
     assert(tunnelState->waitingForConnectRequest());
 
+    tunnelState->request->hier.notePeerWrite();
+
     if (flag != Comm::OK) {
         *tunnelState->status_ptr = Http::scInternalServerError;
         tunnelErrorComplete(conn->fd, data, 0);
         return;
     }
 
+    statCounter.server.all.kbytes_out += ioSize;
+    statCounter.server.other.kbytes_out += ioSize;
+
     tunnelState->connectReqWriting = false;
     tunnelState->connectExchangeCheckpoint();
 }
@@ -943,6 +977,7 @@ tunnelConnected(const Comm::ConnectionPointer &server, void *data)
     if (!tunnelState->clientExpectsConnectResponse())
         tunnelStartShoveling(tunnelState); // ssl-bumped connection, be quiet
     else {
+        *tunnelState->status_ptr = Http::scOkay;
         AsyncCall::Pointer call = commCbCall(5,5, "tunnelConnectedWriteDone",
                                              CommIoCbPtrFun(tunnelConnectedWriteDone, tunnelState));
         tunnelState->client.write(conn_established, strlen(conn_established), call, NULL);
@@ -965,45 +1000,47 @@ tunnelErrorComplete(int fd/*const Comm::ConnectionPointer &*/, void *data, size_
         tunnelState->server.conn->close();
 }
 
+/// reacts to a failure to establish the given TCP connection
+void
+TunnelStateData::noteConnectFailure(const Comm::ConnectionPointer &conn)
+{
+    debugs(26, 4, "removing the failed one from " << serverDestinations.size() <<
+           " destinations: " << conn);
+
+    if (CachePeer *peer = conn->getPeer())
+        peerConnectFailed(peer);
+
+    assert(!serverDestinations.empty());
+    serverDestinations.erase(serverDestinations.begin());
+
+    // Since no TCP payload has been passed to client or server, we may
+    // TCP-connect to other destinations (including alternate IPs).
+
+    if (!FwdState::EnoughTimeToReForward(startTime))
+        return sendError(savedError, "forwarding timeout");
+
+    if (!serverDestinations.empty())
+        return startConnecting();
+
+    if (!PeerSelectionInitiator::subscribed)
+        return sendError(savedError, "tried all destinations");
+
+    debugs(26, 4, "wait for more destinations to try");
+    // expect a noteDestination*() call
+}
+
 static void
 tunnelConnectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno, void *data)
 {
     TunnelStateData *tunnelState = (TunnelStateData *)data;
 
     if (status != Comm::OK) {
-        debugs(26, 4, HERE << conn << ", comm failure recovery.");
-        /* At this point only the TCP handshake has failed. no data has been passed.
-         * we are allowed to re-try the TCP-level connection to alternate IPs for CONNECT.
-         */
-        debugs(26, 4, "removing server 1 of " << tunnelState->serverDestinations.size() <<
-               " from destinations (" << tunnelState->serverDestinations[0] << ")");
-        tunnelState->serverDestinations.erase(tunnelState->serverDestinations.begin());
-        time_t fwdTimeout = tunnelState->started + Config.Timeout.forward;
-        if (fwdTimeout > squid_curtime && tunnelState->serverDestinations.size() > 0) {
-            // find remaining forward_timeout available for this attempt
-            fwdTimeout -= squid_curtime;
-            if (fwdTimeout > Config.Timeout.connect)
-                fwdTimeout = Config.Timeout.connect;
-            /* Try another IP of this destination host */
-            GetMarkingsToServer(tunnelState->request.getRaw(), *tunnelState->serverDestinations[0]);
-            debugs(26, 4, HERE << "retry with : " << tunnelState->serverDestinations[0]);
-            AsyncCall::Pointer call = commCbCall(26,3, "tunnelConnectDone", CommConnectCbPtrFun(tunnelConnectDone, tunnelState));
-            Comm::ConnOpener *cs = new Comm::ConnOpener(tunnelState->serverDestinations[0], call, fwdTimeout);
-            cs->setHost(tunnelState->url);
-            AsyncJob::Start(cs);
-        } else {
-            debugs(26, 4, HERE << "terminate with error.");
-            ErrorState *err = new ErrorState(ERR_CONNECT_FAIL, Http::scServiceUnavailable, tunnelState->request.getRaw());
-            *tunnelState->status_ptr = Http::scServiceUnavailable;
-            err->xerrno = xerrno;
-            // on timeout is this still:    err->xerrno = ETIMEDOUT;
-            err->port = conn->remote.port();
-            err->callback = tunnelErrorComplete;
-            err->callback_data = tunnelState;
-            errorSend(tunnelState->client.conn, err);
-            if (tunnelState->request != NULL)
-                tunnelState->request->hier.stopPeerClock(false);
-        }
+        ErrorState *err = new ErrorState(ERR_CONNECT_FAIL, Http::scServiceUnavailable, tunnelState->request.getRaw());
+        err->xerrno = xerrno;
+        // on timeout is this still:    err->xerrno = ETIMEDOUT;
+        err->port = conn->remote.port();
+        tunnelState->saveError(err);
+        tunnelState->noteConnectFailure(conn);
         return;
     }
 
@@ -1013,7 +1050,7 @@ tunnelConnectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, int xe
         tunnelState->server.setDelayId(DelayId());
 #endif
 
-    tunnelState->request->hier.note(conn, tunnelState->getHost());
+    tunnelState->request->hier.resetPeerNotes(conn, tunnelState->getHost());
 
     tunnelState->server.conn = conn;
     tunnelState->request->peer_host = conn->getPeer() ? conn->getPeer()->host : NULL;
@@ -1065,9 +1102,11 @@ tunnelStart(ClientHttpRequest * http)
          * default is to allow.
          */
         ACLFilledChecklist ch(Config.accessList.miss, request, NULL);
+        ch.al = http->al;
         ch.src_addr = request->client_addr;
         ch.my_addr = request->my_addr;
-        if (ch.fastCheck() == ACCESS_DENIED) {
+        ch.syncAle(request, http->log_uri);
+        if (ch.fastCheck().denied()) {
             debugs(26, 4, HERE << "MISS access forbidden.");
             err = new ErrorState(ERR_FORWARDING_DENIED, Http::scForbidden, request);
             http->al->http.code = Http::scForbidden;
@@ -1080,51 +1119,26 @@ tunnelStart(ClientHttpRequest * http)
     ++statCounter.server.all.requests;
     ++statCounter.server.other.requests;
 
-    tunnelState = new TunnelStateData;
+    tunnelState = new TunnelStateData(http);
 #if USE_DELAY_POOLS
-    tunnelState->server.setDelayId(DelayId::DelayClient(http));
+    //server.setDelayId called from tunnelConnectDone after server side connection established
 #endif
-    tunnelState->url = xstrdup(url);
-    tunnelState->request = request;
-    tunnelState->server.size_ptr = &http->out.size;
-    tunnelState->client.size_ptr = &http->al->http.clientRequestSz.payloadData;
-    tunnelState->status_ptr = &http->al->http.code;
-    tunnelState->logTag_ptr = &http->logType;
-    tunnelState->client.conn = http->getConn()->clientConnection;
-    tunnelState->http = http;
-    tunnelState->al = http->al;
-    //tunnelState->started is set in TunnelStateData ctor
-
-    comm_add_close_handler(tunnelState->client.conn->fd,
-                           tunnelClientClosed,
-                           tunnelState);
-
-    AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
-                                     CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
-    commSetConnTimeout(tunnelState->client.conn, Config.Timeout.lifetime, timeoutCall);
-
-    peerSelect(&(tunnelState->serverDestinations), request, http->al,
-               NULL,
-               tunnelPeerSelectComplete,
-               tunnelState);
+    tunnelState->startSelectingDestinations(request, http->al, nullptr);
 }
 
 void
 TunnelStateData::connectToPeer()
 {
-#if USE_OPENSSL
     if (CachePeer *p = server.conn->getPeer()) {
         if (p->secure.encryptTransport) {
             AsyncCall::Pointer callback = asyncCall(5,4,
                                                     "TunnelStateData::ConnectedToPeer",
                                                     MyAnswerDialer(&TunnelStateData::connectedToPeer, this));
-            Ssl::BlindPeerConnector *connector =
-                new Ssl::BlindPeerConnector(request, server.conn, callback, al);
+            auto *connector = new Security::BlindPeerConnector(request, server.conn, callback, al);
             AsyncJob::Start(connector); // will call our callback
             return;
         }
     }
-#endif
 
     Security::EncryptorAnswer nil;
     connectedToPeer(nil);
@@ -1134,11 +1148,8 @@ void
 TunnelStateData::connectedToPeer(Security::EncryptorAnswer &answer)
 {
     if (ErrorState *error = answer.error.get()) {
-        *status_ptr = error->httpStatus;
-        error->callback = tunnelErrorComplete;
-        error->callback_data = this;
-        errorSend(client.conn, error);
-        answer.error.clear(); // preserve error for errorSendComplete()
+        answer.error.clear(); // sendError() will own the error
+        sendError(error, "TLS peer connection error");
         return;
     }
 
@@ -1151,7 +1162,7 @@ tunnelRelayConnectRequest(const Comm::ConnectionPointer &srv, void *data)
     TunnelStateData *tunnelState = (TunnelStateData *)data;
     assert(!tunnelState->waitingForConnectExchange());
     HttpHeader hdr_out(hoRequest);
-    HttpStateFlags flags;
+    Http::StateFlags flags;
     debugs(26, 3, HERE << srv << ", tunnelState=" << tunnelState);
     memset(&flags, '\0', sizeof(flags));
     flags.proxying = tunnelState->request->flags.proxying;
@@ -1180,7 +1191,7 @@ tunnelRelayConnectRequest(const Comm::ConnectionPointer &srv, void *data)
     tunnelState->connectRespBuf = new MemBuf;
     // SQUID_TCP_SO_RCVBUF: we should not accumulate more than regular I/O buffer
     // can hold since any CONNECT response leftovers have to fit into server.buf.
-    // 2*SQUID_TCP_SO_RCVBUF: HttpMsg::parse() zero-terminates, which uses space.
+    // 2*SQUID_TCP_SO_RCVBUF: Http::Message::parse() zero-terminates, which uses space.
     tunnelState->connectRespBuf->init(SQUID_TCP_SO_RCVBUF, 2*SQUID_TCP_SO_RCVBUF);
     tunnelState->readConnectResponse();
 
@@ -1191,35 +1202,114 @@ tunnelRelayConnectRequest(const Comm::ConnectionPointer &srv, void *data)
     commSetConnTimeout(srv, Config.Timeout.read, timeoutCall);
 }
 
-static void
-tunnelPeerSelectComplete(Comm::ConnectionList *peer_paths, ErrorState *err, void *data)
+static Comm::ConnectionPointer
+borrowPinnedConnection(HttpRequest *request, Comm::ConnectionPointer &serverDestination)
 {
-    TunnelStateData *tunnelState = (TunnelStateData *)data;
+    // pinned_connection may become nil after a pconn race
+    if (ConnStateData *pinned_connection = request ? request->pinnedConnection() : nullptr) {
+        Comm::ConnectionPointer serverConn = pinned_connection->borrowPinnedConnection(request, serverDestination->getPeer());
+        return serverConn;
+    }
 
-    if (peer_paths == NULL || peer_paths->size() < 1) {
-        debugs(26, 3, HERE << "No paths found. Aborting CONNECT");
-        if (!err) {
-            err = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, tunnelState->request.getRaw());
-        }
-        *tunnelState->status_ptr = err->httpStatus;
-        err->callback = tunnelErrorComplete;
-        err->callback_data = tunnelState;
-        errorSend(tunnelState->client.conn, err);
-        return;
+    return nullptr;
+}
+
+void
+TunnelStateData::noteDestination(Comm::ConnectionPointer path)
+{
+    const bool wasBlocked = serverDestinations.empty();
+    serverDestinations.push_back(path);
+    if (wasBlocked)
+        startConnecting();
+    // else continue to use one of the previously noted destinations;
+    // if all of them fail, we may try this path
+}
+
+void
+TunnelStateData::noteDestinationsEnd(ErrorState *selectionError)
+{
+    PeerSelectionInitiator::subscribed = false;
+    if (serverDestinations.empty()) { // was blocked, waiting for more paths
+
+        if (selectionError)
+            return sendError(selectionError, "path selection has failed");
+
+        if (savedError)
+            return sendError(savedError, "all found paths have failed");
+
+        return sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw()),
+                         "path selection found no paths");
     }
-    delete err;
+    // else continue to use one of the previously noted destinations;
+    // if all of them fail, tunneling as whole will fail
+    Must(!selectionError); // finding at least one path means selection succeeded
+}
+
+/// remembers an error to be used if there will be no more connection attempts
+void
+TunnelStateData::saveError(ErrorState *error)
+{
+    debugs(26, 4, savedError << " ? " << error);
+    assert(error);
+    delete savedError; // may be nil
+    savedError = error;
+}
 
-    GetMarkingsToServer(tunnelState->request.getRaw(), *tunnelState->serverDestinations[0]);
+/// Starts sending the given error message to the client, leading to the
+/// eventual transaction termination. Call with savedError to send savedError.
+void
+TunnelStateData::sendError(ErrorState *finalError, const char *reason)
+{
+    debugs(26, 3, "aborting transaction for " << reason);
 
-    if (tunnelState->request != NULL)
-        tunnelState->request->hier.startPeerClock();
+    if (request)
+        request->hier.stopPeerClock(false);
+
+    assert(finalError);
+
+    // get rid of any cached error unless that is what the caller is sending
+    if (savedError != finalError)
+        delete savedError; // may be nil
+    savedError = nullptr;
+
+    // we cannot try other destinations after responding with an error
+    PeerSelectionInitiator::subscribed = false; // may already be false
+
+    *status_ptr = finalError->httpStatus;
+    finalError->callback = tunnelErrorComplete;
+    finalError->callback_data = this;
+    errorSend(client.conn, finalError);
+}
+
+void
+TunnelStateData::startConnecting()
+{
+    if (request)
+        request->hier.startPeerClock();
+
+    assert(!serverDestinations.empty());
+    Comm::ConnectionPointer &dest = serverDestinations.front();
+    debugs(26, 3, "to " << dest);
+
+    if (dest->peerType == PINNED) {
+        Comm::ConnectionPointer serverConn = borrowPinnedConnection(request.getRaw(), dest);
+        debugs(26,7, "pinned peer connection: " << serverConn);
+        if (Comm::IsConnOpen(serverConn)) {
+            tunnelConnectDone(serverConn, Comm::OK, 0, (void *)this);
+            return;
+        }
+        // a PINNED path failure is fatal; do not wait for more paths
+        sendError(new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request.getRaw()),
+                  "pinned path failure");
+        return;
+    }
 
-    debugs(26, 3, HERE << "paths=" << peer_paths->size() << ", p[0]={" << (*peer_paths)[0] << "}, serverDest[0]={" <<
-           tunnelState->serverDestinations[0] << "}");
+    GetMarkingsToServer(request.getRaw(), *dest);
 
-    AsyncCall::Pointer call = commCbCall(26,3, "tunnelConnectDone", CommConnectCbPtrFun(tunnelConnectDone, tunnelState));
-    Comm::ConnOpener *cs = new Comm::ConnOpener(tunnelState->serverDestinations[0], call, Config.Timeout.connect);
-    cs->setHost(tunnelState->url);
+    const time_t connectTimeout = dest->connectTimeout(startTime);
+    AsyncCall::Pointer call = commCbCall(26,3, "tunnelConnectDone", CommConnectCbPtrFun(tunnelConnectDone, this));
+    Comm::ConnOpener *cs = new Comm::ConnOpener(dest, call, connectTimeout);
+    cs->setHost(url);
     AsyncJob::Start(cs);
 }
 
@@ -1245,71 +1335,51 @@ void
 switchToTunnel(HttpRequest *request, Comm::ConnectionPointer &clientConn, Comm::ConnectionPointer &srvConn)
 {
     debugs(26,5, "Revert to tunnel FD " << clientConn->fd << " with FD " << srvConn->fd);
-    /* Create state structure. */
-    TunnelStateData *tunnelState = NULL;
-    const SBuf url(request->effectiveRequestUri());
 
-    debugs(26, 3, request->method << " " << url << " " << request->http_ver);
+    /* Create state structure. */
     ++statCounter.server.all.requests;
     ++statCounter.server.other.requests;
 
-    tunnelState = new TunnelStateData;
-    tunnelState->url = SBufToCstring(url);
-    tunnelState->request = request;
-    tunnelState->server.size_ptr = NULL; //Set later if Http::Stream is available
+    auto conn = request->clientConnectionManager.get();
+    Must(conn);
+    Http::StreamPointer context = conn->pipeline.front();
+    Must(context && context->http);
 
-    // Temporary static variable to store the unneeded for our case status code
-    static int status_code = 0;
-    tunnelState->status_ptr = &status_code;
-    tunnelState->client.conn = clientConn;
-
-    ConnStateData *conn;
-    if ((conn = request->clientConnectionManager.get())) {
-        Http::StreamPointer context = conn->pipeline.front();
-        if (context != nullptr && context->http != nullptr) {
-            tunnelState->logTag_ptr = &context->http->logType;
-            tunnelState->server.size_ptr = &context->http->out.size;
-            tunnelState->al = context->http->al;
-
-#if USE_DELAY_POOLS
-            /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
-            if (srvConn->getPeer() && srvConn->getPeer()->options.no_delay)
-                tunnelState->server.setDelayId(DelayId::DelayClient(context->http));
-#endif
-        }
-    }
+    debugs(26, 3, request->method << " " << context->http->uri << " " << request->http_ver);
 
-    comm_add_close_handler(tunnelState->client.conn->fd,
-                           tunnelClientClosed,
-                           tunnelState);
+    TunnelStateData *tunnelState = new TunnelStateData(context->http);
 
-    AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
-                                     CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
-    commSetConnTimeout(tunnelState->client.conn, Config.Timeout.lifetime, timeoutCall);
     fd_table[clientConn->fd].read_method = &default_read_method;
     fd_table[clientConn->fd].write_method = &default_write_method;
 
-    tunnelState->request->hier.note(srvConn, tunnelState->getHost());
+    request->hier.resetPeerNotes(srvConn, tunnelState->getHost());
 
     tunnelState->server.conn = srvConn;
-    tunnelState->request->peer_host = srvConn->getPeer() ? srvConn->getPeer()->host : NULL;
+
+#if USE_DELAY_POOLS
+    /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
+    if (srvConn->getPeer() && srvConn->getPeer()->options.no_delay)
+        tunnelState->server.setDelayId(DelayId::DelayClient(context->http));
+#endif
+
+    request->peer_host = srvConn->getPeer() ? srvConn->getPeer()->host : nullptr;
     comm_add_close_handler(srvConn->fd, tunnelServerClosed, tunnelState);
 
     debugs(26, 4, "determine post-connect handling pathway.");
     if (srvConn->getPeer()) {
-        tunnelState->request->peer_login = srvConn->getPeer()->login;
-        tunnelState->request->peer_domain = srvConn->getPeer()->domain;
-        tunnelState->request->flags.auth_no_keytab = srvConn->getPeer()->options.auth_no_keytab;
-        tunnelState->request->flags.proxying = !(srvConn->getPeer()->options.originserver);
+        request->peer_login = srvConn->getPeer()->login;
+        request->peer_domain = srvConn->getPeer()->domain;
+        request->flags.auth_no_keytab = srvConn->getPeer()->options.auth_no_keytab;
+        request->flags.proxying = !(srvConn->getPeer()->options.originserver);
     } else {
-        tunnelState->request->peer_login = NULL;
-        tunnelState->request->peer_domain = NULL;
-        tunnelState->request->flags.auth_no_keytab = false;
-        tunnelState->request->flags.proxying = false;
+        request->peer_login = nullptr;
+        request->peer_domain = nullptr;
+        request->flags.auth_no_keytab = false;
+        request->flags.proxying = false;
     }
 
-    timeoutCall = commCbCall(5, 4, "tunnelTimeout",
-                             CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
+    AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "tunnelTimeout",
+                                     CommTimeoutCbPtrFun(tunnelTimeout, tunnelState));
     commSetConnTimeout(srvConn, Config.Timeout.read, timeoutCall);
     fd_table[srvConn->fd].read_method = &default_read_method;
     fd_table[srvConn->fd].write_method = &default_write_method;
@@ -1317,7 +1387,7 @@ switchToTunnel(HttpRequest *request, Comm::ConnectionPointer &clientConn, Comm::
     auto ssl = fd_table[srvConn->fd].ssl.get();
     assert(ssl);
     BIO *b = SSL_get_rbio(ssl);
-    Ssl::ServerBio *srvBio = static_cast<Ssl::ServerBio *>(b->ptr);
+    Ssl::ServerBio *srvBio = static_cast<Ssl::ServerBio *>(BIO_get_data(b));
     tunnelState->preReadServerData = srvBio->rBufData();
     tunnelStartShoveling(tunnelState);
 }