2 * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 17 Request Forwarding */
12 #include "AccessLogEntry.h"
13 #include "acl/Address.h"
14 #include "acl/FilledChecklist.h"
15 #include "acl/Gadgets.h"
16 #include "anyp/PortCfg.h"
17 #include "CacheManager.h"
18 #include "CachePeer.h"
19 #include "client_side.h"
20 #include "clients/forward.h"
21 #include "comm/Connection.h"
22 #include "comm/ConnOpener.h"
23 #include "comm/Loops.h"
24 #include "CommCalls.h"
25 #include "errorpage.h"
32 #include "hier_code.h"
34 #include "http/Stream.h"
35 #include "HttpReply.h"
36 #include "HttpRequest.h"
37 #include "icmp/net_db.h"
39 #include "ip/Intercept.h"
40 #include "ip/QosConfig.h"
42 #include "MemObject.h"
43 #include "mgr/Registration.h"
44 #include "neighbors.h"
46 #include "PeerPoolMgr.h"
47 #include "security/BlindPeerConnector.h"
48 #include "SquidConfig.h"
49 #include "SquidTime.h"
50 #include "ssl/PeekingPeerConnector.h"
52 #include "StoreClient.h"
56 #include "ssl/cert_validate_message.h"
57 #include "ssl/Config.h"
58 #include "ssl/ErrorDetail.h"
59 #include "ssl/helper.h"
60 #include "ssl/ServerBump.h"
61 #include "ssl/support.h"
63 #include "security/EncryptorAnswer.h"
68 static CLCB fwdServerClosedWrapper
;
69 static CNCB fwdConnectDoneWrapper
;
73 #define MAX_FWD_STATS_IDX 9
74 static int FwdReplyCodes
[MAX_FWD_STATS_IDX
+ 1][Http::scInvalidHeader
+ 1];
76 static PconnPool
*fwdPconnPool
= new PconnPool("server-peers", NULL
);
77 CBDATA_CLASS_INIT(FwdState
);
79 class FwdStatePeerAnswerDialer
: public CallDialer
, public Security::PeerConnector::CbDialer
82 typedef void (FwdState::*Method
)(Security::EncryptorAnswer
&);
84 FwdStatePeerAnswerDialer(Method method
, FwdState
*fwd
):
85 method_(method
), fwd_(fwd
), answer_() {}
88 virtual bool canDial(AsyncCall
&call
) { return fwd_
.valid(); }
89 void dial(AsyncCall
&call
) { ((&(*fwd_
))->*method_
)(answer_
); }
90 virtual void print(std::ostream
&os
) const {
91 os
<< '(' << fwd_
.get() << ", " << answer_
<< ')';
94 /* Security::PeerConnector::CbDialer API */
95 virtual Security::EncryptorAnswer
&answer() { return answer_
; }
99 CbcPointer
<FwdState
> fwd_
;
100 Security::EncryptorAnswer answer_
;
104 FwdState::abort(void* d
)
106 FwdState
* fwd
= (FwdState
*)d
;
107 Pointer tmp
= fwd
; // Grab a temporary pointer to keep the object alive during our scope.
109 if (Comm::IsConnOpen(fwd
->serverConnection())) {
110 fwd
->closeServerConnection("store entry aborted");
112 debugs(17, 7, HERE
<< "store entry aborted; no connection to close");
114 fwd
->serverDestinations
.clear();
115 fwd
->stopAndDestroy("store entry aborted");
119 FwdState::closeServerConnection(const char *reason
)
121 debugs(17, 3, "because " << reason
<< "; " << serverConn
);
122 comm_remove_close_handler(serverConn
->fd
, closeHandler
);
124 fwdPconnPool
->noteUses(fd_table
[serverConn
->fd
].pconn
.uses
);
128 /**** PUBLIC INTERFACE ********************************************************/
130 FwdState::FwdState(const Comm::ConnectionPointer
&client
, StoreEntry
* e
, HttpRequest
* r
, const AccessLogEntryPointer
&alp
):
136 start_t(squid_curtime
),
138 pconnRace(raceImpossible
)
140 debugs(17, 2, "Forwarding client request " << client
<< ", url=" << e
->url());
141 HTTPMSGLOCK(request
);
142 serverDestinations
.reserve(Config
.forward_max_tries
);
144 EBIT_SET(e
->flags
, ENTRY_FWD_HDR_WAIT
);
145 flags
.connected_okay
= false;
146 flags
.dont_retry
= false;
147 flags
.forward_completed
= false;
148 debugs(17, 3, "FwdState constructed, this=" << this);
151 // Called once, right after object creation, when it is safe to set self
152 void FwdState::start(Pointer aSelf
)
154 // Protect ourselves from being destroyed when the only Server pointing
155 // to us is gone (while we expect to talk to more Servers later).
156 // Once we set self, we are responsible for clearing it when we do not
157 // expect to talk to any servers.
158 self
= aSelf
; // refcounted
160 // We hope that either the store entry aborts or peer is selected.
161 // Otherwise we are going to leak our object.
163 // Ftp::Relay needs to preserve control connection on data aborts
164 // so it registers its own abort handler that calls ours when needed.
165 if (!request
->flags
.ftpNative
)
166 entry
->registerAbort(FwdState::abort
, this);
168 #if STRICT_ORIGINAL_DST
169 // Bug 3243: CVE 2009-0801
170 // Bypass of browser same-origin access control in intercepted communication
171 // To resolve this we must force DIRECT and only to the original client destination.
172 const bool isIntercepted
= request
&& !request
->flags
.redirected
&& (request
->flags
.intercepted
|| request
->flags
.interceptTproxy
);
173 const bool useOriginalDst
= Config
.onoff
.client_dst_passthru
|| (request
&& !request
->flags
.hostVerified
);
174 if (isIntercepted
&& useOriginalDst
) {
175 selectPeerForIntercepted();
176 // 3.2 does not suppro re-wrapping inside CONNECT.
177 // our only alternative is to fake destination "found" and continue with the forwarding.
178 startConnectionOrFail();
183 // do full route options selection
184 startSelectingDestinations(request
, al
, entry
);
187 /// ends forwarding; relies on refcounting so the effect may not be immediate
189 FwdState::stopAndDestroy(const char *reason
)
191 debugs(17, 3, "for " << reason
);
192 PeerSelectionInitiator::subscribed
= false; // may already be false
193 self
= nullptr; // we hope refcounting destroys us soon; may already be nil
194 /* do not place any code here as this object may be gone by now */
197 #if STRICT_ORIGINAL_DST
198 /// bypasses peerSelect() when dealing with intercepted requests
200 FwdState::selectPeerForIntercepted()
202 // use pinned connection if available
203 Comm::ConnectionPointer p
;
204 if (ConnStateData
*client
= request
->pinnedConnection()) {
205 p
= client
->validatePinnedConnection(request
, NULL
);
206 if (Comm::IsConnOpen(p
)) {
207 /* duplicate peerSelectPinned() effects */
208 p
->peerType
= PINNED
;
209 entry
->ping_status
= PING_DONE
; /* Skip ICP */
211 debugs(17, 3, "reusing a pinned conn: " << *p
);
212 serverDestinations
.push_back(p
);
214 debugs(17,2, "Pinned connection is not valid: " << p
);
215 ErrorState
*anErr
= new ErrorState(ERR_ZERO_SIZE_OBJECT
, Http::scServiceUnavailable
, request
);
218 // Either use the valid pinned connection or fail if it is invalid.
222 // use client original destination as second preferred choice
223 p
= new Comm::Connection();
224 p
->peerType
= ORIGINAL_DST
;
225 p
->remote
= clientConn
->local
;
226 getOutgoingAddress(request
, p
);
228 debugs(17, 3, HERE
<< "using client original destination: " << *p
);
229 serverDestinations
.push_back(p
);
234 FwdState::completed()
236 if (flags
.forward_completed
) {
237 debugs(17, DBG_IMPORTANT
, HERE
<< "FwdState::completed called on a completed request! Bad!");
241 flags
.forward_completed
= true;
243 request
->hier
.stopPeerClock(false);
245 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
246 debugs(17, 3, HERE
<< "entry aborted");
250 #if URL_CHECKSUM_DEBUG
252 entry
->mem_obj
->checkUrlChecksum();
255 if (entry
->store_status
== STORE_PENDING
) {
256 if (entry
->isEmpty()) {
257 if (!err
) // we quit (e.g., fd closed) before an error or content
258 fail(new ErrorState(ERR_READ_ERROR
, Http::scBadGateway
, request
));
260 errorAppendEntry(entry
, err
);
263 if (request
->flags
.sslPeek
&& request
->clientConnectionManager
.valid()) {
264 CallJobHere1(17, 4, request
->clientConnectionManager
, ConnStateData
,
265 ConnStateData::httpsPeeked
, ConnStateData::PinnedIdleContext(Comm::ConnectionPointer(nullptr), request
));
269 EBIT_CLR(entry
->flags
, ENTRY_FWD_HDR_WAIT
);
271 entry
->releaseRequest();
275 if (storePendingNClients(entry
) > 0)
276 assert(!EBIT_TEST(entry
->flags
, ENTRY_FWD_HDR_WAIT
));
280 FwdState::~FwdState()
282 debugs(17, 3, "FwdState destructor start");
284 if (! flags
.forward_completed
)
289 HTTPMSGUNLOCK(request
);
293 entry
->unregisterAbort();
295 entry
->unlock("FwdState");
299 if (calls
.connector
!= NULL
) {
300 calls
.connector
->cancel("FwdState destructed");
301 calls
.connector
= NULL
;
304 if (Comm::IsConnOpen(serverConn
))
305 closeServerConnection("~FwdState");
307 serverDestinations
.clear();
309 debugs(17, 3, "FwdState destructed, this=" << this);
313 * This is the entry point for client-side to start forwarding
314 * a transaction. It is a static method that may or may not
315 * allocate a FwdState.
318 FwdState::Start(const Comm::ConnectionPointer
&clientConn
, StoreEntry
*entry
, HttpRequest
*request
, const AccessLogEntryPointer
&al
)
321 * client_addr == no_addr indicates this is an "internal" request
322 * from peer_digest.c, asn.c, netdb.c, etc and should always
323 * be allowed. yuck, I know.
326 if ( Config
.accessList
.miss
&& !request
->client_addr
.isNoAddr() &&
327 !request
->flags
.internal
&& request
->url
.getScheme() != AnyP::PROTO_CACHE_OBJECT
) {
329 * Check if this host is allowed to fetch MISSES from us (miss_access).
330 * Intentionally replace the src_addr automatically selected by the checklist code
331 * we do NOT want the indirect client address to be tested here.
333 ACLFilledChecklist
ch(Config
.accessList
.miss
, request
, NULL
);
334 ch
.src_addr
= request
->client_addr
;
335 if (ch
.fastCheck().denied()) {
337 page_id
= aclGetDenyInfoPage(&Config
.denyInfoList
, AclMatchedName
, 1);
339 if (page_id
== ERR_NONE
)
340 page_id
= ERR_FORWARDING_DENIED
;
342 ErrorState
*anErr
= new ErrorState(page_id
, Http::scForbidden
, request
);
343 errorAppendEntry(entry
, anErr
); // frees anErr
348 debugs(17, 3, HERE
<< "'" << entry
->url() << "'");
350 * This seems like an odd place to bind mem_obj and request.
351 * Might want to assert that request is NULL at this point
353 entry
->mem_obj
->request
= request
;
354 #if URL_CHECKSUM_DEBUG
356 entry
->mem_obj
->checkUrlChecksum();
361 ErrorState
*anErr
= new ErrorState(ERR_SHUTTING_DOWN
, Http::scServiceUnavailable
, request
);
362 errorAppendEntry(entry
, anErr
); // frees anErr
366 if (request
->flags
.internal
) {
367 debugs(17, 2, "calling internalStart() due to request flag");
368 internalStart(clientConn
, request
, entry
);
372 switch (request
->url
.getScheme()) {
374 case AnyP::PROTO_CACHE_OBJECT
:
375 debugs(17, 2, "calling CacheManager due to request scheme " << request
->url
.getScheme());
376 CacheManager::GetInstance()->Start(clientConn
, request
, entry
);
379 case AnyP::PROTO_URN
:
380 urnStart(request
, entry
);
384 FwdState::Pointer fwd
= new FwdState(clientConn
, entry
, request
, al
);
393 FwdState::fwdStart(const Comm::ConnectionPointer
&clientConn
, StoreEntry
*entry
, HttpRequest
*request
)
395 // Hides AccessLogEntry.h from code that does not supply ALE anyway.
396 Start(clientConn
, entry
, request
, NULL
);
399 /// subtracts time_t values, returning zero if smaller exceeds the larger value
400 /// time_t might be unsigned so we need to be careful when subtracting times...
402 diffOrZero(const time_t larger
, const time_t smaller
)
404 return (larger
> smaller
) ? (larger
- smaller
) : 0;
407 /// time left to finish the whole forwarding process (which started at fwdStart)
409 FwdState::ForwardTimeout(const time_t fwdStart
)
411 // time already spent on forwarding (0 if clock went backwards)
412 const time_t timeSpent
= diffOrZero(squid_curtime
, fwdStart
);
413 return diffOrZero(Config
.Timeout
.forward
, timeSpent
);
417 FwdState::EnoughTimeToReForward(const time_t fwdStart
)
419 return ForwardTimeout(fwdStart
) > 0;
423 FwdState::startConnectionOrFail()
425 debugs(17, 3, HERE
<< entry
->url());
427 if (serverDestinations
.size() > 0) {
428 // Ditch error page if it was created before.
429 // A new one will be created if there's another problem
433 // Update the logging information about this new server connection.
434 // Done here before anything else so the errors get logged for
435 // this server link regardless of what happens when connecting to it.
436 // IF sucessfuly connected this top destination will become the serverConnection().
437 syncHierNote(serverDestinations
[0], request
->url
.host());
438 request
->clearError();
442 if (PeerSelectionInitiator::subscribed
) {
443 debugs(17, 4, "wait for more destinations to try");
444 return; // expect a noteDestination*() call
447 debugs(17, 3, HERE
<< "Connection failed: " << entry
->url());
449 ErrorState
*anErr
= new ErrorState(ERR_CANNOT_FORWARD
, Http::scInternalServerError
, request
);
451 } // else use actual error from last connection attempt
453 stopAndDestroy("tried all destinations");
458 FwdState::fail(ErrorState
* errorState
)
460 debugs(17, 3, err_type_str
[errorState
->type
] << " \"" << Http::StatusCodeString(errorState
->httpStatus
) << "\"\n\t" << entry
->url());
465 if (!errorState
->request
)
466 errorState
->request
= request
;
468 if (err
->type
!= ERR_ZERO_SIZE_OBJECT
)
471 if (pconnRace
== racePossible
) {
472 debugs(17, 5, HERE
<< "pconn race happened");
473 pconnRace
= raceHappened
;
476 if (ConnStateData
*pinned_connection
= request
->pinnedConnection()) {
477 pinned_connection
->pinning
.zeroReply
= true;
478 flags
.dont_retry
= true; // we want to propagate failure to the client
479 debugs(17, 4, "zero reply on pinned connection");
484 * Frees fwdState without closing FD or generating an abort
487 FwdState::unregister(Comm::ConnectionPointer
&conn
)
489 debugs(17, 3, HERE
<< entry
->url() );
490 assert(serverConnection() == conn
);
491 assert(Comm::IsConnOpen(conn
));
492 comm_remove_close_handler(conn
->fd
, closeHandler
);
497 // \deprecated use unregister(Comm::ConnectionPointer &conn) instead
499 FwdState::unregister(int fd
)
501 debugs(17, 3, HERE
<< entry
->url() );
502 assert(fd
== serverConnection()->fd
);
503 unregister(serverConn
);
507 * FooClient modules call fwdComplete() when they are done
508 * downloading an object. Then, we either 1) re-forward the
509 * request somewhere else if needed, or 2) call storeComplete()
515 debugs(17, 3, HERE
<< entry
->url() << "\n\tstatus " << entry
->getReply()->sline
.status());
516 #if URL_CHECKSUM_DEBUG
518 entry
->mem_obj
->checkUrlChecksum();
521 logReplyStatus(n_tries
, entry
->getReply()->sline
.status());
524 debugs(17, 3, HERE
<< "re-forwarding " << entry
->getReply()->sline
.status() << " " << entry
->url());
526 if (Comm::IsConnOpen(serverConn
))
527 unregister(serverConn
);
531 // drop the last path off the selection list. try the next one.
532 if (!serverDestinations
.empty()) // paranoid
533 serverDestinations
.erase(serverDestinations
.begin());
534 startConnectionOrFail();
537 if (Comm::IsConnOpen(serverConn
))
538 debugs(17, 3, HERE
<< "server FD " << serverConnection()->fd
<< " not re-forwarding status " << entry
->getReply()->sline
.status());
540 debugs(17, 3, HERE
<< "server (FD closed) not re-forwarding status " << entry
->getReply()->sline
.status());
541 EBIT_CLR(entry
->flags
, ENTRY_FWD_HDR_WAIT
);
544 if (!Comm::IsConnOpen(serverConn
))
547 stopAndDestroy("forwarding completed");
552 FwdState::noteDestination(Comm::ConnectionPointer path
)
554 const bool wasBlocked
= serverDestinations
.empty();
555 serverDestinations
.push_back(path
);
557 startConnectionOrFail();
558 // else continue to use one of the previously noted destinations;
559 // if all of them fail, we may try this path
563 FwdState::noteDestinationsEnd(ErrorState
*selectionError
)
565 PeerSelectionInitiator::subscribed
= false;
566 if (serverDestinations
.empty()) { // was blocked, waiting for more paths
568 if (selectionError
) {
569 debugs(17, 3, "Will abort forwarding because path selection has failed.");
570 Must(!err
); // if we tried to connect, then path selection succeeded
571 fail(selectionError
);
574 debugs(17, 3, "Will abort forwarding because all found paths have failed.");
576 debugs(17, 3, "Will abort forwarding because path selection found no paths.");
578 startConnectionOrFail(); // will choose the OrFail code path
581 // else continue to use one of the previously noted destinations;
582 // if all of them fail, forwarding as whole will fail
583 Must(!selectionError
); // finding at least one path means selection succeeded
586 /**** CALLBACK WRAPPERS ************************************************************/
589 fwdServerClosedWrapper(const CommCloseCbParams
¶ms
)
591 FwdState
*fwd
= (FwdState
*)params
.data
;
592 fwd
->serverClosed(params
.fd
);
596 fwdConnectDoneWrapper(const Comm::ConnectionPointer
&conn
, Comm::Flag status
, int xerrno
, void *data
)
598 FwdState
*fwd
= (FwdState
*) data
;
599 fwd
->connectDone(conn
, status
, xerrno
);
602 /**** PRIVATE *****************************************************************/
605 * FwdState::checkRetry
607 * Return TRUE if the request SHOULD be retried. This method is
608 * called when the HTTP connection fails, or when the connection
609 * is closed before reading the end of HTTP headers from the server.
612 FwdState::checkRetry()
617 if (!self
) { // we have aborted before the server called us back
618 debugs(17, 5, HERE
<< "not retrying because of earlier abort");
619 // we will be destroyed when the server clears its Pointer to us
623 if (entry
->store_status
!= STORE_PENDING
)
626 if (!entry
->isEmpty())
629 if (n_tries
> Config
.forward_max_tries
)
632 if (!EnoughTimeToReForward(start_t
))
635 if (flags
.dont_retry
)
638 if (request
->bodyNibbled())
641 // NP: not yet actually connected anywhere. retry is safe.
642 if (!flags
.connected_okay
)
645 if (!checkRetriable())
651 /// Whether we may try sending this request again after a failure.
653 FwdState::checkRetriable()
655 // Optimize: A compliant proxy may retry PUTs, but Squid lacks the [rather
656 // complicated] code required to protect the PUT request body from being
657 // nibbled during the first try. Thus, Squid cannot retry some PUTs today.
658 if (request
->body_pipe
!= NULL
)
661 // RFC2616 9.1 Safe and Idempotent Methods
662 return (request
->method
.isHttpSafe() || request
->method
.isIdempotent());
666 FwdState::serverClosed(int fd
)
668 // XXX: fd is often -1 here
669 debugs(17, 2, "FD " << fd
<< " " << entry
->url() << " after " <<
670 (fd
>= 0 ? fd_table
[fd
].pconn
.uses
: -1) << " requests");
671 if (fd
>= 0 && serverConnection()->fd
== fd
)
672 fwdPconnPool
->noteUses(fd_table
[fd
].pconn
.uses
);
677 FwdState::retryOrBail()
680 debugs(17, 3, HERE
<< "re-forwarding (" << n_tries
<< " tries, " << (squid_curtime
- start_t
) << " secs)");
681 // we should retry the same destination if it failed due to pconn race
682 if (pconnRace
== raceHappened
)
683 debugs(17, 4, HERE
<< "retrying the same destination");
685 serverDestinations
.erase(serverDestinations
.begin()); // last one failed. try another.
686 startConnectionOrFail();
690 // TODO: should we call completed() here and move doneWithRetries there?
693 request
->hier
.stopPeerClock(false);
695 if (self
!= NULL
&& !err
&& shutting_down
&& entry
->isEmpty()) {
696 ErrorState
*anErr
= new ErrorState(ERR_SHUTTING_DOWN
, Http::scServiceUnavailable
, request
);
697 errorAppendEntry(entry
, anErr
);
700 stopAndDestroy("cannot retry");
703 // If the Server quits before nibbling at the request body, the body sender
704 // will not know (so that we can retry). Call this if we will not retry. We
705 // will notify the sender so that it does not get stuck waiting for space.
707 FwdState::doneWithRetries()
709 if (request
&& request
->body_pipe
!= NULL
)
710 request
->body_pipe
->expectNoConsumption();
713 // called by the server that failed after calling unregister()
715 FwdState::handleUnregisteredServerEnd()
717 debugs(17, 2, HERE
<< "self=" << self
<< " err=" << err
<< ' ' << entry
->url());
718 assert(!Comm::IsConnOpen(serverConn
));
723 FwdState::connectDone(const Comm::ConnectionPointer
&conn
, Comm::Flag status
, int xerrno
)
725 if (status
!= Comm::OK
) {
726 ErrorState
*const anErr
= makeConnectingError(ERR_CONNECT_FAIL
);
727 anErr
->xerrno
= xerrno
;
730 /* it might have been a timeout with a partially open link */
733 peerConnectFailed(conn
->getPeer());
742 debugs(17, 3, HERE
<< serverConnection() << ": '" << entry
->url() << "'" );
744 closeHandler
= comm_add_close_handler(serverConnection()->fd
, fwdServerClosedWrapper
, this);
746 if (!request
->flags
.pinned
) {
747 const CachePeer
*p
= serverConnection()->getPeer();
748 const bool peerWantsTls
= p
&& p
->secure
.encryptTransport
;
749 // userWillTlsToPeerForUs assumes CONNECT == HTTPS
750 const bool userWillTlsToPeerForUs
= p
&& p
->options
.originserver
&&
751 request
->method
== Http::METHOD_CONNECT
;
752 const bool needTlsToPeer
= peerWantsTls
&& !userWillTlsToPeerForUs
;
753 const bool needTlsToOrigin
= !p
&& request
->url
.getScheme() == AnyP::PROTO_HTTPS
;
754 if (needTlsToPeer
|| needTlsToOrigin
|| request
->flags
.sslPeek
) {
755 HttpRequest::Pointer requestPointer
= request
;
756 AsyncCall::Pointer callback
= asyncCall(17,4,
757 "FwdState::ConnectedToPeer",
758 FwdStatePeerAnswerDialer(&FwdState::connectedToPeer
, this));
759 // Use positive timeout when less than one second is left.
760 const time_t connTimeout
= serverDestinations
[0]->connectTimeout(start_t
);
761 const time_t sslNegotiationTimeout
= positiveTimeout(connTimeout
);
762 Security::PeerConnector
*connector
= nullptr;
764 if (request
->flags
.sslPeek
)
765 connector
= new Ssl::PeekingPeerConnector(requestPointer
, serverConnection(), clientConn
, callback
, al
, sslNegotiationTimeout
);
768 connector
= new Security::BlindPeerConnector(requestPointer
, serverConnection(), callback
, al
, sslNegotiationTimeout
);
769 AsyncJob::Start(connector
); // will call our callback
774 // if not encrypting just run the post-connect actions
775 Security::EncryptorAnswer nil
;
776 connectedToPeer(nil
);
780 FwdState::connectedToPeer(Security::EncryptorAnswer
&answer
)
782 if (ErrorState
*error
= answer
.error
.get()) {
784 answer
.error
.clear(); // preserve error for errorSendComplete()
785 if (CachePeer
*p
= serverConnection()->getPeer())
786 peerConnectFailed(p
);
787 serverConnection()->close();
791 if (answer
.tunneled
) {
792 // TODO: When ConnStateData establishes tunnels, its state changes
793 // [in ways that may affect logging?]. Consider informing
794 // ConnStateData about our tunnel or otherwise unifying tunnel
795 // establishment [side effects].
796 unregister(serverConn
); // async call owns it now
797 complete(); // destroys us
801 // should reach ConnStateData before the dispatched Client job starts
802 CallJobHere1(17, 4, request
->clientConnectionManager
, ConnStateData
,
803 ConnStateData::notePeerConnection
, serverConnection());
805 if (serverConnection()->getPeer())
806 peerConnectSucceded(serverConnection()->getPeer());
808 flags
.connected_okay
= true;
813 FwdState::connectTimeout(int fd
)
815 debugs(17, 2, "fwdConnectTimeout: FD " << fd
<< ": '" << entry
->url() << "'" );
816 assert(serverDestinations
[0] != NULL
);
817 assert(fd
== serverDestinations
[0]->fd
);
819 if (entry
->isEmpty()) {
820 ErrorState
*anErr
= new ErrorState(ERR_CONNECT_FAIL
, Http::scGatewayTimeout
, request
);
821 anErr
->xerrno
= ETIMEDOUT
;
824 /* This marks the peer DOWN ... */
825 if (serverDestinations
[0]->getPeer())
826 peerConnectFailed(serverDestinations
[0]->getPeer());
829 if (Comm::IsConnOpen(serverDestinations
[0])) {
830 serverDestinations
[0]->close();
834 /// called when serverConn is set to an _open_ to-peer connection
836 FwdState::syncWithServerConn(const char *host
)
838 if (Ip::Qos::TheConfig
.isAclTosActive())
839 Ip::Qos::setSockTos(serverConn
, GetTosToServer(request
));
842 if (Ip::Qos::TheConfig
.isAclNfmarkActive())
843 Ip::Qos::setSockNfmark(serverConn
, GetNfmarkToServer(request
));
846 syncHierNote(serverConn
, host
);
850 FwdState::syncHierNote(const Comm::ConnectionPointer
&server
, const char *host
)
853 request
->hier
.resetPeerNotes(server
, host
);
855 al
->hier
.resetPeerNotes(server
, host
);
859 * Called after forwarding path selection (via peer select) has taken place
860 * and whenever forwarding needs to attempt a new connection (routing failover).
861 * We have a vector of possible localIP->remoteIP paths now ready to start being connected.
864 FwdState::connectStart()
866 assert(serverDestinations
.size() > 0);
868 debugs(17, 3, "fwdConnectStart: " << entry
->url());
870 request
->hier
.startPeerClock();
872 // Do not fowrward bumped connections to parent proxy unless it is an
874 if (serverDestinations
[0]->getPeer() && !serverDestinations
[0]->getPeer()->options
.originserver
&& request
->flags
.sslBumped
) {
875 debugs(50, 4, "fwdConnectStart: Ssl bumped connections through parent proxy are not allowed");
876 ErrorState
*anErr
= new ErrorState(ERR_CANNOT_FORWARD
, Http::scServiceUnavailable
, request
);
878 stopAndDestroy("SslBump misconfiguration");
882 request
->flags
.pinned
= false; // XXX: what if the ConnStateData set this to flag existing credentials?
883 // XXX: answer: the peer selection *should* catch it and give us only the pinned peer. so we reverse the =0 step below.
884 // XXX: also, logs will now lie if pinning is broken and leads to an error message.
885 if (serverDestinations
[0]->peerType
== PINNED
) {
886 ConnStateData
*pinned_connection
= request
->pinnedConnection();
887 debugs(17,7, "pinned peer connection: " << pinned_connection
);
888 // pinned_connection may become nil after a pconn race
889 serverConn
= pinned_connection
? pinned_connection
->borrowPinnedConnection(request
, serverDestinations
[0]->getPeer()) : nullptr;
890 if (Comm::IsConnOpen(serverConn
)) {
891 flags
.connected_okay
= true;
893 request
->flags
.pinned
= true;
895 if (pinned_connection
->pinnedAuth())
896 request
->flags
.auth
= true;
898 closeHandler
= comm_add_close_handler(serverConn
->fd
, fwdServerClosedWrapper
, this);
900 syncWithServerConn(pinned_connection
->pinning
.host
);
902 // the server may close the pinned connection before this request
903 pconnRace
= racePossible
;
908 // Pinned connection failure.
909 debugs(17,2,HERE
<< "Pinned connection failed: " << pinned_connection
);
910 ErrorState
*anErr
= new ErrorState(ERR_ZERO_SIZE_OBJECT
, Http::scServiceUnavailable
, request
);
912 stopAndDestroy("pinned connection failure");
916 // Use pconn to avoid opening a new connection.
917 const char *host
= NULL
;
918 if (!serverDestinations
[0]->getPeer())
919 host
= request
->url
.host();
921 Comm::ConnectionPointer temp
;
922 // Avoid pconns after races so that the same client does not suffer twice.
923 // This does not increase the total number of connections because we just
924 // closed the connection that failed the race. And re-pinning assumes this.
925 if (pconnRace
!= raceHappened
)
926 temp
= pconnPop(serverDestinations
[0], host
);
928 const bool openedPconn
= Comm::IsConnOpen(temp
);
929 pconnRace
= openedPconn
? racePossible
: raceImpossible
;
931 // if we found an open persistent connection to use. use it.
934 flags
.connected_okay
= true;
935 debugs(17, 3, HERE
<< "reusing pconn " << serverConnection());
938 closeHandler
= comm_add_close_handler(serverConnection()->fd
, fwdServerClosedWrapper
, this);
940 syncWithServerConn(request
->url
.host());
946 // We will try to open a new connection, possibly to the same destination.
947 // We reset serverDestinations[0] in case we are using it again because
948 // ConnOpener modifies its destination argument.
949 serverDestinations
[0]->local
.port(0);
952 #if URL_CHECKSUM_DEBUG
953 entry
->mem_obj
->checkUrlChecksum();
956 GetMarkingsToServer(request
, *serverDestinations
[0]);
958 calls
.connector
= commCbCall(17,3, "fwdConnectDoneWrapper", CommConnectCbPtrFun(fwdConnectDoneWrapper
, this));
959 const time_t connTimeout
= serverDestinations
[0]->connectTimeout(start_t
);
960 Comm::ConnOpener
*cs
= new Comm::ConnOpener(serverDestinations
[0], calls
.connector
, connTimeout
);
969 debugs(17, 3, clientConn
<< ": Fetching " << request
->method
<< ' ' << entry
->url());
971 * Assert that server_fd is set. This is to guarantee that fwdState
972 * is attached to something and will be deallocated when server_fd
975 assert(Comm::IsConnOpen(serverConn
));
977 fd_note(serverConnection()->fd
, entry
->url());
979 fd_table
[serverConnection()->fd
].noteUse();
981 /*assert(!EBIT_TEST(entry->flags, ENTRY_DISPATCHED)); */
982 assert(entry
->ping_status
!= PING_WAITING
);
984 assert(entry
->locked());
986 EBIT_SET(entry
->flags
, ENTRY_DISPATCHED
);
988 netdbPingSite(request
->url
.host());
990 /* Retrieves remote server TOS or MARK value, and stores it as part of the
991 * original client request FD object. It is later used to forward
992 * remote server's TOS/MARK in the response to the client in case of a MISS.
994 if (Ip::Qos::TheConfig
.isHitNfmarkActive()) {
995 if (Comm::IsConnOpen(clientConn
) && Comm::IsConnOpen(serverConnection())) {
996 fde
* clientFde
= &fd_table
[clientConn
->fd
]; // XXX: move the fd_table access into Ip::Qos
997 /* Get the netfilter mark for the connection */
998 Ip::Qos::getNfmarkFromServer(serverConnection(), clientFde
);
1003 /* Bug 2537: The TOS forward part of QOS only applies to patched Linux kernels. */
1004 if (Ip::Qos::TheConfig
.isHitTosActive()) {
1005 if (Comm::IsConnOpen(clientConn
)) {
1006 fde
* clientFde
= &fd_table
[clientConn
->fd
]; // XXX: move the fd_table access into Ip::Qos
1007 /* Get the TOS value for the packet */
1008 Ip::Qos::getTosFromServer(serverConnection(), clientFde
);
1014 if (request
->flags
.sslPeek
) {
1015 CallJobHere1(17, 4, request
->clientConnectionManager
, ConnStateData
,
1016 ConnStateData::httpsPeeked
, ConnStateData::PinnedIdleContext(serverConnection(), request
));
1017 unregister(serverConn
); // async call owns it now
1018 complete(); // destroys us
1023 if (serverConnection()->getPeer() != NULL
) {
1024 ++ serverConnection()->getPeer()->stats
.fetches
;
1025 request
->peer_login
= serverConnection()->getPeer()->login
;
1026 request
->peer_domain
= serverConnection()->getPeer()->domain
;
1027 request
->flags
.auth_no_keytab
= serverConnection()->getPeer()->options
.auth_no_keytab
;
1030 assert(!request
->flags
.sslPeek
);
1031 request
->peer_login
= NULL
;
1032 request
->peer_domain
= NULL
;
1033 request
->flags
.auth_no_keytab
= 0;
1035 switch (request
->url
.getScheme()) {
1037 case AnyP::PROTO_HTTPS
:
1041 case AnyP::PROTO_HTTP
:
1045 case AnyP::PROTO_GOPHER
:
1049 case AnyP::PROTO_FTP
:
1050 if (request
->flags
.ftpNative
)
1051 Ftp::StartRelay(this);
1053 Ftp::StartGateway(this);
1056 case AnyP::PROTO_CACHE_OBJECT
:
1058 case AnyP::PROTO_URN
:
1059 fatal_dump("Should never get here");
1062 case AnyP::PROTO_WHOIS
:
1066 case AnyP::PROTO_WAIS
: /* Not implemented */
1069 debugs(17, DBG_IMPORTANT
, "WARNING: Cannot retrieve '" << entry
->url() << "'.");
1070 ErrorState
*anErr
= new ErrorState(ERR_UNSUP_REQ
, Http::scBadRequest
, request
);
1072 // Set the dont_retry flag because this is not a transient (network) error.
1073 flags
.dont_retry
= true;
1074 if (Comm::IsConnOpen(serverConn
)) {
1075 serverConn
->close();
1083 * FwdState::reforward
1085 * returns TRUE if the transaction SHOULD be re-forwarded to the
1086 * next choice in the serverDestinations list. This method is called when
1087 * peer communication completes normally, or experiences
1088 * some error after receiving the end of HTTP headers.
1091 FwdState::reforward()
1093 StoreEntry
*e
= entry
;
1095 if (EBIT_TEST(e
->flags
, ENTRY_ABORTED
)) {
1096 debugs(17, 3, HERE
<< "entry aborted");
1100 assert(e
->store_status
== STORE_PENDING
);
1102 #if URL_CHECKSUM_DEBUG
1104 e
->mem_obj
->checkUrlChecksum();
1107 debugs(17, 3, HERE
<< e
->url() << "?" );
1109 if (!EBIT_TEST(e
->flags
, ENTRY_FWD_HDR_WAIT
)) {
1110 debugs(17, 3, HERE
<< "No, ENTRY_FWD_HDR_WAIT isn't set");
1114 if (n_tries
> Config
.forward_max_tries
)
1117 if (request
->bodyNibbled())
1120 if (serverDestinations
.size() <= 1 && !PeerSelectionInitiator::subscribed
) {
1121 // NP: <= 1 since total count includes the recently failed one.
1122 debugs(17, 3, HERE
<< "No alternative forwarding paths left");
1126 const Http::StatusCode s
= e
->getReply()->sline
.status();
1127 debugs(17, 3, HERE
<< "status " << s
);
1128 return reforwardableStatus(s
);
1132 * Create "503 Service Unavailable" or "504 Gateway Timeout" error depending
1133 * on whether this is a validation request. RFC 2616 says that we MUST reply
1134 * with "504 Gateway Timeout" if validation fails and cached reply has
1135 * proxy-revalidate, must-revalidate or s-maxage Cache-Control directive.
1138 FwdState::makeConnectingError(const err_type type
) const
1140 return new ErrorState(type
, request
->flags
.needValidation
?
1141 Http::scGatewayTimeout
: Http::scServiceUnavailable
, request
);
1145 fwdStats(StoreEntry
* s
)
1149 storeAppendPrintf(s
, "Status");
1151 for (j
= 1; j
< MAX_FWD_STATS_IDX
; ++j
) {
1152 storeAppendPrintf(s
, "\ttry#%d", j
);
1155 storeAppendPrintf(s
, "\n");
1157 for (i
= 0; i
<= (int) Http::scInvalidHeader
; ++i
) {
1158 if (FwdReplyCodes
[0][i
] == 0)
1161 storeAppendPrintf(s
, "%3d", i
);
1163 for (j
= 0; j
<= MAX_FWD_STATS_IDX
; ++j
) {
1164 storeAppendPrintf(s
, "\t%d", FwdReplyCodes
[j
][i
]);
1167 storeAppendPrintf(s
, "\n");
1171 /**** STATIC MEMBER FUNCTIONS *************************************************/
1174 FwdState::reforwardableStatus(const Http::StatusCode s
) const
1178 case Http::scBadGateway
:
1180 case Http::scGatewayTimeout
:
1183 case Http::scForbidden
:
1185 case Http::scInternalServerError
:
1187 case Http::scNotImplemented
:
1189 case Http::scServiceUnavailable
:
1190 return Config
.retry
.onerror
;
1200 * Decide where details need to be gathered to correctly describe a persistent connection.
1202 * - the address/port details about this link
1203 * - domain name of server at other end of this link (either peer or requested host)
1206 FwdState::pconnPush(Comm::ConnectionPointer
&conn
, const char *domain
)
1208 if (conn
->getPeer()) {
1209 fwdPconnPool
->push(conn
, NULL
);
1211 fwdPconnPool
->push(conn
, domain
);
1215 Comm::ConnectionPointer
1216 FwdState::pconnPop(const Comm::ConnectionPointer
&dest
, const char *domain
)
1218 bool retriable
= checkRetriable();
1219 if (!retriable
&& Config
.accessList
.serverPconnForNonretriable
) {
1220 ACLFilledChecklist
ch(Config
.accessList
.serverPconnForNonretriable
, request
, NULL
);
1221 retriable
= ch
.fastCheck().allowed();
1223 // always call shared pool first because we need to close an idle
1224 // connection there if we have to use a standby connection.
1225 Comm::ConnectionPointer conn
= fwdPconnPool
->pop(dest
, domain
, retriable
);
1226 if (!Comm::IsConnOpen(conn
)) {
1227 // either there was no pconn to pop or this is not a retriable xaction
1228 if (CachePeer
*peer
= dest
->getPeer()) {
1229 if (peer
->standby
.pool
)
1230 conn
= peer
->standby
.pool
->pop(dest
, domain
, true);
1233 return conn
; // open, closed, or nil
1237 FwdState::initModule()
1239 RegisterWithCacheManager();
1243 FwdState::RegisterWithCacheManager(void)
1245 Mgr::RegisterAction("forward", "Request Forwarding Statistics", fwdStats
, 0, 1);
1249 FwdState::logReplyStatus(int tries
, const Http::StatusCode status
)
1251 if (status
> Http::scInvalidHeader
)
1256 if (tries
> MAX_FWD_STATS_IDX
)
1257 tries
= MAX_FWD_STATS_IDX
;
1259 ++ FwdReplyCodes
[tries
][status
];
1262 /**** PRIVATE NON-MEMBER FUNCTIONS ********************************************/
1266 * Formerly static, but now used by client_side_request.cc
1268 /// Checks for a TOS value to apply depending on the ACL
1270 aclMapTOS(acl_tos
* head
, ACLChecklist
* ch
)
1272 for (acl_tos
*l
= head
; l
; l
= l
->next
) {
1273 if (!l
->aclList
|| ch
->fastCheck(l
->aclList
).allowed())
1280 /// Checks for a netfilter mark value to apply depending on the ACL
1282 aclMapNfmark(acl_nfmark
* head
, ACLChecklist
* ch
)
1284 for (acl_nfmark
*l
= head
; l
; l
= l
->next
) {
1285 if (!l
->aclList
|| ch
->fastCheck(l
->aclList
).allowed())
1293 getOutgoingAddress(HttpRequest
* request
, Comm::ConnectionPointer conn
)
1295 // skip if an outgoing address is already set.
1296 if (!conn
->local
.isAnyAddr()) return;
1298 // ensure that at minimum the wildcard local matches remote protocol
1299 if (conn
->remote
.isIPv4())
1300 conn
->local
.setIPv4();
1302 // maybe use TPROXY client address
1303 if (request
&& request
->flags
.spoofClientIp
) {
1304 if (!conn
->getPeer() || !conn
->getPeer()->options
.no_tproxy
) {
1305 #if FOLLOW_X_FORWARDED_FOR && LINUX_NETFILTER
1306 if (Config
.onoff
.tproxy_uses_indirect_client
)
1307 conn
->local
= request
->indirect_client_addr
;
1310 conn
->local
= request
->client_addr
;
1311 // some flags need setting on the socket to use this address
1312 conn
->flags
|= COMM_DOBIND
;
1313 conn
->flags
|= COMM_TRANSPARENT
;
1316 // else no tproxy today ...
1319 if (!Config
.accessList
.outgoing_address
) {
1320 return; // anything will do.
1323 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1324 ch
.dst_peer_name
= conn
->getPeer() ? conn
->getPeer()->name
: NULL
;
1325 ch
.dst_addr
= conn
->remote
;
1327 // TODO use the connection details in ACL.
1328 // needs a bit of rework in ACLFilledChecklist to use Comm::Connection instead of ConnStateData
1330 for (Acl::Address
*l
= Config
.accessList
.outgoing_address
; l
; l
= l
->next
) {
1332 /* check if the outgoing address is usable to the destination */
1333 if (conn
->remote
.isIPv4() != l
->addr
.isIPv4()) continue;
1335 /* check ACLs for this outgoing address */
1336 if (!l
->aclList
|| ch
.fastCheck(l
->aclList
).allowed()) {
1337 conn
->local
= l
->addr
;
1344 GetTosToServer(HttpRequest
* request
)
1346 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1347 return aclMapTOS(Ip::Qos::TheConfig
.tosToServer
, &ch
);
1351 GetNfmarkToServer(HttpRequest
* request
)
1353 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1354 return aclMapNfmark(Ip::Qos::TheConfig
.nfmarkToServer
, &ch
);
1358 GetMarkingsToServer(HttpRequest
* request
, Comm::Connection
&conn
)
1360 // Get the server side TOS and Netfilter mark to be set on the connection.
1361 if (Ip::Qos::TheConfig
.isAclTosActive()) {
1362 conn
.tos
= GetTosToServer(request
);
1363 debugs(17, 3, "from " << conn
.local
<< " tos " << int(conn
.tos
));
1366 #if SO_MARK && USE_LIBCAP
1367 conn
.nfmark
= GetNfmarkToServer(request
);
1368 debugs(17, 3, "from " << conn
.local
<< " netfilter mark " << conn
.nfmark
);