2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 11 Hypertext Transfer Protocol (HTTP) */
12 * Anonymizing patch by lutz@as-node.jena.thur.de
13 * have a look into http-anon.c to get more information.
17 #include "acl/FilledChecklist.h"
18 #include "base/AsyncJobCalls.h"
19 #include "base/TextException.h"
21 #include "CachePeer.h"
22 #include "client_side.h"
23 #include "comm/Connection.h"
24 #include "comm/Read.h"
25 #include "comm/Write.h"
27 #include "error/Detail.h"
28 #include "errorpage.h"
33 #include "http/one/ResponseParser.h"
34 #include "http/one/TeChunkedParser.h"
35 #include "http/Stream.h"
36 #include "HttpControlMsg.h"
37 #include "HttpHdrCc.h"
38 #include "HttpHdrContRange.h"
39 #include "HttpHdrSc.h"
40 #include "HttpHdrScTarget.h"
41 #include "HttpHeaderTools.h"
42 #include "HttpReply.h"
43 #include "HttpRequest.h"
44 #include "HttpUpgradeProtocolAccess.h"
45 #include "log/access_log.h"
47 #include "MemObject.h"
48 #include "neighbors.h"
50 #include "peer_proxy_negotiate_auth.h"
51 #include "profiler/Profiler.h"
53 #include "RefreshPattern.h"
55 #include "SquidConfig.h"
56 #include "SquidTime.h"
57 #include "StatCounters.h"
64 #include "auth/UserRequest.h"
67 #include "DelayPools.h"
70 CBDATA_CLASS_INIT(HttpStateData
);
72 static const char *const crlf
= "\r\n";
74 static void httpMaybeRemovePublic(StoreEntry
*, Http::StatusCode
);
75 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry
*e
, const String strConnection
, const HttpRequest
* request
,
76 HttpHeader
* hdr_out
, const int we_do_ranges
, const Http::StateFlags
&);
78 HttpStateData::HttpStateData(FwdState
*theFwdState
) :
79 AsyncJob("HttpStateData"),
82 httpChunkDecoder(NULL
),
87 debugs(11,5,HERE
<< "HttpStateData " << this << " created");
88 ignoreCacheControl
= false;
89 surrogateNoStore
= false;
90 serverConnection
= fwd
->serverConnection();
92 if (fwd
->serverConnection() != NULL
)
93 _peer
= cbdataReference(fwd
->serverConnection()->getPeer()); /* might be NULL */
95 flags
.peering
= _peer
;
96 flags
.tunneling
= (_peer
&& request
->flags
.sslBumped
);
97 flags
.toOrigin
= (!_peer
|| _peer
->options
.originserver
|| request
->flags
.sslBumped
);
101 * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
102 * We might end up getting the object from somewhere else if,
103 * for example, the request to this neighbor fails.
105 if (!flags
.tunneling
&& _peer
->options
.proxy_only
)
106 entry
->releaseRequest(true);
109 entry
->setNoDelay(_peer
->options
.no_delay
);
114 * register the handler to free HTTP state data when the FD closes
116 typedef CommCbMemFunT
<HttpStateData
, CommCloseCbParams
> Dialer
;
117 closeHandler
= JobCallback(9, 5, Dialer
, this, HttpStateData::httpStateConnClosed
);
118 comm_add_close_handler(serverConnection
->fd
, closeHandler
);
121 HttpStateData::~HttpStateData()
124 * don't forget that ~Client() gets called automatically
127 if (httpChunkDecoder
)
128 delete httpChunkDecoder
;
130 cbdataReferenceDone(_peer
);
132 delete upgradeHeaderOut
;
134 debugs(11,5, HERE
<< "HttpStateData " << this << " destroyed; " << serverConnection
);
137 const Comm::ConnectionPointer
&
138 HttpStateData::dataConnection() const
140 return serverConnection
;
144 HttpStateData::httpStateConnClosed(const CommCloseCbParams
¶ms
)
146 debugs(11, 5, "httpStateFree: FD " << params
.fd
<< ", httpState=" << params
.data
);
147 doneWithFwd
= "httpStateConnClosed()"; // assume FwdState is monitoring too
148 mustStop("HttpStateData::httpStateConnClosed");
152 HttpStateData::httpTimeout(const CommTimeoutCbParams
&)
154 debugs(11, 4, serverConnection
<< ": '" << entry
->url() << "'");
156 if (entry
->store_status
== STORE_PENDING
) {
157 fwd
->fail(new ErrorState(ERR_READ_TIMEOUT
, Http::scGatewayTimeout
, fwd
->request
, fwd
->al
));
161 mustStop("HttpStateData::httpTimeout");
165 findPreviouslyCachedEntry(StoreEntry
*newEntry
) {
166 assert(newEntry
->mem_obj
);
167 return newEntry
->mem_obj
->request
?
168 storeGetPublicByRequest(newEntry
->mem_obj
->request
.getRaw()) :
169 storeGetPublic(newEntry
->mem_obj
->storeId(), newEntry
->mem_obj
->method
);
172 /// Remove an existing public store entry if the incoming response (to be
173 /// stored in a currently private entry) is going to invalidate it.
175 httpMaybeRemovePublic(StoreEntry
* e
, Http::StatusCode status
)
180 // If the incoming response already goes into a public entry, then there is
181 // nothing to remove. This protects ready-for-collapsing entries as well.
182 if (!EBIT_TEST(e
->flags
, KEY_PRIVATE
))
185 // If the new/incoming response cannot be stored, then it does not
186 // compete with the old stored response for the public key, and the
187 // old stored response should be left as is.
188 if (e
->mem_obj
->request
&& !e
->mem_obj
->request
->flags
.cachable
)
195 case Http::scNonAuthoritativeInformation
:
197 case Http::scMultipleChoices
:
199 case Http::scMovedPermanently
:
203 case Http::scSeeOther
:
207 case Http::scNotFound
:
212 case Http::scForbidden
:
214 case Http::scMethodNotAllowed
:
221 case Http::scUnauthorized
:
231 * Any 2xx response should eject previously cached entities...
234 if (status
>= 200 && status
< 300)
242 if (!remove
&& !forbidden
)
245 StoreEntry
*pe
= findPreviouslyCachedEntry(e
);
250 neighborsHtcpClear(e
, e
->mem_obj
->request
.getRaw(), e
->mem_obj
->method
, HTCP_CLR_INVALIDATION
);
256 * Also remove any cached HEAD response in case the object has
259 if (e
->mem_obj
->request
)
260 pe
= storeGetPublicByRequestMethod(e
->mem_obj
->request
.getRaw(), Http::METHOD_HEAD
);
262 pe
= storeGetPublic(e
->mem_obj
->storeId(), Http::METHOD_HEAD
);
267 neighborsHtcpClear(e
, e
->mem_obj
->request
.getRaw(), HttpRequestMethod(Http::METHOD_HEAD
), HTCP_CLR_INVALIDATION
);
274 HttpStateData::processSurrogateControl(HttpReply
*reply
)
276 if (request
->flags
.accelerated
&& reply
->surrogate_control
) {
277 HttpHdrScTarget
*sctusable
= reply
->surrogate_control
->getMergedTarget(Config
.Accel
.surrogate_id
);
280 if (sctusable
->hasNoStore() ||
281 (Config
.onoff
.surrogate_is_remote
282 && sctusable
->noStoreRemote())) {
283 surrogateNoStore
= true;
284 // Be conservative for now and make it non-shareable because
285 // there is no enough information here to make the decision.
286 entry
->makePrivate(false);
289 /* The HttpHeader logic cannot tell if the header it's parsing is a reply to an
290 * accelerated request or not...
291 * Still, this is an abstraction breach. - RC
293 if (sctusable
->hasMaxAge()) {
294 if (sctusable
->maxAge() < sctusable
->maxStale())
295 reply
->expires
= reply
->date
+ sctusable
->maxAge();
297 reply
->expires
= reply
->date
+ sctusable
->maxStale();
299 /* And update the timestamps */
300 entry
->timestampsSet();
303 /* We ignore cache-control directives as per the Surrogate specification */
304 ignoreCacheControl
= true;
311 HttpStateData::ReuseDecision::Answers
312 HttpStateData::reusableReply(HttpStateData::ReuseDecision
&decision
)
314 HttpReply
const *rep
= finalReply();
315 HttpHeader
const *hdr
= &rep
->header
;
317 #if USE_HTTP_VIOLATIONS
319 const RefreshPattern
*R
= NULL
;
321 /* This strange looking define first looks up the refresh pattern
322 * and then checks if the specified flag is set. The main purpose
323 * of this is to simplify the refresh pattern lookup and USE_HTTP_VIOLATIONS
326 #define REFRESH_OVERRIDE(flag) \
327 ((R = (R ? R : refreshLimits(entry->mem_obj->storeId()))) , \
328 (R && R->flags.flag))
330 #define REFRESH_OVERRIDE(flag) 0
333 if (EBIT_TEST(entry
->flags
, RELEASE_REQUEST
))
334 return decision
.make(ReuseDecision::doNotCacheButShare
, "the entry has been released");
336 // RFC 7234 section 4: a cache MUST use the most recent response
337 // (as determined by the Date header field)
338 // TODO: whether such responses could be shareable?
340 return decision
.make(ReuseDecision::reuseNot
, "the response has an older date header");
342 // Check for Surrogate/1.0 protocol conditions
343 // NP: reverse-proxy traffic our parent server has instructed us never to cache
344 if (surrogateNoStore
)
345 return decision
.make(ReuseDecision::reuseNot
, "Surrogate-Control:no-store");
347 // RFC 2616: HTTP/1.1 Cache-Control conditions
348 if (!ignoreCacheControl
) {
349 // XXX: check to see if the request headers alone were enough to prevent caching earlier
350 // (ie no-store request header) no need to check those all again here if so.
351 // for now we are not reliably doing that so we waste CPU re-checking request CC
353 // RFC 2616 section 14.9.2 - MUST NOT cache any response with request CC:no-store
354 if (request
&& request
->cache_control
&& request
->cache_control
->hasNoStore() &&
355 !REFRESH_OVERRIDE(ignore_no_store
))
356 return decision
.make(ReuseDecision::reuseNot
,
357 "client request Cache-Control:no-store");
359 // NP: request CC:no-cache only means cache READ is forbidden. STORE is permitted.
360 if (rep
->cache_control
&& rep
->cache_control
->hasNoCacheWithParameters()) {
361 /* TODO: we are allowed to cache when no-cache= has parameters.
362 * Provided we strip away any of the listed headers unless they are revalidated
363 * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
364 * That is a bit tricky for squid right now so we avoid caching entirely.
366 return decision
.make(ReuseDecision::reuseNot
,
367 "server reply Cache-Control:no-cache has parameters");
370 // NP: request CC:private is undefined. We ignore.
371 // NP: other request CC flags are limiters on HIT/MISS. We don't care about here.
373 // RFC 2616 section 14.9.2 - MUST NOT cache any response with CC:no-store
374 if (rep
->cache_control
&& rep
->cache_control
->hasNoStore() &&
375 !REFRESH_OVERRIDE(ignore_no_store
))
376 return decision
.make(ReuseDecision::reuseNot
,
377 "server reply Cache-Control:no-store");
379 // RFC 2616 section 14.9.1 - MUST NOT cache any response with CC:private in a shared cache like Squid.
380 // CC:private overrides CC:public when both are present in a response.
381 // TODO: add a shared/private cache configuration possibility.
382 if (rep
->cache_control
&&
383 rep
->cache_control
->hasPrivate() &&
384 !REFRESH_OVERRIDE(ignore_private
)) {
385 /* TODO: we are allowed to cache when private= has parameters.
386 * Provided we strip away any of the listed headers unless they are revalidated
387 * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
388 * That is a bit tricky for squid right now so we avoid caching entirely.
390 return decision
.make(ReuseDecision::reuseNot
,
391 "server reply Cache-Control:private");
395 // RFC 2068, sec 14.9.4 - MUST NOT cache any response with Authentication UNLESS certain CC controls are present
396 // allow HTTP violations to IGNORE those controls (ie re-block caching Auth)
397 if (request
&& (request
->flags
.auth
|| request
->flags
.authSent
)) {
398 if (!rep
->cache_control
)
399 return decision
.make(ReuseDecision::reuseNot
,
400 "authenticated and server reply missing Cache-Control");
402 if (ignoreCacheControl
)
403 return decision
.make(ReuseDecision::reuseNot
,
404 "authenticated and ignoring Cache-Control");
406 bool mayStore
= false;
407 // HTTPbis pt6 section 3.2: a response CC:public is present
408 if (rep
->cache_control
->hasPublic()) {
409 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:public");
412 // HTTPbis pt6 section 3.2: a response CC:must-revalidate is present
413 } else if (rep
->cache_control
->hasMustRevalidate()) {
414 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:must-revalidate");
417 #if USE_HTTP_VIOLATIONS
418 // NP: given the must-revalidate exception we should also be able to exempt no-cache.
419 // HTTPbis WG verdict on this is that it is omitted from the spec due to being 'unexpected' by
420 // some. The caching+revalidate is not exactly unsafe though with Squids interpretation of no-cache
421 // (without parameters) as equivalent to must-revalidate in the reply.
422 } else if (rep
->cache_control
->hasNoCacheWithoutParameters()) {
423 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:no-cache (equivalent to must-revalidate)");
427 // HTTPbis pt6 section 3.2: a response CC:s-maxage is present
428 } else if (rep
->cache_control
->hasSMaxAge()) {
429 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:s-maxage");
434 return decision
.make(ReuseDecision::reuseNot
, "authenticated transaction");
436 // NP: response CC:no-cache is equivalent to CC:must-revalidate,max-age=0. We MAY cache, and do so.
437 // NP: other request CC flags are limiters on HIT/MISS/REFRESH. We don't care about here.
440 /* HACK: The "multipart/x-mixed-replace" content type is used for
441 * continuous push replies. These are generally dynamic and
442 * probably should not be cachable
444 if ((v
= hdr
->getStr(Http::HdrType::CONTENT_TYPE
)))
445 if (!strncasecmp(v
, "multipart/x-mixed-replace", 25))
446 return decision
.make(ReuseDecision::reuseNot
, "Content-Type:multipart/x-mixed-replace");
448 // TODO: if possible, provide more specific message for each status code
449 static const char *shareableError
= "shareable error status code";
450 static const char *nonShareableError
= "non-shareable error status code";
451 ReuseDecision::Answers statusAnswer
= ReuseDecision::reuseNot
;
452 const char *statusReason
= nonShareableError
;
454 switch (rep
->sline
.status()) {
456 /* There are several situations when a non-cacheable response may be
457 * still shareable (e.g., among collapsed clients). We assume that these
458 * are 3xx and 5xx responses, indicating server problems and some of
459 * 4xx responses, common for all clients with a given cache key (e.g.,
460 * 404 Not Found or 414 URI Too Long). On the other hand, we should not
461 * share non-cacheable client-specific errors, such as 400 Bad Request
462 * or 406 Not Acceptable.
465 /* Responses that are cacheable */
469 case Http::scNonAuthoritativeInformation
:
471 case Http::scMultipleChoices
:
473 case Http::scMovedPermanently
:
474 case Http::scPermanentRedirect
:
478 * Don't cache objects that need to be refreshed on next request,
479 * unless we know how to refresh it.
482 if (refreshIsCachable(entry
) || REFRESH_OVERRIDE(store_stale
))
483 decision
.make(ReuseDecision::cachePositively
, "refresh check returned cacheable");
485 decision
.make(ReuseDecision::doNotCacheButShare
, "refresh check returned non-cacheable");
488 /* Responses that only are cacheable if the server says so */
491 case Http::scTemporaryRedirect
:
493 decision
.make(ReuseDecision::doNotCacheButShare
, "Date is missing/invalid");
494 else if (rep
->expires
> rep
->date
)
495 decision
.make(ReuseDecision::cachePositively
, "Expires > Date");
497 decision
.make(ReuseDecision::doNotCacheButShare
, "Expires <= Date");
500 /* These responses can be negatively cached. Most can also be shared. */
501 case Http::scNoContent
:
502 case Http::scUseProxy
:
503 case Http::scForbidden
:
504 case Http::scNotFound
:
505 case Http::scMethodNotAllowed
:
506 case Http::scUriTooLong
:
507 case Http::scInternalServerError
:
508 case Http::scNotImplemented
:
509 case Http::scBadGateway
:
510 case Http::scServiceUnavailable
:
511 case Http::scGatewayTimeout
:
512 case Http::scMisdirectedRequest
:
513 statusAnswer
= ReuseDecision::doNotCacheButShare
;
514 statusReason
= shareableError
;
515 // fall through to the actual decision making below
517 case Http::scBadRequest
: // no sharing; perhaps the server did not like something specific to this request
518 #if USE_HTTP_VIOLATIONS
519 if (Config
.negativeTtl
> 0)
520 decision
.make(ReuseDecision::cacheNegatively
, "Config.negativeTtl > 0");
523 decision
.make(statusAnswer
, statusReason
);
526 /* these responses can never be cached, some
527 of them can be shared though */
528 case Http::scSeeOther
:
529 case Http::scNotModified
:
530 case Http::scUnauthorized
:
531 case Http::scProxyAuthenticationRequired
:
532 case Http::scPaymentRequired
:
533 case Http::scInsufficientStorage
:
534 // TODO: use more specific reason for non-error status codes
535 decision
.make(ReuseDecision::doNotCacheButShare
, shareableError
);
538 case Http::scPartialContent
: /* Not yet supported. TODO: make shareable for suitable ranges */
539 case Http::scNotAcceptable
:
540 case Http::scRequestTimeout
: // TODO: is this shareable?
541 case Http::scConflict
: // TODO: is this shareable?
542 case Http::scLengthRequired
:
543 case Http::scPreconditionFailed
:
544 case Http::scPayloadTooLarge
:
545 case Http::scUnsupportedMediaType
:
546 case Http::scUnprocessableEntity
:
547 case Http::scLocked
: // TODO: is this shareable?
548 case Http::scFailedDependency
:
549 case Http::scRequestedRangeNotSatisfied
:
550 case Http::scExpectationFailed
:
551 case Http::scInvalidHeader
: /* Squid header parsing error */
552 case Http::scHeaderTooLarge
:
553 decision
.make(ReuseDecision::reuseNot
, nonShareableError
);
557 /* RFC 2616 section 6.1.1: an unrecognized response MUST NOT be cached. */
558 decision
.make(ReuseDecision::reuseNot
, "unknown status code");
562 return decision
.answer
;
565 /// assemble a variant key (vary-mark) from the given Vary header and HTTP request
567 assembleVaryKey(String
&vary
, SBuf
&vstr
, const HttpRequest
&request
)
569 static const SBuf
asterisk("*");
570 const char *pos
= nullptr;
571 const char *item
= nullptr;
574 while (strListGetItem(&vary
, ',', &item
, &ilen
, &pos
)) {
575 SBuf
name(item
, ilen
);
576 if (name
== asterisk
) {
582 vstr
.append(", ", 2);
584 String
hdr(request
.header
.getByName(name
));
585 const char *value
= hdr
.termedBuf();
587 value
= rfc1738_escape_part(value
);
588 vstr
.append("=\"", 2);
590 vstr
.append("\"", 1);
598 * For Vary, store the relevant request headers as
599 * virtual headers in the reply
600 * Returns an empty SBuf if the variance cannot be stored
603 httpMakeVaryMark(HttpRequest
* request
, HttpReply
const * reply
)
608 vary
= reply
->header
.getList(Http::HdrType::VARY
);
609 assembleVaryKey(vary
, vstr
, *request
);
611 #if X_ACCELERATOR_VARY
613 vary
= reply
->header
.getList(Http::HdrType::HDR_X_ACCELERATOR_VARY
);
614 assembleVaryKey(vary
, vstr
, *request
);
622 HttpStateData::keepaliveAccounting(HttpReply
*reply
)
625 if (flags
.peering
&& !flags
.tunneling
)
626 ++ _peer
->stats
.n_keepalives_sent
;
628 if (reply
->keep_alive
) {
629 if (flags
.peering
&& !flags
.tunneling
)
630 ++ _peer
->stats
.n_keepalives_recv
;
632 if (Config
.onoff
.detect_broken_server_pconns
633 && reply
->bodySize(request
->method
) == -1 && !flags
.chunked
) {
634 debugs(11, DBG_IMPORTANT
, "keepaliveAccounting: Impossible keep-alive header from '" << entry
->url() << "'" );
635 // debugs(11, 2, "GOT HTTP REPLY HDR:\n---------\n" << readBuf->content() << "\n----------" );
636 flags
.keepalive_broken
= true;
642 HttpStateData::checkDateSkew(HttpReply
*reply
)
644 if (reply
->date
> -1 && flags
.toOrigin
) {
645 int skew
= abs((int)(reply
->date
- squid_curtime
));
648 debugs(11, 3, "" << request
->url
.host() << "'s clock is skewed by " << skew
<< " seconds!");
653 * This creates the error page itself.. its likely
654 * that the forward ported reply header max size patch
655 * generates non http conformant error pages - in which
656 * case the errors where should be 'BAD_GATEWAY' etc
659 HttpStateData::processReplyHeader()
661 /** Creates a blank header. If this routine is made incremental, this will not do */
663 /* NP: all exit points to this function MUST call ctx_exit(ctx) */
664 Ctx ctx
= ctx_enter(entry
->mem_obj
->urlXXX());
666 debugs(11, 3, "processReplyHeader: key '" << entry
->getMD5Text() << "'");
668 assert(!flags
.headers_parsed
);
670 if (!inBuf
.length()) {
675 /* Attempt to parse the first line; this will define where the protocol, status, reason-phrase and header begin */
678 hp
= new Http1::ResponseParser
;
680 bool parsedOk
= hp
->parse(inBuf
);
681 // remember the actual received status-code before returning on errors,
682 // overwriting any previously stored value from earlier forwarding attempts
683 request
->hier
.peer_reply_status
= hp
->messageStatus(); // may still be scNone
685 // sync the buffers after parsing.
686 inBuf
= hp
->remaining();
688 if (hp
->needsMoreData()) {
689 if (eof
) { // no more data coming
690 /* Bug 2879: Replies may terminate with \r\n then EOF instead of \r\n\r\n.
691 * We also may receive truncated responses.
692 * Ensure here that we have at minimum two \r\n when EOF is seen.
694 inBuf
.append("\r\n\r\n", 4);
696 parsedOk
= hp
->parse(inBuf
);
697 // sync the buffers after parsing.
698 inBuf
= hp
->remaining();
700 debugs(33, 5, "Incomplete response, waiting for end of response headers");
707 // unrecoverable parsing error
708 // TODO: Use Raw! XXX: inBuf no longer has the [beginning of the] malformed header.
709 debugs(11, 3, "Non-HTTP-compliant header:\n---------\n" << inBuf
<< "\n----------");
710 flags
.headers_parsed
= true;
711 HttpReply
*newrep
= new HttpReply
;
712 newrep
->sline
.set(Http::ProtocolVersion(), hp
->parseStatusCode
);
713 setVirginReply(newrep
);
719 /* We know the whole response is in parser now */
720 debugs(11, 2, "HTTP Server " << serverConnection
);
721 debugs(11, 2, "HTTP Server RESPONSE:\n---------\n" <<
722 hp
->messageProtocol() << " " << hp
->messageStatus() << " " << hp
->reasonPhrase() << "\n" <<
726 // reset payload tracking to begin after message headers
727 payloadSeen
= inBuf
.length();
729 HttpReply
*newrep
= new HttpReply
;
730 // XXX: RFC 7230 indicates we MAY ignore the reason phrase,
731 // and use an empty string on unknown status.
732 // We do that now to avoid performance regression from using SBuf::c_str()
733 newrep
->sline
.set(hp
->messageProtocol(), hp
->messageStatus() /* , hp->reasonPhrase() */);
736 if (!newrep
->parseHeader(*hp
)) {
737 newrep
->sline
.set(hp
->messageProtocol(), Http::scInvalidHeader
);
738 debugs(11, 2, "error parsing response headers mime block");
741 // done with Parser, now process using the HttpReply
744 newrep
->sources
|= request
->url
.getScheme() == AnyP::PROTO_HTTPS
? Http::Message::srcHttps
: Http::Message::srcHttp
;
746 newrep
->removeStaleWarnings();
748 if (newrep
->sline
.version
.protocol
== AnyP::PROTO_HTTP
&& Http::Is1xx(newrep
->sline
.status())) {
754 flags
.chunked
= false;
755 if (newrep
->sline
.version
.protocol
== AnyP::PROTO_HTTP
&& newrep
->header
.chunked()) {
756 flags
.chunked
= true;
757 httpChunkDecoder
= new Http1::TeChunkedParser
;
760 if (!peerSupportsConnectionPinning())
761 request
->flags
.connectionAuthDisabled
= true;
763 HttpReply
*vrep
= setVirginReply(newrep
);
764 flags
.headers_parsed
= true;
766 keepaliveAccounting(vrep
);
770 processSurrogateControl (vrep
);
775 /// ignore or start forwarding the 1xx response (a.k.a., control message)
777 HttpStateData::handle1xx(HttpReply
*reply
)
780 fwd
->al
->reply
= reply
;
782 HttpReply::Pointer
msg(reply
); // will destroy reply if unused
784 // one 1xx at a time: we must not be called while waiting for previous 1xx
785 Must(!flags
.handling1xx
);
786 flags
.handling1xx
= true;
788 const auto statusCode
= reply
->sline
.status();
790 // drop1xx() needs to handle HTTP 101 (Switching Protocols) responses
791 // specially because they indicate that the server has stopped speaking HTTP
792 Must(!flags
.serverSwitchedProtocols
);
793 flags
.serverSwitchedProtocols
= (statusCode
== Http::scSwitchingProtocols
);
795 if (statusCode
== Http::scContinue
&& request
->forcedBodyContinuation
)
796 return drop1xx("we have sent it already");
798 if (!request
->canHandle1xx())
799 return drop1xx("the client does not support it");
801 #if USE_HTTP_VIOLATIONS
802 // check whether the 1xx response forwarding is allowed by squid.conf
803 if (Config
.accessList
.reply
) {
804 ACLFilledChecklist
ch(Config
.accessList
.reply
, originalRequest().getRaw());
807 ch
.syncAle(originalRequest().getRaw(), nullptr);
808 HTTPMSGLOCK(ch
.reply
);
809 if (!ch
.fastCheck().allowed()) // TODO: support slow lookups?
810 return drop1xx("http_reply_access blocked it");
812 #endif // USE_HTTP_VIOLATIONS
814 if (flags
.serverSwitchedProtocols
) {
815 if (const auto reason
= blockSwitchingProtocols(*reply
))
816 return drop1xx(reason
);
819 debugs(11, 2, HERE
<< "forwarding 1xx to client");
821 // the Sink will use this to call us back after writing 1xx to the client
822 typedef NullaryMemFunT
<HttpStateData
> CbDialer
;
823 const AsyncCall::Pointer cb
= JobCallback(11, 3, CbDialer
, this,
824 HttpStateData::proceedAfter1xx
);
825 CallJobHere1(11, 4, request
->clientConnectionManager
, ConnStateData
,
826 ConnStateData::sendControlMsg
, HttpControlMsg(msg
, cb
));
827 // If the call is not fired, then the Sink is gone, and HttpStateData
828 // will terminate due to an aborted store entry or another similar error.
829 // If we get stuck, it is not handle1xx fault if we could get stuck
830 // for similar reasons without a 1xx response.
833 /// if possible, safely ignores the received 1xx control message
834 /// otherwise, terminates the server connection
836 HttpStateData::drop1xx(const char *reason
)
838 if (flags
.serverSwitchedProtocols
) {
839 debugs(11, 2, "bad 101 because " << reason
);
840 const auto err
= new ErrorState(ERR_INVALID_RESP
, Http::scBadGateway
, request
.getRaw(), fwd
->al
);
843 mustStop("prohibited HTTP/101 response");
847 debugs(11, 2, "ignoring 1xx because " << reason
);
851 /// \retval nil if the HTTP/101 (Switching Protocols) reply should be forwarded
852 /// \retval reason why an attempt to switch protocols should be stopped
854 HttpStateData::blockSwitchingProtocols(const HttpReply
&reply
) const
856 if (!upgradeHeaderOut
)
857 return "Squid offered no Upgrade at all, but server switched to a tunnel";
859 // See RFC 7230 section 6.7 for the corresponding MUSTs
861 if (!reply
.header
.has(Http::HdrType::UPGRADE
))
862 return "server did not send an Upgrade header field";
864 if (!reply
.header
.hasListMember(Http::HdrType::CONNECTION
, "upgrade", ','))
865 return "server did not send 'Connection: upgrade'";
867 const auto acceptedProtos
= reply
.header
.getList(Http::HdrType::UPGRADE
);
868 const char *pos
= nullptr;
869 const char *accepted
= nullptr;
871 while (strListGetItem(&acceptedProtos
, ',', &accepted
, &acceptedLen
, &pos
)) {
872 debugs(11, 5, "server accepted at least" << Raw(nullptr, accepted
, acceptedLen
));
873 return nullptr; // OK: let the client validate server's selection
876 return "server sent an essentially empty Upgrade header field";
879 /// restores state and resumes processing after 1xx is ignored or forwarded
881 HttpStateData::proceedAfter1xx()
883 Must(flags
.handling1xx
);
885 if (flags
.serverSwitchedProtocols
) {
886 // pass server connection ownership to request->clientConnectionManager
887 ConnStateData::ServerConnectionContext
scc(serverConnection
, request
, inBuf
);
888 typedef UnaryMemFunT
<ConnStateData
, ConnStateData::ServerConnectionContext
> MyDialer
;
889 AsyncCall::Pointer call
= asyncCall(11, 3, "ConnStateData::noteTakeServerConnectionControl",
890 MyDialer(request
->clientConnectionManager
,
891 &ConnStateData::noteTakeServerConnectionControl
, scc
));
892 ScheduleCallHere(call
);
893 fwd
->unregister(serverConnection
);
894 comm_remove_close_handler(serverConnection
->fd
, closeHandler
);
895 closeHandler
= nullptr;
896 serverConnection
= nullptr;
897 doneWithFwd
= "switched protocols";
898 mustStop(doneWithFwd
);
902 debugs(11, 2, "continuing with " << payloadSeen
<< " bytes in buffer after 1xx");
903 CallJobHere(11, 3, this, HttpStateData
, HttpStateData::processReply
);
907 * returns true if the peer can support connection pinning
910 HttpStateData::peerSupportsConnectionPinning() const
915 // we are talking "through" rather than "to" our _peer
919 /*If this peer does not support connection pinning (authenticated
920 connections) return false
922 if (!_peer
->connection_auth
)
925 const auto &rep
= entry
->mem().freshestReply();
927 /*The peer supports connection pinning and the http reply status
928 is not unauthorized, so the related connection can be pinned
930 if (rep
.sline
.status() != Http::scUnauthorized
)
933 /*The server respond with Http::scUnauthorized and the peer configured
934 with "connection-auth=on" we know that the peer supports pinned
937 if (_peer
->connection_auth
== 1)
940 /*At this point peer has configured with "connection-auth=auto"
941 parameter so we need some extra checks to decide if we are going
942 to allow pinned connections or not
945 /*if the peer configured with originserver just allow connection
946 pinning (squid 2.6 behaviour)
948 if (_peer
->options
.originserver
)
951 /*if the connections it is already pinned it is OK*/
952 if (request
->flags
.pinned
)
955 /*Allow pinned connections only if the Proxy-support header exists in
956 reply and has in its list the "Session-Based-Authentication"
957 which means that the peer supports connection pinning.
959 if (rep
.header
.hasListMember(Http::HdrType::PROXY_SUPPORT
, "Session-Based-Authentication", ','))
965 // Called when we parsed (and possibly adapted) the headers but
966 // had not starting storing (a.k.a., sending) the body yet.
968 HttpStateData::haveParsedReplyHeaders()
970 Client::haveParsedReplyHeaders();
972 Ctx ctx
= ctx_enter(entry
->mem_obj
->urlXXX());
973 HttpReply
*rep
= finalReply();
974 const Http::StatusCode statusCode
= rep
->sline
.status();
976 entry
->timestampsSet();
978 /* Check if object is cacheable or not based on reply code */
979 debugs(11, 3, "HTTP CODE: " << statusCode
);
981 if (StoreEntry
*oldEntry
= findPreviouslyCachedEntry(entry
)) {
982 oldEntry
->lock("HttpStateData::haveParsedReplyHeaders");
983 sawDateGoBack
= rep
->olderThan(oldEntry
->hasFreshestReply());
984 oldEntry
->unlock("HttpStateData::haveParsedReplyHeaders");
987 if (neighbors_do_private_keys
&& !sawDateGoBack
)
988 httpMaybeRemovePublic(entry
, rep
->sline
.status());
990 bool varyFailure
= false;
991 if (rep
->header
.has(Http::HdrType::VARY
)
992 #if X_ACCELERATOR_VARY
993 || rep
->header
.has(Http::HdrType::HDR_X_ACCELERATOR_VARY
)
996 const SBuf
vary(httpMakeVaryMark(request
.getRaw(), rep
));
998 if (vary
.isEmpty()) {
999 // TODO: check whether such responses are shareable.
1000 // Do not share for now.
1001 entry
->makePrivate(false);
1002 if (fwd
->reforwardableStatus(rep
->sline
.status()))
1003 EBIT_SET(entry
->flags
, ENTRY_FWD_HDR_WAIT
);
1006 entry
->mem_obj
->vary_headers
= vary
;
1008 // RFC 7231 section 7.1.4
1009 // Vary:* can be cached, but has mandatory revalidation
1010 static const SBuf
asterisk("*");
1011 if (vary
== asterisk
)
1012 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_ALWAYS
);
1018 * If its not a reply that we will re-forward, then
1019 * allow the client to get it.
1021 if (fwd
->reforwardableStatus(rep
->sline
.status()))
1022 EBIT_SET(entry
->flags
, ENTRY_FWD_HDR_WAIT
);
1024 ReuseDecision
decision(entry
, statusCode
);
1026 switch (reusableReply(decision
)) {
1028 case ReuseDecision::reuseNot
:
1029 entry
->makePrivate(false);
1032 case ReuseDecision::cachePositively
:
1033 if (!entry
->makePublic()) {
1034 decision
.make(ReuseDecision::doNotCacheButShare
, "public key creation error");
1035 entry
->makePrivate(true);
1039 case ReuseDecision::cacheNegatively
:
1040 if (!entry
->cacheNegatively()) {
1041 decision
.make(ReuseDecision::doNotCacheButShare
, "public key creation error");
1042 entry
->makePrivate(true);
1046 case ReuseDecision::doNotCacheButShare
:
1047 entry
->makePrivate(true);
1054 debugs(11, 3, "decided: " << decision
);
1057 if (!ignoreCacheControl
) {
1058 if (rep
->cache_control
) {
1059 // We are required to revalidate on many conditions.
1060 // For security reasons we do so even if storage was caused by refresh_pattern ignore-* option
1062 // CC:must-revalidate or CC:proxy-revalidate
1063 const bool ccMustRevalidate
= (rep
->cache_control
->hasProxyRevalidate() || rep
->cache_control
->hasMustRevalidate());
1065 // CC:no-cache (only if there are no parameters)
1066 const bool ccNoCacheNoParams
= rep
->cache_control
->hasNoCacheWithoutParameters();
1069 const bool ccSMaxAge
= rep
->cache_control
->hasSMaxAge();
1071 // CC:private (yes, these can sometimes be stored)
1072 const bool ccPrivate
= rep
->cache_control
->hasPrivate();
1074 if (ccNoCacheNoParams
|| ccPrivate
)
1075 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_ALWAYS
);
1076 else if (ccMustRevalidate
|| ccSMaxAge
)
1077 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_STALE
);
1079 #if USE_HTTP_VIOLATIONS // response header Pragma::no-cache is undefined in HTTP
1081 // Expensive calculation. So only do it IF the CC: header is not present.
1083 /* HACK: Pragma: no-cache in _replies_ is not documented in HTTP,
1084 * but servers like "Active Imaging Webcast/2.0" sure do use it */
1085 if (rep
->header
.has(Http::HdrType::PRAGMA
) &&
1086 rep
->header
.hasListMember(Http::HdrType::PRAGMA
,"no-cache",','))
1087 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_ALWAYS
);
1093 headersLog(1, 0, request
->method
, rep
);
1100 HttpStateData::ConnectionStatus
1101 HttpStateData::statusIfComplete() const
1103 const HttpReply
*rep
= virginReply();
1105 * If the reply wants to close the connection, it takes precedence */
1107 static SBuf
close("close", 5);
1108 if (httpHeaderHasConnDir(&rep
->header
, close
))
1109 return COMPLETE_NONPERSISTENT_MSG
;
1112 * If we sent a Connection:close request header, then this
1113 * can not be a persistent connection.
1115 if (!flags
.keepalive
)
1116 return COMPLETE_NONPERSISTENT_MSG
;
1119 * If we banned reuse, then this cannot be a persistent connection.
1121 if (flags
.forceClose
)
1122 return COMPLETE_NONPERSISTENT_MSG
;
1125 * If we haven't sent the whole request then this can not be a persistent
1128 if (!flags
.request_sent
) {
1129 debugs(11, 2, "Request not yet fully sent " << request
->method
<< ' ' << entry
->url());
1130 return COMPLETE_NONPERSISTENT_MSG
;
1134 * What does the reply have to say about keep-alive?
1138 * If the origin server (HTTP/1.0) does not send a keep-alive
1139 * header, but keeps the connection open anyway, what happens?
1140 * We'll return here and http.c waits for an EOF before changing
1141 * store_status to STORE_OK. Combine this with ENTRY_FWD_HDR_WAIT
1142 * and an error status code, and we might have to wait until
1143 * the server times out the socket.
1145 if (!rep
->keep_alive
)
1146 return COMPLETE_NONPERSISTENT_MSG
;
1148 return COMPLETE_PERSISTENT_MSG
;
1151 HttpStateData::ConnectionStatus
1152 HttpStateData::persistentConnStatus() const
1154 debugs(11, 3, HERE
<< serverConnection
<< " eof=" << eof
);
1155 if (eof
) // already reached EOF
1156 return COMPLETE_NONPERSISTENT_MSG
;
1158 /* If server fd is closing (but we have not been notified yet), stop Comm
1159 I/O to avoid assertions. TODO: Change Comm API to handle callers that
1160 want more I/O after async closing (usually initiated by others). */
1161 // XXX: add canReceive or s/canSend/canTalkToServer/
1162 if (!Comm::IsConnOpen(serverConnection
))
1163 return COMPLETE_NONPERSISTENT_MSG
;
1166 * In chunked response we do not know the content length but we are absolutely
1167 * sure about the end of response, so we are calling the statusIfComplete to
1168 * decide if we can be persistent
1170 if (lastChunk
&& flags
.chunked
)
1171 return statusIfComplete();
1173 const HttpReply
*vrep
= virginReply();
1174 debugs(11, 5, "persistentConnStatus: content_length=" << vrep
->content_length
);
1176 const int64_t clen
= vrep
->bodySize(request
->method
);
1178 debugs(11, 5, "persistentConnStatus: clen=" << clen
);
1180 /* If the body size is unknown we must wait for EOF */
1182 return INCOMPLETE_MSG
;
1185 * If the body size is known, we must wait until we've gotten all of it. */
1187 debugs(11,5, "payloadSeen=" << payloadSeen
<< " content_length=" << vrep
->content_length
);
1189 if (payloadSeen
< vrep
->content_length
)
1190 return INCOMPLETE_MSG
;
1192 if (payloadTruncated
> 0) // already read more than needed
1193 return COMPLETE_NONPERSISTENT_MSG
; // disable pconns
1197 * If there is no message body or we got it all, we can be persistent */
1198 return statusIfComplete();
1202 readDelayed(void *context
, CommRead
const &)
1204 HttpStateData
*state
= static_cast<HttpStateData
*>(context
);
1205 state
->flags
.do_next_read
= true;
1206 state
->maybeReadVirginBody();
1210 HttpStateData::readReply(const CommIoCbParams
&io
)
1212 Must(!flags
.do_next_read
); // XXX: should have been set false by mayReadVirginBody()
1213 flags
.do_next_read
= false;
1215 debugs(11, 5, io
.conn
);
1217 // Bail out early on Comm::ERR_CLOSING - close handlers will tidy up for us
1218 if (io
.flag
== Comm::ERR_CLOSING
) {
1219 debugs(11, 3, "http socket closing");
1223 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
1224 abortTransaction("store entry aborted while reading reply");
1228 Must(Comm::IsConnOpen(serverConnection
));
1229 Must(io
.conn
->fd
== serverConnection
->fd
);
1232 * Don't reset the timeout value here. The value should be
1233 * counting Config.Timeout.request and applies to the request
1234 * as a whole, not individual read() calls.
1235 * Plus, it breaks our lame *HalfClosed() detection
1238 Must(maybeMakeSpaceAvailable(true));
1239 CommIoCbParams
rd(this); // will be expanded with ReadNow results
1241 rd
.size
= entry
->bytesWanted(Range
<size_t>(0, inBuf
.spaceSize()));
1244 assert(entry
->mem_obj
);
1245 AsyncCall::Pointer nilCall
;
1246 entry
->mem_obj
->delayRead(DeferredRead(readDelayed
, this, CommRead(io
.conn
, NULL
, 0, nilCall
)));
1250 switch (Comm::ReadNow(rd
, inBuf
)) {
1251 case Comm::INPROGRESS
:
1252 if (inBuf
.isEmpty())
1253 debugs(33, 2, io
.conn
<< ": no data to process, " << xstrerr(rd
.xerrno
));
1254 flags
.do_next_read
= true;
1255 maybeReadVirginBody();
1260 payloadSeen
+= rd
.size
;
1262 DelayId delayId
= entry
->mem_obj
->mostBytesAllowed();
1263 delayId
.bytesIn(rd
.size
);
1266 statCounter
.server
.all
.kbytes_in
+= rd
.size
;
1267 statCounter
.server
.http
.kbytes_in
+= rd
.size
;
1268 ++ IOStats
.Http
.reads
;
1271 for (int clen
= rd
.size
- 1; clen
; ++bin
)
1274 ++ IOStats
.Http
.read_hist
[bin
];
1276 request
->hier
.notePeerRead();
1279 /* Continue to process previously read data */
1282 case Comm::ENDFILE
: // close detected by 0-byte read
1284 flags
.do_next_read
= false;
1286 /* Continue to process previously read data */
1289 // case Comm::COMM_ERROR:
1290 default: // no other flags should ever occur
1291 debugs(11, 2, io
.conn
<< ": read failure: " << xstrerr(rd
.xerrno
));
1292 const auto err
= new ErrorState(ERR_READ_ERROR
, Http::scBadGateway
, fwd
->request
, fwd
->al
);
1293 err
->xerrno
= rd
.xerrno
;
1295 flags
.do_next_read
= false;
1297 mustStop("HttpStateData::readReply");
1301 /* Process next response from buffer */
1305 /// processes the already read and buffered response data, possibly after
1306 /// waiting for asynchronous 1xx control message processing
1308 HttpStateData::processReply()
1311 if (flags
.handling1xx
) { // we came back after handling a 1xx response
1312 debugs(11, 5, HERE
<< "done with 1xx handling");
1313 flags
.handling1xx
= false;
1314 Must(!flags
.headers_parsed
);
1317 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
1318 abortTransaction("store entry aborted while we were waiting for processReply()");
1322 if (!flags
.headers_parsed
) { // have not parsed headers yet?
1323 PROF_start(HttpStateData_processReplyHeader
);
1324 processReplyHeader();
1325 PROF_stop(HttpStateData_processReplyHeader
);
1327 if (!continueAfterParsingHeader()) // parsing error or need more data
1328 return; // TODO: send errors to ICAP
1330 adaptOrFinalizeReply(); // may write to, abort, or "close" the entry
1333 // kick more reads if needed and/or process the response body, if any
1334 PROF_start(HttpStateData_processReplyBody
);
1335 processReplyBody(); // may call serverComplete()
1336 PROF_stop(HttpStateData_processReplyBody
);
1340 \retval true if we can continue with processing the body or doing ICAP.
1343 HttpStateData::continueAfterParsingHeader()
1345 if (flags
.handling1xx
) {
1346 debugs(11, 5, HERE
<< "wait for 1xx handling");
1347 Must(!flags
.headers_parsed
);
1351 if (!flags
.headers_parsed
&& !eof
) {
1352 debugs(11, 9, "needs more at " << inBuf
.length());
1353 flags
.do_next_read
= true;
1354 /** \retval false If we have not finished parsing the headers and may get more data.
1355 * Schedules more reads to retrieve the missing data.
1357 maybeReadVirginBody(); // schedules all kinds of reads; TODO: rename
1361 /** If we are done with parsing, check for errors */
1363 err_type error
= ERR_NONE
;
1365 if (flags
.headers_parsed
) { // parsed headers, possibly with errors
1366 // check for header parsing errors
1367 if (HttpReply
*vrep
= virginReply()) {
1368 const Http::StatusCode s
= vrep
->sline
.status();
1369 const AnyP::ProtocolVersion
&v
= vrep
->sline
.version
;
1370 if (s
== Http::scInvalidHeader
&& v
!= Http::ProtocolVersion(0,9)) {
1371 debugs(11, DBG_IMPORTANT
, "WARNING: HTTP: Invalid Response: Bad header encountered from " << entry
->url() << " AKA " << request
->url
);
1372 error
= ERR_INVALID_RESP
;
1373 } else if (s
== Http::scHeaderTooLarge
) {
1374 fwd
->dontRetry(true);
1375 error
= ERR_TOO_BIG
;
1376 } else if (vrep
->header
.conflictingContentLength()) {
1377 fwd
->dontRetry(true);
1378 error
= ERR_INVALID_RESP
;
1379 } else if (vrep
->header
.unsupportedTe()) {
1380 fwd
->dontRetry(true);
1381 error
= ERR_INVALID_RESP
;
1383 return true; // done parsing, got reply, and no error
1386 // parsed headers but got no reply
1387 debugs(11, DBG_IMPORTANT
, "WARNING: HTTP: Invalid Response: No reply at all for " << entry
->url() << " AKA " << request
->url
);
1388 error
= ERR_INVALID_RESP
;
1392 if (inBuf
.length()) {
1393 error
= ERR_INVALID_RESP
;
1394 debugs(11, DBG_IMPORTANT
, "WARNING: HTTP: Invalid Response: Headers did not parse at all for " << entry
->url() << " AKA " << request
->url
);
1396 error
= ERR_ZERO_SIZE_OBJECT
;
1397 debugs(11, (request
->flags
.accelerated
?DBG_IMPORTANT
:2), "WARNING: HTTP: Invalid Response: No object data received for " << entry
->url() << " AKA " << request
->url
);
1401 assert(error
!= ERR_NONE
);
1403 fwd
->fail(new ErrorState(error
, Http::scBadGateway
, fwd
->request
, fwd
->al
));
1404 flags
.do_next_read
= false;
1406 mustStop("HttpStateData::continueAfterParsingHeader");
1407 return false; // quit on error
1410 /** truncate what we read if we read too much so that writeReplyBody()
1411 writes no more than what we should have read */
1413 HttpStateData::truncateVirginBody()
1415 assert(flags
.headers_parsed
);
1417 HttpReply
*vrep
= virginReply();
1419 if (!vrep
->expectingBody(request
->method
, clen
) || clen
< 0)
1420 return; // no body or a body of unknown size, including chunked
1422 if (payloadSeen
- payloadTruncated
<= clen
)
1423 return; // we did not read too much or already took care of the extras
1425 if (const int64_t extras
= payloadSeen
- payloadTruncated
- clen
) {
1426 // server sent more that the advertised content length
1427 debugs(11, 5, "payloadSeen=" << payloadSeen
<<
1428 " clen=" << clen
<< '/' << vrep
->content_length
<<
1429 " truncated=" << payloadTruncated
<< '+' << extras
);
1431 inBuf
.chop(0, inBuf
.length() - extras
);
1432 payloadTruncated
+= extras
;
1437 * Call this when there is data from the origin server
1438 * which should be sent to either StoreEntry, or to ICAP...
1441 HttpStateData::writeReplyBody()
1443 truncateVirginBody(); // if needed
1444 const char *data
= inBuf
.rawContent();
1445 int len
= inBuf
.length();
1446 addVirginReplyBody(data
, len
);
1451 HttpStateData::decodeAndWriteReplyBody()
1453 assert(flags
.chunked
);
1454 assert(httpChunkDecoder
);
1458 httpChunkDecoder
->setPayloadBuffer(&decodedData
);
1459 const bool doneParsing
= httpChunkDecoder
->parse(inBuf
);
1460 inBuf
= httpChunkDecoder
->remaining(); // sync buffers after parse
1461 addVirginReplyBody(decodedData
.content(), decodedData
.contentSize());
1464 flags
.do_next_read
= false;
1469 debugs (11, 2, "de-chunking failure: " << CurrentException
);
1475 * processReplyBody has two purposes:
1476 * 1 - take the reply body data, if any, and put it into either
1477 * the StoreEntry, or give it over to ICAP.
1478 * 2 - see if we made it to the end of the response (persistent
1479 * connections and such)
1482 HttpStateData::processReplyBody()
1484 if (!flags
.headers_parsed
) {
1485 flags
.do_next_read
= true;
1486 maybeReadVirginBody();
1491 debugs(11,5, HERE
<< "adaptationAccessCheckPending=" << adaptationAccessCheckPending
);
1492 if (adaptationAccessCheckPending
)
1498 * At this point the reply headers have been parsed and consumed.
1499 * That means header content has been removed from readBuf and
1500 * it contains only body data.
1502 if (entry
->isAccepting()) {
1503 if (flags
.chunked
) {
1504 if (!decodeAndWriteReplyBody()) {
1505 flags
.do_next_read
= false;
1513 // storing/sending methods like earlier adaptOrFinalizeReply() or
1514 // above writeReplyBody() may release/abort the store entry.
1515 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
1516 // TODO: In some cases (e.g., 304), we should keep persistent conn open.
1517 // Detect end-of-reply (and, hence, pool our idle pconn) earlier (ASAP).
1518 abortTransaction("store entry aborted while storing reply");
1521 switch (persistentConnStatus()) {
1522 case INCOMPLETE_MSG
: {
1523 debugs(11, 5, "processReplyBody: INCOMPLETE_MSG from " << serverConnection
);
1524 /* Wait for more data or EOF condition */
1525 AsyncCall::Pointer nil
;
1526 if (flags
.keepalive_broken
) {
1527 commSetConnTimeout(serverConnection
, 10, nil
);
1529 commSetConnTimeout(serverConnection
, Config
.Timeout
.read
, nil
);
1532 flags
.do_next_read
= true;
1536 case COMPLETE_PERSISTENT_MSG
: {
1537 debugs(11, 5, "processReplyBody: COMPLETE_PERSISTENT_MSG from " << serverConnection
);
1539 // TODO: Remove serverConnectionSaved but preserve exception safety.
1541 commUnsetConnTimeout(serverConnection
);
1542 flags
.do_next_read
= false;
1544 comm_remove_close_handler(serverConnection
->fd
, closeHandler
);
1545 closeHandler
= NULL
;
1547 Ip::Address client_addr
; // XXX: Remove as unused. Why was it added?
1548 if (request
->flags
.spoofClientIp
)
1549 client_addr
= request
->client_addr
;
1551 auto serverConnectionSaved
= serverConnection
;
1552 fwd
->unregister(serverConnection
);
1553 serverConnection
= nullptr;
1555 bool ispinned
= false; // TODO: Rename to isOrShouldBePinned
1556 if (request
->flags
.pinned
) {
1558 } else if (request
->flags
.connectionAuth
&& request
->flags
.authSent
) {
1563 if (request
->clientConnectionManager
.valid()) {
1564 CallJobHere1(11, 4, request
->clientConnectionManager
,
1566 notePinnedConnectionBecameIdle
,
1567 ConnStateData::PinnedIdleContext(serverConnectionSaved
, request
));
1569 // must not pool/share ispinned connections, even orphaned ones
1570 serverConnectionSaved
->close();
1573 fwdPconnPool
->push(serverConnectionSaved
, request
->url
.host());
1580 case COMPLETE_NONPERSISTENT_MSG
:
1581 debugs(11, 5, "processReplyBody: COMPLETE_NONPERSISTENT_MSG from " << serverConnection
);
1582 if (flags
.chunked
&& !lastChunk
)
1583 entry
->lengthWentBad("missing last-chunk");
1589 maybeReadVirginBody();
1593 HttpStateData::mayReadVirginReplyBody() const
1595 // TODO: Be more precise here. For example, if/when reading trailer, we may
1596 // not be doneWithServer() yet, but we should return false. Similarly, we
1597 // could still be writing the request body after receiving the whole reply.
1598 return !doneWithServer();
1602 HttpStateData::maybeReadVirginBody()
1605 if (!Comm::IsConnOpen(serverConnection
) || fd_table
[serverConnection
->fd
].closing())
1608 if (!maybeMakeSpaceAvailable(false))
1611 // XXX: get rid of the do_next_read flag
1612 // check for the proper reasons preventing read(2)
1613 if (!flags
.do_next_read
)
1616 flags
.do_next_read
= false;
1618 // must not already be waiting for read(2) ...
1619 assert(!Comm::MonitorsRead(serverConnection
->fd
));
1621 // wait for read(2) to be possible.
1622 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
1623 AsyncCall::Pointer call
= JobCallback(11, 5, Dialer
, this, HttpStateData::readReply
);
1624 Comm::Read(serverConnection
, call
);
1628 HttpStateData::maybeMakeSpaceAvailable(bool doGrow
)
1630 // how much we are allowed to buffer
1631 const int limitBuffer
= (flags
.headers_parsed
? Config
.readAheadGap
: Config
.maxReplyHeaderSize
);
1633 if (limitBuffer
< 0 || inBuf
.length() >= (SBuf::size_type
)limitBuffer
) {
1634 // when buffer is at or over limit already
1635 debugs(11, 7, "will not read up to " << limitBuffer
<< ". buffer has (" << inBuf
.length() << "/" << inBuf
.spaceSize() << ") from " << serverConnection
);
1636 debugs(11, DBG_DATA
, "buffer has {" << inBuf
<< "}");
1637 // Process next response from buffer
1642 // how much we want to read
1643 const size_t read_size
= calcBufferSpaceToReserve(inBuf
.spaceSize(), (limitBuffer
- inBuf
.length()));
1646 debugs(11, 7, "will not read up to " << read_size
<< " into buffer (" << inBuf
.length() << "/" << inBuf
.spaceSize() << ") from " << serverConnection
);
1650 // just report whether we could grow or not, do not actually do it
1652 return (read_size
>= 2);
1654 // we may need to grow the buffer
1655 inBuf
.reserveSpace(read_size
);
1656 debugs(11, 8, (!flags
.do_next_read
? "will not" : "may") <<
1657 " read up to " << read_size
<< " bytes info buf(" << inBuf
.length() << "/" << inBuf
.spaceSize() <<
1658 ") from " << serverConnection
);
1660 return (inBuf
.spaceSize() >= 2); // only read if there is 1+ bytes of space available
1663 /// called after writing the very last request byte (body, last-chunk, etc)
1665 HttpStateData::wroteLast(const CommIoCbParams
&io
)
1667 debugs(11, 5, HERE
<< serverConnection
<< ": size " << io
.size
<< ": errflag " << io
.flag
<< ".");
1668 #if URL_CHECKSUM_DEBUG
1670 entry
->mem_obj
->checkUrlChecksum();
1673 // XXX: Keep in sync with Client::sentRequestBody().
1674 // TODO: Extract common parts.
1677 fd_bytes(io
.fd
, io
.size
, FD_WRITE
);
1678 statCounter
.server
.all
.kbytes_out
+= io
.size
;
1679 statCounter
.server
.http
.kbytes_out
+= io
.size
;
1682 if (io
.flag
== Comm::ERR_CLOSING
)
1685 // both successful and failed writes affect response times
1686 request
->hier
.notePeerWrite();
1689 const auto err
= new ErrorState(ERR_WRITE_ERROR
, Http::scBadGateway
, fwd
->request
, fwd
->al
);
1690 err
->xerrno
= io
.xerrno
;
1693 mustStop("HttpStateData::wroteLast");
1700 /// successfully wrote the entire request (including body, last-chunk, etc.)
1702 HttpStateData::sendComplete()
1705 * Set the read timeout here because it hasn't been set yet.
1706 * We only set the read timeout after the request has been
1707 * fully written to the peer. If we start the timeout
1708 * after connection establishment, then we are likely to hit
1709 * the timeout for POST/PUT requests that have very large
1712 typedef CommCbMemFunT
<HttpStateData
, CommTimeoutCbParams
> TimeoutDialer
;
1713 AsyncCall::Pointer timeoutCall
= JobCallback(11, 5,
1714 TimeoutDialer
, this, HttpStateData::httpTimeout
);
1716 commSetConnTimeout(serverConnection
, Config
.Timeout
.read
, timeoutCall
);
1717 flags
.request_sent
= true;
1721 HttpStateData::closeServer()
1723 debugs(11,5, HERE
<< "closing HTTP server " << serverConnection
<< " this " << this);
1725 if (Comm::IsConnOpen(serverConnection
)) {
1726 fwd
->unregister(serverConnection
);
1727 comm_remove_close_handler(serverConnection
->fd
, closeHandler
);
1728 closeHandler
= NULL
;
1729 serverConnection
->close();
1734 HttpStateData::doneWithServer() const
1736 return !Comm::IsConnOpen(serverConnection
);
1740 * Fixup authentication request headers for special cases
1743 httpFixupAuthentication(HttpRequest
* request
, const HttpHeader
* hdr_in
, HttpHeader
* hdr_out
, const Http::StateFlags
&flags
)
1745 /* Nothing to do unless we are forwarding to a peer */
1749 // This request is going "through" rather than "to" our _peer.
1750 if (flags
.tunneling
)
1753 /* Needs to be explicitly enabled */
1754 if (!request
->peer_login
)
1757 const auto header
= flags
.toOrigin
? Http::HdrType::AUTHORIZATION
: Http::HdrType::PROXY_AUTHORIZATION
;
1758 /* Maybe already dealt with? */
1759 if (hdr_out
->has(header
))
1762 /* Nothing to do here for PASSTHRU */
1763 if (strcmp(request
->peer_login
, "PASSTHRU") == 0)
1766 // Dangerous and undocumented PROXYPASS is a single-signon to servers with
1767 // the proxy password. Only Basic Authentication can work this way. This
1768 // statement forwards a "basic" Proxy-Authorization value from our client
1769 // to an originserver peer. Other PROXYPASS cases are handled lower.
1770 if (flags
.toOrigin
&&
1771 strcmp(request
->peer_login
, "PROXYPASS") == 0 &&
1772 hdr_in
->has(Http::HdrType::PROXY_AUTHORIZATION
)) {
1774 const char *auth
= hdr_in
->getStr(Http::HdrType::PROXY_AUTHORIZATION
);
1776 if (auth
&& strncasecmp(auth
, "basic ", 6) == 0) {
1777 hdr_out
->putStr(header
, auth
);
1782 char loginbuf
[base64_encode_len(MAX_LOGIN_SZ
)];
1784 struct base64_encode_ctx ctx
;
1785 base64_encode_init(&ctx
);
1787 /* Special mode to pass the username to the upstream cache */
1788 if (*request
->peer_login
== '*') {
1789 const char *username
= "-";
1791 if (request
->extacl_user
.size())
1792 username
= request
->extacl_user
.termedBuf();
1794 else if (request
->auth_user_request
!= NULL
)
1795 username
= request
->auth_user_request
->username();
1798 blen
= base64_encode_update(&ctx
, loginbuf
, strlen(username
), reinterpret_cast<const uint8_t*>(username
));
1799 blen
+= base64_encode_update(&ctx
, loginbuf
+blen
, strlen(request
->peer_login
+1), reinterpret_cast<const uint8_t*>(request
->peer_login
+1));
1800 blen
+= base64_encode_final(&ctx
, loginbuf
+blen
);
1801 httpHeaderPutStrf(hdr_out
, header
, "Basic %.*s", (int)blen
, loginbuf
);
1805 /* external_acl provided credentials */
1806 if (request
->extacl_user
.size() && request
->extacl_passwd
.size() &&
1807 (strcmp(request
->peer_login
, "PASS") == 0 ||
1808 strcmp(request
->peer_login
, "PROXYPASS") == 0)) {
1810 blen
= base64_encode_update(&ctx
, loginbuf
, request
->extacl_user
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_user
.rawBuf()));
1811 blen
+= base64_encode_update(&ctx
, loginbuf
+blen
, 1, reinterpret_cast<const uint8_t*>(":"));
1812 blen
+= base64_encode_update(&ctx
, loginbuf
+blen
, request
->extacl_passwd
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_passwd
.rawBuf()));
1813 blen
+= base64_encode_final(&ctx
, loginbuf
+blen
);
1814 httpHeaderPutStrf(hdr_out
, header
, "Basic %.*s", (int)blen
, loginbuf
);
1817 // if no external user credentials are available to fake authentication with PASS acts like PASSTHRU
1818 if (strcmp(request
->peer_login
, "PASS") == 0)
1821 /* Kerberos login to peer */
1822 #if HAVE_AUTH_MODULE_NEGOTIATE && HAVE_KRB5 && HAVE_GSSAPI
1823 if (strncmp(request
->peer_login
, "NEGOTIATE",strlen("NEGOTIATE")) == 0) {
1825 char *PrincipalName
=NULL
,*p
;
1826 int negotiate_flags
= 0;
1828 if ((p
=strchr(request
->peer_login
,':')) != NULL
) {
1831 if (request
->flags
.auth_no_keytab
) {
1832 negotiate_flags
|= PEER_PROXY_NEGOTIATE_NOKEYTAB
;
1834 Token
= peer_proxy_negotiate_auth(PrincipalName
, request
->peer_host
, negotiate_flags
);
1836 httpHeaderPutStrf(hdr_out
, header
, "Negotiate %s",Token
);
1840 #endif /* HAVE_KRB5 && HAVE_GSSAPI */
1842 blen
= base64_encode_update(&ctx
, loginbuf
, strlen(request
->peer_login
), reinterpret_cast<const uint8_t*>(request
->peer_login
));
1843 blen
+= base64_encode_final(&ctx
, loginbuf
+blen
);
1844 httpHeaderPutStrf(hdr_out
, header
, "Basic %.*s", (int)blen
, loginbuf
);
1849 * build request headers and append them to a given MemBuf
1850 * used by buildRequestPrefix()
1851 * note: initialised the HttpHeader, the caller is responsible for Clean()-ing
1854 HttpStateData::httpBuildRequestHeader(HttpRequest
* request
,
1856 const AccessLogEntryPointer
&al
,
1857 HttpHeader
* hdr_out
,
1858 const Http::StateFlags
&flags
)
1860 /* building buffer for complex strings */
1861 #define BBUF_SZ (MAX_URL+32)
1862 LOCAL_ARRAY(char, bbuf
, BBUF_SZ
);
1863 LOCAL_ARRAY(char, ntoabuf
, MAX_IPSTRLEN
);
1864 const HttpHeader
*hdr_in
= &request
->header
;
1865 const HttpHeaderEntry
*e
= NULL
;
1866 HttpHeaderPos pos
= HttpHeaderInitPos
;
1867 assert (hdr_out
->owner
== hoRequest
);
1869 /* use our IMS header if the cached entry has Last-Modified time */
1870 if (request
->lastmod
> -1)
1871 hdr_out
->putTime(Http::HdrType::IF_MODIFIED_SINCE
, request
->lastmod
);
1873 // Add our own If-None-Match field if the cached entry has a strong ETag.
1874 // copyOneHeaderFromClientsideRequestToUpstreamRequest() adds client ones.
1875 if (request
->etag
.size() > 0) {
1876 hdr_out
->addEntry(new HttpHeaderEntry(Http::HdrType::IF_NONE_MATCH
, SBuf(),
1877 request
->etag
.termedBuf()));
1880 bool we_do_ranges
= decideIfWeDoRanges (request
);
1882 String
strConnection (hdr_in
->getList(Http::HdrType::CONNECTION
));
1884 while ((e
= hdr_in
->getEntry(&pos
)))
1885 copyOneHeaderFromClientsideRequestToUpstreamRequest(e
, strConnection
, request
, hdr_out
, we_do_ranges
, flags
);
1887 /* Abstraction break: We should interpret multipart/byterange responses
1888 * into offset-length data, and this works around our inability to do so.
1890 if (!we_do_ranges
&& request
->multipartRangeRequest()) {
1891 /* don't cache the result */
1892 request
->flags
.cachable
= false;
1893 /* pretend it's not a range request */
1894 request
->ignoreRange("want to request the whole object");
1895 request
->flags
.isRanged
= false;
1898 hdr_out
->addVia(request
->http_ver
, hdr_in
);
1900 if (request
->flags
.accelerated
) {
1901 /* Append Surrogate-Capabilities */
1902 String
strSurrogate(hdr_in
->getList(Http::HdrType::SURROGATE_CAPABILITY
));
1904 snprintf(bbuf
, BBUF_SZ
, "%s=\"Surrogate/1.0 ESI/1.0\"", Config
.Accel
.surrogate_id
);
1906 snprintf(bbuf
, BBUF_SZ
, "%s=\"Surrogate/1.0\"", Config
.Accel
.surrogate_id
);
1908 strListAdd(&strSurrogate
, bbuf
, ',');
1909 hdr_out
->putStr(Http::HdrType::SURROGATE_CAPABILITY
, strSurrogate
.termedBuf());
1912 /** \pre Handle X-Forwarded-For */
1913 if (strcmp(opt_forwarded_for
, "delete") != 0) {
1915 String strFwd
= hdr_in
->getList(Http::HdrType::X_FORWARDED_FOR
);
1917 // if we cannot double strFwd size, then it grew past 50% of the limit
1918 if (!strFwd
.canGrowBy(strFwd
.size())) {
1919 // There is probably a forwarding loop with Via detection disabled.
1920 // If we do nothing, String will assert on overflow soon.
1921 // TODO: Terminate all transactions with huge XFF?
1924 static int warnedCount
= 0;
1925 if (warnedCount
++ < 100) {
1926 const SBuf
url(entry
? SBuf(entry
->url()) : request
->effectiveRequestUri());
1927 debugs(11, DBG_IMPORTANT
, "Warning: likely forwarding loop with " << url
);
1931 if (strcmp(opt_forwarded_for
, "on") == 0) {
1932 /** If set to ON - append client IP or 'unknown'. */
1933 if ( request
->client_addr
.isNoAddr() )
1934 strListAdd(&strFwd
, "unknown", ',');
1936 strListAdd(&strFwd
, request
->client_addr
.toStr(ntoabuf
, MAX_IPSTRLEN
), ',');
1937 } else if (strcmp(opt_forwarded_for
, "off") == 0) {
1938 /** If set to OFF - append 'unknown'. */
1939 strListAdd(&strFwd
, "unknown", ',');
1940 } else if (strcmp(opt_forwarded_for
, "transparent") == 0) {
1941 /** If set to TRANSPARENT - pass through unchanged. */
1942 } else if (strcmp(opt_forwarded_for
, "truncate") == 0) {
1943 /** If set to TRUNCATE - drop existing list and replace with client IP or 'unknown'. */
1944 if ( request
->client_addr
.isNoAddr() )
1947 strFwd
= request
->client_addr
.toStr(ntoabuf
, MAX_IPSTRLEN
);
1949 if (strFwd
.size() > 0)
1950 hdr_out
->putStr(Http::HdrType::X_FORWARDED_FOR
, strFwd
.termedBuf());
1952 /** If set to DELETE - do not copy through. */
1954 /* append Host if not there already */
1955 if (!hdr_out
->has(Http::HdrType::HOST
)) {
1956 if (request
->peer_domain
) {
1957 hdr_out
->putStr(Http::HdrType::HOST
, request
->peer_domain
);
1959 SBuf authority
= request
->url
.authority();
1960 hdr_out
->putStr(Http::HdrType::HOST
, authority
.c_str());
1964 /* append Authorization if known in URL, not in header and going direct */
1965 if (!hdr_out
->has(Http::HdrType::AUTHORIZATION
)) {
1966 if (flags
.toOrigin
&& !request
->url
.userInfo().isEmpty()) {
1967 static char result
[base64_encode_len(MAX_URL
*2)]; // should be big enough for a single URI segment
1968 struct base64_encode_ctx ctx
;
1969 base64_encode_init(&ctx
);
1970 size_t blen
= base64_encode_update(&ctx
, result
, request
->url
.userInfo().length(), reinterpret_cast<const uint8_t*>(request
->url
.userInfo().rawContent()));
1971 blen
+= base64_encode_final(&ctx
, result
+blen
);
1972 result
[blen
] = '\0';
1974 httpHeaderPutStrf(hdr_out
, Http::HdrType::AUTHORIZATION
, "Basic %.*s", (int)blen
, result
);
1978 /* Fixup (Proxy-)Authorization special cases. Plain relaying dealt with above */
1979 httpFixupAuthentication(request
, hdr_in
, hdr_out
, flags
);
1981 /* append Cache-Control, add max-age if not there already */
1983 HttpHdrCc
*cc
= hdr_in
->getCc();
1986 cc
= new HttpHdrCc();
1988 #if 0 /* see bug 2330 */
1989 /* Set no-cache if determined needed but not found */
1990 if (request
->flags
.nocache
)
1991 EBIT_SET(cc
->mask
, HttpHdrCcType::CC_NO_CACHE
);
1994 /* Add max-age only without no-cache */
1995 if (!cc
->hasMaxAge() && !cc
->hasNoCache()) {
1996 // XXX: performance regression. c_str() reallocates
1997 SBuf
tmp(request
->effectiveRequestUri());
1998 cc
->maxAge(getMaxAge(entry
? entry
->url() : tmp
.c_str()));
2001 /* Enforce sibling relations */
2002 if (flags
.only_if_cached
)
2003 cc
->onlyIfCached(true);
2010 // Always send Connection because HTTP/1.0 servers need explicit
2011 // "keep-alive", HTTP/1.1 servers need explicit "close", Upgrade recipients
2012 // need bare "upgrade", and we do not always know the server expectations.
2013 if (!hdr_out
->has(Http::HdrType::CONNECTION
)) // forwardUpgrade() may add it
2014 hdr_out
->putStr(Http::HdrType::CONNECTION
, flags
.keepalive
? "keep-alive" : "close");
2016 /* append Front-End-Https */
2017 if (flags
.front_end_https
) {
2018 if (flags
.front_end_https
== 1 || request
->url
.getScheme() == AnyP::PROTO_HTTPS
)
2019 hdr_out
->putStr(Http::HdrType::FRONT_END_HTTPS
, "On");
2022 if (flags
.chunked_request
) {
2023 // Do not just copy the original value so that if the client-side
2024 // starts decode other encodings, this code may remain valid.
2025 hdr_out
->putStr(Http::HdrType::TRANSFER_ENCODING
, "chunked");
2028 /* Now mangle the headers. */
2029 httpHdrMangleList(hdr_out
, request
, al
, ROR_REQUEST
);
2031 strConnection
.clean();
2034 /// copies from-client Upgrade info into the given to-server header while
2035 /// honoring configuration filters and following HTTP requirements
2037 HttpStateData::forwardUpgrade(HttpHeader
&hdrOut
)
2039 if (!Config
.http_upgrade_request_protocols
)
2040 return; // forward nothing by default
2042 /* RFC 7230 section 6.7 paragraph 10:
2043 * A server MUST ignore an Upgrade header field that is received in
2044 * an HTTP/1.0 request.
2046 if (request
->http_ver
== Http::ProtocolVersion(1,0))
2049 const auto &hdrIn
= request
->header
;
2050 if (!hdrIn
.has(Http::HdrType::UPGRADE
))
2052 const auto upgradeIn
= hdrIn
.getList(Http::HdrType::UPGRADE
);
2056 ACLFilledChecklist
ch(nullptr, request
.getRaw());
2058 const char *pos
= nullptr;
2059 const char *offeredStr
= nullptr;
2060 int offeredStrLen
= 0;
2061 while (strListGetItem(&upgradeIn
, ',', &offeredStr
, &offeredStrLen
, &pos
)) {
2062 const ProtocolView
offeredProto(offeredStr
, offeredStrLen
);
2063 debugs(11, 5, "checks all rules applicable to " << offeredProto
);
2064 Config
.http_upgrade_request_protocols
->forApplicable(offeredProto
, [&ch
, offeredStr
, offeredStrLen
, &upgradeOut
] (const SBuf
&cfgProto
, const acl_access
*guard
) {
2065 debugs(11, 5, "checks " << cfgProto
<< " rule(s)");
2066 ch
.changeAcl(guard
);
2067 const auto answer
= ch
.fastCheck();
2068 if (answer
.implicit
)
2069 return false; // keep looking for an explicit rule match
2070 if (answer
.allowed())
2071 strListAdd(upgradeOut
, offeredStr
, offeredStrLen
);
2072 // else drop the offer (explicitly denied cases and ACL errors)
2073 return true; // stop after an explicit rule match or an error
2077 if (upgradeOut
.size()) {
2078 hdrOut
.putStr(Http::HdrType::UPGRADE
, upgradeOut
.termedBuf());
2080 /* RFC 7230 section 6.7 paragraph 10:
2081 * When Upgrade is sent, the sender MUST also send a Connection header
2082 * field that contains an "upgrade" connection option, in
2083 * order to prevent Upgrade from being accidentally forwarded by
2084 * intermediaries that might not implement the listed protocols.
2086 * NP: Squid does not truly implement the protocol(s) in this Upgrade.
2087 * For now we are treating an explicit blind tunnel as "implemented"
2088 * regardless of the security implications.
2090 hdrOut
.putStr(Http::HdrType::CONNECTION
, "upgrade");
2092 // Connection:close and Connection:keepalive confuse some Upgrade
2093 // recipients, so we do not send those headers. Our Upgrade request
2094 // implicitly offers connection persistency per HTTP/1.1 defaults.
2095 // Update the keepalive flag to reflect that offer.
2096 // * If the server upgrades, then we would not be talking HTTP past the
2097 // HTTP 101 control message, and HTTP persistence would be irrelevant.
2098 // * Otherwise, our request will contradict onoff.server_pconns=off or
2099 // other no-keepalive conditions (if any). We compensate by copying
2100 // the original no-keepalive decision now and honoring it later.
2101 flags
.forceClose
= !flags
.keepalive
;
2102 flags
.keepalive
= true; // should already be true in most cases
2107 * Decides whether a particular header may be cloned from the received Clients request
2108 * to our outgoing fetch request.
2111 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry
*e
, const String strConnection
, const HttpRequest
* request
, HttpHeader
* hdr_out
, const int we_do_ranges
, const Http::StateFlags
&flags
)
2113 debugs(11, 5, "httpBuildRequestHeader: " << e
->name
<< ": " << e
->value
);
2117 /** \par RFC 2616 sect 13.5.1 - Hop-by-Hop headers which Squid should not pass on. */
2119 case Http::HdrType::PROXY_AUTHORIZATION
:
2120 /** \par Proxy-Authorization:
2121 * Only pass on proxy authentication to peers for which
2122 * authentication forwarding is explicitly enabled
2124 if (!flags
.toOrigin
&& request
->peer_login
&&
2125 (strcmp(request
->peer_login
, "PASS") == 0 ||
2126 strcmp(request
->peer_login
, "PROXYPASS") == 0 ||
2127 strcmp(request
->peer_login
, "PASSTHRU") == 0)) {
2128 hdr_out
->addEntry(e
->clone());
2132 /** \par RFC 2616 sect 13.5.1 - Hop-by-Hop headers which Squid does not pass on. */
2134 case Http::HdrType::CONNECTION
: /** \par Connection: */
2135 case Http::HdrType::TE
: /** \par TE: */
2136 case Http::HdrType::KEEP_ALIVE
: /** \par Keep-Alive: */
2137 case Http::HdrType::PROXY_AUTHENTICATE
: /** \par Proxy-Authenticate: */
2138 case Http::HdrType::TRAILER
: /** \par Trailer: */
2139 case Http::HdrType::TRANSFER_ENCODING
: /** \par Transfer-Encoding: */
2142 /// \par Upgrade is hop-by-hop but forwardUpgrade() may send a filtered one
2143 case Http::HdrType::UPGRADE
:
2146 /** \par OTHER headers I haven't bothered to track down yet. */
2148 case Http::HdrType::AUTHORIZATION
:
2149 /** \par WWW-Authorization:
2150 * Pass on WWW authentication */
2152 if (!flags
.toOriginPeer()) {
2153 hdr_out
->addEntry(e
->clone());
2155 /** \note Assume that talking to a cache_peer originserver makes
2156 * us a reverse proxy and only forward authentication if enabled
2157 * (see also httpFixupAuthentication for special cases)
2159 if (request
->peer_login
&&
2160 (strcmp(request
->peer_login
, "PASS") == 0 ||
2161 strcmp(request
->peer_login
, "PASSTHRU") == 0 ||
2162 strcmp(request
->peer_login
, "PROXYPASS") == 0)) {
2163 hdr_out
->addEntry(e
->clone());
2169 case Http::HdrType::HOST
:
2171 * Normally Squid rewrites the Host: header.
2172 * However, there is one case when we don't: If the URL
2173 * went through our redirector and the admin configured
2174 * 'redir_rewrites_host' to be off.
2176 if (request
->peer_domain
)
2177 hdr_out
->putStr(Http::HdrType::HOST
, request
->peer_domain
);
2178 else if (request
->flags
.redirected
&& !Config
.onoff
.redir_rewrites_host
)
2179 hdr_out
->addEntry(e
->clone());
2181 SBuf authority
= request
->url
.authority();
2182 hdr_out
->putStr(Http::HdrType::HOST
, authority
.c_str());
2187 case Http::HdrType::IF_MODIFIED_SINCE
:
2188 /** \par If-Modified-Since:
2189 * append unless we added our own,
2190 * but only if cache_miss_revalidate is enabled, or
2191 * the request is not cacheable, or
2192 * the request contains authentication credentials.
2193 * \note at most one client's If-Modified-Since header can pass through
2195 // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2196 if (hdr_out
->has(Http::HdrType::IF_MODIFIED_SINCE
))
2198 else if (Config
.onoff
.cache_miss_revalidate
|| !request
->flags
.cachable
|| request
->flags
.auth
)
2199 hdr_out
->addEntry(e
->clone());
2202 case Http::HdrType::IF_NONE_MATCH
:
2203 /** \par If-None-Match:
2204 * append if the wildcard '*' special case value is present, or
2205 * cache_miss_revalidate is disabled, or
2206 * the request is not cacheable in this proxy, or
2207 * the request contains authentication credentials.
2208 * \note this header lists a set of responses for the server to elide sending. Squid added values are extending that set.
2210 // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2211 if (hdr_out
->hasListMember(Http::HdrType::IF_MATCH
, "*", ',') || Config
.onoff
.cache_miss_revalidate
|| !request
->flags
.cachable
|| request
->flags
.auth
)
2212 hdr_out
->addEntry(e
->clone());
2215 case Http::HdrType::MAX_FORWARDS
:
2216 /** \par Max-Forwards:
2217 * pass only on TRACE or OPTIONS requests */
2218 if (request
->method
== Http::METHOD_TRACE
|| request
->method
== Http::METHOD_OPTIONS
) {
2219 const int64_t hops
= e
->getInt64();
2222 hdr_out
->putInt64(Http::HdrType::MAX_FORWARDS
, hops
- 1);
2227 case Http::HdrType::VIA
:
2229 * If Via is disabled then forward any received header as-is.
2230 * Otherwise leave for explicit updated addition later. */
2232 if (!Config
.onoff
.via
)
2233 hdr_out
->addEntry(e
->clone());
2237 case Http::HdrType::RANGE
:
2239 case Http::HdrType::IF_RANGE
:
2241 case Http::HdrType::REQUEST_RANGE
:
2242 /** \par Range:, If-Range:, Request-Range:
2243 * Only pass if we accept ranges */
2245 hdr_out
->addEntry(e
->clone());
2249 case Http::HdrType::PROXY_CONNECTION
: // SHOULD ignore. But doing so breaks things.
2252 case Http::HdrType::CONTENT_LENGTH
:
2253 // pass through unless we chunk; also, keeping this away from default
2254 // prevents request smuggling via Connection: Content-Length tricks
2255 if (!flags
.chunked_request
)
2256 hdr_out
->addEntry(e
->clone());
2259 case Http::HdrType::X_FORWARDED_FOR
:
2261 case Http::HdrType::CACHE_CONTROL
:
2262 /** \par X-Forwarded-For:, Cache-Control:
2263 * handled specially by Squid, so leave off for now.
2264 * append these after the loop if needed */
2267 case Http::HdrType::FRONT_END_HTTPS
:
2268 /** \par Front-End-Https:
2269 * Pass thru only if peer is configured with front-end-https */
2270 if (!flags
.front_end_https
)
2271 hdr_out
->addEntry(e
->clone());
2277 * pass on all other header fields
2278 * which are NOT listed by the special Connection: header. */
2279 if (strConnection
.size()>0 && strListIsMember(&strConnection
, e
->name
, ',')) {
2280 debugs(11, 2, "'" << e
->name
<< "' header cropped by Connection: definition");
2284 hdr_out
->addEntry(e
->clone());
2289 HttpStateData::decideIfWeDoRanges (HttpRequest
* request
)
2292 /* decide if we want to do Ranges ourselves
2293 * and fetch the whole object now)
2294 * We want to handle Ranges ourselves iff
2295 * - we can actually parse client Range specs
2296 * - the specs are expected to be simple enough (e.g. no out-of-order ranges)
2297 * - reply will be cachable
2298 * (If the reply will be uncachable we have to throw it away after
2299 * serving this request, so it is better to forward ranges to
2300 * the server and fetch only the requested content)
2303 int64_t roffLimit
= request
->getRangeOffsetLimit();
2305 if (NULL
== request
->range
|| !request
->flags
.cachable
2306 || request
->range
->offsetLimitExceeded(roffLimit
) || request
->flags
.connectionAuth
)
2309 debugs(11, 8, "decideIfWeDoRanges: range specs: " <<
2310 request
->range
<< ", cachable: " <<
2311 request
->flags
.cachable
<< "; we_do_ranges: " << result
);
2316 /* build request prefix and append it to a given MemBuf;
2317 * return the length of the prefix */
2319 HttpStateData::buildRequestPrefix(MemBuf
* mb
)
2321 const int offset
= mb
->size
;
2322 /* Uses a local httpver variable to print the HTTP label
2323 * since the HttpRequest may have an older version label.
2324 * XXX: This could create protocol bugs as the headers sent and
2325 * flow control should all be based on the HttpRequest version
2326 * not the one we are sending. Needs checking.
2328 const AnyP::ProtocolVersion httpver
= Http::ProtocolVersion();
2329 const SBuf
url(flags
.toOrigin
? request
->url
.path() : request
->effectiveRequestUri());
2330 mb
->appendf(SQUIDSBUFPH
" " SQUIDSBUFPH
" %s/%d.%d\r\n",
2331 SQUIDSBUFPRINT(request
->method
.image()),
2332 SQUIDSBUFPRINT(url
),
2333 AnyP::ProtocolType_str
[httpver
.protocol
],
2334 httpver
.major
,httpver
.minor
);
2335 /* build and pack headers */
2337 HttpHeader
hdr(hoRequest
);
2338 forwardUpgrade(hdr
); // before httpBuildRequestHeader() for CONNECTION
2339 httpBuildRequestHeader(request
.getRaw(), entry
, fwd
->al
, &hdr
, flags
);
2341 if (request
->flags
.pinned
&& request
->flags
.connectionAuth
)
2342 request
->flags
.authSent
= true;
2343 else if (hdr
.has(Http::HdrType::AUTHORIZATION
))
2344 request
->flags
.authSent
= true;
2346 // The late placement of this check supports reply_header_add mangling,
2347 // but also complicates optimizing upgradeHeaderOut-like lookups.
2348 if (hdr
.has(Http::HdrType::UPGRADE
)) {
2349 assert(!upgradeHeaderOut
);
2350 upgradeHeaderOut
= new String(hdr
.getList(Http::HdrType::UPGRADE
));
2356 /* append header terminator */
2357 mb
->append(crlf
, 2);
2358 return mb
->size
- offset
;
2361 /* This will be called when connect completes. Write request. */
2363 HttpStateData::sendRequest()
2367 debugs(11, 5, HERE
<< serverConnection
<< ", request " << request
<< ", this " << this << ".");
2369 if (!Comm::IsConnOpen(serverConnection
)) {
2370 debugs(11,3, HERE
<< "cannot send request to closing " << serverConnection
);
2371 assert(closeHandler
!= NULL
);
2375 typedef CommCbMemFunT
<HttpStateData
, CommTimeoutCbParams
> TimeoutDialer
;
2376 AsyncCall::Pointer timeoutCall
= JobCallback(11, 5,
2377 TimeoutDialer
, this, HttpStateData::httpTimeout
);
2378 commSetConnTimeout(serverConnection
, Config
.Timeout
.lifetime
, timeoutCall
);
2379 flags
.do_next_read
= true;
2380 maybeReadVirginBody();
2382 if (request
->body_pipe
!= NULL
) {
2383 if (!startRequestBodyFlow()) // register to receive body data
2385 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2386 requestSender
= JobCallback(11,5,
2387 Dialer
, this, HttpStateData::sentRequestBody
);
2389 Must(!flags
.chunked_request
);
2390 // use chunked encoding if we do not know the length
2391 if (request
->content_length
< 0)
2392 flags
.chunked_request
= true;
2394 assert(!requestBodySource
);
2395 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2396 requestSender
= JobCallback(11,5,
2397 Dialer
, this, HttpStateData::wroteLast
);
2401 * Is keep-alive okay for all request methods?
2403 if (request
->flags
.mustKeepalive
)
2404 flags
.keepalive
= true;
2405 else if (request
->flags
.pinned
)
2406 flags
.keepalive
= request
->persistent();
2407 else if (!Config
.onoff
.server_pconns
)
2408 flags
.keepalive
= false;
2409 else if (flags
.tunneling
)
2410 // tunneled non pinned bumped requests must not keepalive
2411 flags
.keepalive
= !request
->flags
.sslBumped
;
2412 else if (_peer
== NULL
)
2413 flags
.keepalive
= true;
2414 else if (_peer
->stats
.n_keepalives_sent
< 10)
2415 flags
.keepalive
= true;
2416 else if ((double) _peer
->stats
.n_keepalives_recv
/
2417 (double) _peer
->stats
.n_keepalives_sent
> 0.50)
2418 flags
.keepalive
= true;
2420 if (_peer
&& !flags
.tunneling
) {
2421 /*The old code here was
2422 if (neighborType(_peer, request->url) == PEER_SIBLING && ...
2423 which is equivalent to:
2424 if (neighborType(_peer, URL()) == PEER_SIBLING && ...
2426 if (((_peer->type == PEER_MULTICAST && p->options.mcast_siblings) ||
2427 _peer->type == PEER_SIBLINGS ) && _peer->options.allow_miss)
2428 flags.only_if_cached = 1;
2430 But I suppose it was a bug
2432 if (neighborType(_peer
, request
->url
) == PEER_SIBLING
&& !_peer
->options
.allow_miss
)
2433 flags
.only_if_cached
= true;
2435 flags
.front_end_https
= _peer
->front_end_https
;
2439 request
->peer_host
=_peer
?_peer
->host
:NULL
;
2440 buildRequestPrefix(&mb
);
2442 debugs(11, 2, "HTTP Server " << serverConnection
);
2443 debugs(11, 2, "HTTP Server REQUEST:\n---------\n" << mb
.buf
<< "\n----------");
2445 Comm::Write(serverConnection
, &mb
, requestSender
);
2450 HttpStateData::getMoreRequestBody(MemBuf
&buf
)
2452 // parent's implementation can handle the no-encoding case
2453 if (!flags
.chunked_request
)
2454 return Client::getMoreRequestBody(buf
);
2458 Must(requestBodySource
!= NULL
);
2459 if (!requestBodySource
->getMoreData(raw
))
2460 return false; // no request body bytes to chunk yet
2462 // optimization: pre-allocate buffer size that should be enough
2463 const mb_size_t rawDataSize
= raw
.contentSize();
2464 // we may need to send: hex-chunk-size CRLF raw-data CRLF last-chunk
2465 buf
.init(16 + 2 + rawDataSize
+ 2 + 5, raw
.max_capacity
);
2467 buf
.appendf("%x\r\n", static_cast<unsigned int>(rawDataSize
));
2468 buf
.append(raw
.content(), rawDataSize
);
2469 buf
.append("\r\n", 2);
2471 Must(rawDataSize
> 0); // we did not accidentally created last-chunk above
2473 // Do not send last-chunk unless we successfully received everything
2474 if (receivedWholeRequestBody
) {
2475 Must(!flags
.sentLastChunk
);
2476 flags
.sentLastChunk
= true;
2477 buf
.append("0\r\n\r\n", 5);
2484 httpStart(FwdState
*fwd
)
2486 debugs(11, 3, fwd
->request
->method
<< ' ' << fwd
->entry
->url());
2487 AsyncJob::Start(new HttpStateData(fwd
));
2491 HttpStateData::start()
2493 if (!sendRequest()) {
2494 debugs(11, 3, "httpStart: aborted");
2495 mustStop("HttpStateData::start failed");
2499 ++ statCounter
.server
.all
.requests
;
2500 ++ statCounter
.server
.http
.requests
;
2503 * We used to set the read timeout here, but not any more.
2504 * Now its set in httpSendComplete() after the full request,
2505 * including request body, has been written to the server.
2509 /// if broken posts are enabled for the request, try to fix and return true
2511 HttpStateData::finishingBrokenPost()
2513 #if USE_HTTP_VIOLATIONS
2514 if (!Config
.accessList
.brokenPosts
) {
2515 debugs(11, 5, HERE
<< "No brokenPosts list");
2519 ACLFilledChecklist
ch(Config
.accessList
.brokenPosts
, originalRequest().getRaw());
2521 ch
.syncAle(originalRequest().getRaw(), nullptr);
2522 if (!ch
.fastCheck().allowed()) {
2523 debugs(11, 5, HERE
<< "didn't match brokenPosts");
2527 if (!Comm::IsConnOpen(serverConnection
)) {
2528 debugs(11, 3, HERE
<< "ignoring broken POST for closed " << serverConnection
);
2529 assert(closeHandler
!= NULL
);
2530 return true; // prevent caller from proceeding as if nothing happened
2533 debugs(11, 3, "finishingBrokenPost: fixing broken POST");
2534 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2535 requestSender
= JobCallback(11,5,
2536 Dialer
, this, HttpStateData::wroteLast
);
2537 Comm::Write(serverConnection
, "\r\n", 2, requestSender
, NULL
);
2541 #endif /* USE_HTTP_VIOLATIONS */
2544 /// if needed, write last-chunk to end the request body and return true
2546 HttpStateData::finishingChunkedRequest()
2548 if (flags
.sentLastChunk
) {
2549 debugs(11, 5, HERE
<< "already sent last-chunk");
2553 Must(receivedWholeRequestBody
); // or we should not be sending last-chunk
2554 flags
.sentLastChunk
= true;
2556 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2557 requestSender
= JobCallback(11,5, Dialer
, this, HttpStateData::wroteLast
);
2558 Comm::Write(serverConnection
, "0\r\n\r\n", 5, requestSender
, NULL
);
2563 HttpStateData::doneSendingRequestBody()
2565 Client::doneSendingRequestBody();
2566 debugs(11,5, HERE
<< serverConnection
);
2568 // do we need to write something after the last body byte?
2569 if (flags
.chunked_request
&& finishingChunkedRequest())
2571 if (!flags
.chunked_request
&& finishingBrokenPost())
2577 // more origin request body data is available
2579 HttpStateData::handleMoreRequestBodyAvailable()
2581 if (eof
|| !Comm::IsConnOpen(serverConnection
)) {
2582 // XXX: we should check this condition in other callbacks then!
2583 // TODO: Check whether this can actually happen: We should unsubscribe
2584 // as a body consumer when the above condition(s) are detected.
2585 debugs(11, DBG_IMPORTANT
, HERE
<< "Transaction aborted while reading HTTP body");
2589 assert(requestBodySource
!= NULL
);
2591 if (requestBodySource
->buf().hasContent()) {
2592 // XXX: why does not this trigger a debug message on every request?
2594 if (flags
.headers_parsed
&& !flags
.abuse_detected
) {
2595 flags
.abuse_detected
= true;
2596 debugs(11, DBG_IMPORTANT
, "http handleMoreRequestBodyAvailable: Likely proxy abuse detected '" << request
->client_addr
<< "' -> '" << entry
->url() << "'" );
2598 if (virginReply()->sline
.status() == Http::scInvalidHeader
) {
2600 mustStop("HttpStateData::handleMoreRequestBodyAvailable");
2606 HttpStateData::handleMoreRequestBodyAvailable();
2609 // premature end of the request body
2611 HttpStateData::handleRequestBodyProducerAborted()
2613 Client::handleRequestBodyProducerAborted();
2614 if (entry
->isEmpty()) {
2615 debugs(11, 3, "request body aborted: " << serverConnection
);
2616 // We usually get here when ICAP REQMOD aborts during body processing.
2617 // We might also get here if client-side aborts, but then our response
2618 // should not matter because either client-side will provide its own or
2619 // there will be no response at all (e.g., if the the client has left).
2620 const auto err
= new ErrorState(ERR_ICAP_FAILURE
, Http::scInternalServerError
, fwd
->request
, fwd
->al
);
2621 static const auto d
= MakeNamedErrorDetail("SRV_REQMOD_REQ_BODY");
2622 err
->detailError(d
);
2626 abortTransaction("request body producer aborted");
2629 // called when we wrote request headers(!) or a part of the body
2631 HttpStateData::sentRequestBody(const CommIoCbParams
&io
)
2634 statCounter
.server
.http
.kbytes_out
+= io
.size
;
2636 Client::sentRequestBody(io
);
2640 HttpStateData::abortAll(const char *reason
)
2642 debugs(11,5, HERE
<< "aborting transaction for " << reason
<<
2643 "; " << serverConnection
<< ", this " << this);
2647 HttpStateData::ReuseDecision::ReuseDecision(const StoreEntry
*e
, const Http::StatusCode code
)
2648 : answer(HttpStateData::ReuseDecision::reuseNot
), reason(nullptr), entry(e
), statusCode(code
) {}
2650 HttpStateData::ReuseDecision::Answers
2651 HttpStateData::ReuseDecision::make(const HttpStateData::ReuseDecision::Answers ans
, const char *why
)
2658 std::ostream
&operator <<(std::ostream
&os
, const HttpStateData::ReuseDecision
&d
)
2660 static const char *ReuseMessages
[] = {
2661 "do not cache and do not share", // reuseNot
2662 "cache positively and share", // cachePositively
2663 "cache negatively and share", // cacheNegatively
2664 "do not cache but share" // doNotCacheButShare
2667 assert(d
.answer
>= HttpStateData::ReuseDecision::reuseNot
&&
2668 d
.answer
<= HttpStateData::ReuseDecision::doNotCacheButShare
);
2669 return os
<< ReuseMessages
[d
.answer
] << " because " << d
.reason
<<
2670 "; HTTP status " << d
.statusCode
<< " " << *(d
.entry
);