2 * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 11 Hypertext Transfer Protocol (HTTP) */
12 * Anonymizing patch by lutz@as-node.jena.thur.de
13 * have a look into http-anon.c to get more informations.
17 #include "acl/FilledChecklist.h"
18 #include "base/AsyncJobCalls.h"
19 #include "base/TextException.h"
21 #include "CachePeer.h"
22 #include "client_side.h"
23 #include "comm/Connection.h"
24 #include "comm/Read.h"
25 #include "comm/Write.h"
27 #include "err_detail_type.h"
28 #include "errorpage.h"
33 #include "http/one/ResponseParser.h"
34 #include "http/one/TeChunkedParser.h"
35 #include "http/Stream.h"
36 #include "HttpControlMsg.h"
37 #include "HttpHdrCc.h"
38 #include "HttpHdrContRange.h"
39 #include "HttpHdrSc.h"
40 #include "HttpHdrScTarget.h"
41 #include "HttpHeaderTools.h"
42 #include "HttpReply.h"
43 #include "HttpRequest.h"
44 #include "log/access_log.h"
46 #include "MemObject.h"
47 #include "neighbors.h"
48 #include "peer_proxy_negotiate_auth.h"
49 #include "profiler/Profiler.h"
51 #include "RefreshPattern.h"
53 #include "SquidConfig.h"
54 #include "SquidTime.h"
55 #include "StatCounters.h"
63 #include "auth/UserRequest.h"
66 #include "DelayPools.h"
69 #define SQUID_ENTER_THROWING_CODE() try {
70 #define SQUID_EXIT_THROWING_CODE(status) \
73 catch (const std::exception &e) { \
74 debugs (11, 1, "Exception error:" << e.what()); \
78 CBDATA_CLASS_INIT(HttpStateData
);
80 static const char *const crlf
= "\r\n";
82 static void httpMaybeRemovePublic(StoreEntry
*, Http::StatusCode
);
83 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry
*e
, const String strConnection
, const HttpRequest
* request
,
84 HttpHeader
* hdr_out
, const int we_do_ranges
, const Http::StateFlags
&);
86 HttpStateData::HttpStateData(FwdState
*theFwdState
) :
87 AsyncJob("HttpStateData"),
90 httpChunkDecoder(NULL
),
95 debugs(11,5,HERE
<< "HttpStateData " << this << " created");
96 ignoreCacheControl
= false;
97 surrogateNoStore
= false;
98 serverConnection
= fwd
->serverConnection();
100 // reset peer response time stats for %<pt
101 request
->hier
.peer_http_request_sent
.tv_sec
= 0;
102 request
->hier
.peer_http_request_sent
.tv_usec
= 0;
104 if (fwd
->serverConnection() != NULL
)
105 _peer
= cbdataReference(fwd
->serverConnection()->getPeer()); /* might be NULL */
108 request
->flags
.proxying
= true;
110 * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
111 * We might end up getting the object from somewhere else if,
112 * for example, the request to this neighbor fails.
114 if (_peer
->options
.proxy_only
)
115 entry
->releaseRequest();
118 entry
->setNoDelay(_peer
->options
.no_delay
);
123 * register the handler to free HTTP state data when the FD closes
125 typedef CommCbMemFunT
<HttpStateData
, CommCloseCbParams
> Dialer
;
126 closeHandler
= JobCallback(9, 5, Dialer
, this, HttpStateData::httpStateConnClosed
);
127 comm_add_close_handler(serverConnection
->fd
, closeHandler
);
130 HttpStateData::~HttpStateData()
133 * don't forget that ~Client() gets called automatically
136 if (httpChunkDecoder
)
137 delete httpChunkDecoder
;
139 cbdataReferenceDone(_peer
);
141 debugs(11,5, HERE
<< "HttpStateData " << this << " destroyed; " << serverConnection
);
144 const Comm::ConnectionPointer
&
145 HttpStateData::dataConnection() const
147 return serverConnection
;
151 HttpStateData::httpStateConnClosed(const CommCloseCbParams
¶ms
)
153 debugs(11, 5, "httpStateFree: FD " << params
.fd
<< ", httpState=" << params
.data
);
154 doneWithFwd
= "httpStateConnClosed()"; // assume FwdState is monitoring too
155 mustStop("HttpStateData::httpStateConnClosed");
159 HttpStateData::httpTimeout(const CommTimeoutCbParams
&)
161 debugs(11, 4, serverConnection
<< ": '" << entry
->url() << "'");
163 if (entry
->store_status
== STORE_PENDING
) {
164 fwd
->fail(new ErrorState(ERR_READ_TIMEOUT
, Http::scGatewayTimeout
, fwd
->request
));
168 mustStop("HttpStateData::httpTimeout");
172 findPreviouslyCachedEntry(StoreEntry
*newEntry
) {
173 assert(newEntry
->mem_obj
);
174 return newEntry
->mem_obj
->request
?
175 storeGetPublicByRequest(newEntry
->mem_obj
->request
) :
176 storeGetPublic(newEntry
->mem_obj
->storeId(), newEntry
->mem_obj
->method
);
179 /// Remove an existing public store entry if the incoming response (to be
180 /// stored in a currently private entry) is going to invalidate it.
182 httpMaybeRemovePublic(StoreEntry
* e
, Http::StatusCode status
)
187 // If the incoming response already goes into a public entry, then there is
188 // nothing to remove. This protects ready-for-collapsing entries as well.
189 if (!EBIT_TEST(e
->flags
, KEY_PRIVATE
))
192 // If the new/incoming response cannot be stored, then it does not
193 // compete with the old stored response for the public key, and the
194 // old stored response should be left as is.
195 if (e
->mem_obj
->request
&& !e
->mem_obj
->request
->flags
.cachable
)
202 case Http::scNonAuthoritativeInformation
:
204 case Http::scMultipleChoices
:
206 case Http::scMovedPermanently
:
210 case Http::scSeeOther
:
214 case Http::scNotFound
:
219 case Http::scForbidden
:
221 case Http::scMethodNotAllowed
:
228 case Http::scUnauthorized
:
238 * Any 2xx response should eject previously cached entities...
241 if (status
>= 200 && status
< 300)
249 if (!remove
&& !forbidden
)
252 StoreEntry
*pe
= findPreviouslyCachedEntry(e
);
257 neighborsHtcpClear(e
, NULL
, e
->mem_obj
->request
, e
->mem_obj
->method
, HTCP_CLR_INVALIDATION
);
263 * Also remove any cached HEAD response in case the object has
266 if (e
->mem_obj
->request
)
267 pe
= storeGetPublicByRequestMethod(e
->mem_obj
->request
, Http::METHOD_HEAD
);
269 pe
= storeGetPublic(e
->mem_obj
->storeId(), Http::METHOD_HEAD
);
274 neighborsHtcpClear(e
, NULL
, e
->mem_obj
->request
, HttpRequestMethod(Http::METHOD_HEAD
), HTCP_CLR_INVALIDATION
);
281 HttpStateData::processSurrogateControl(HttpReply
*reply
)
283 if (request
->flags
.accelerated
&& reply
->surrogate_control
) {
284 HttpHdrScTarget
*sctusable
= reply
->surrogate_control
->getMergedTarget(Config
.Accel
.surrogate_id
);
287 if (sctusable
->noStore() ||
288 (Config
.onoff
.surrogate_is_remote
289 && sctusable
->noStoreRemote())) {
290 surrogateNoStore
= true;
291 entry
->makePrivate();
294 /* The HttpHeader logic cannot tell if the header it's parsing is a reply to an
295 * accelerated request or not...
296 * Still, this is an abstraction breach. - RC
298 if (sctusable
->hasMaxAge()) {
299 if (sctusable
->maxAge() < sctusable
->maxStale())
300 reply
->expires
= reply
->date
+ sctusable
->maxAge();
302 reply
->expires
= reply
->date
+ sctusable
->maxStale();
304 /* And update the timestamps */
305 entry
->timestampsSet();
308 /* We ignore cache-control directives as per the Surrogate specification */
309 ignoreCacheControl
= true;
317 HttpStateData::cacheableReply()
319 HttpReply
const *rep
= finalReply();
320 HttpHeader
const *hdr
= &rep
->header
;
322 #if USE_HTTP_VIOLATIONS
324 const RefreshPattern
*R
= NULL
;
326 /* This strange looking define first looks up the refresh pattern
327 * and then checks if the specified flag is set. The main purpose
328 * of this is to simplify the refresh pattern lookup and USE_HTTP_VIOLATIONS
331 #define REFRESH_OVERRIDE(flag) \
332 ((R = (R ? R : refreshLimits(entry->mem_obj->storeId()))) , \
333 (R && R->flags.flag))
335 #define REFRESH_OVERRIDE(flag) 0
338 if (EBIT_TEST(entry
->flags
, RELEASE_REQUEST
)) {
339 debugs(22, 3, "NO because " << *entry
<< " has been released.");
343 // RFC 7234 section 4: a cache MUST use the most recent response
344 // (as determined by the Date header field)
346 debugs(22, 3, "NO because " << *entry
<< " has an older date header.");
350 // Check for Surrogate/1.0 protocol conditions
351 // NP: reverse-proxy traffic our parent server has instructed us never to cache
352 if (surrogateNoStore
) {
353 debugs(22, 3, HERE
<< "NO because Surrogate-Control:no-store");
357 // RFC 2616: HTTP/1.1 Cache-Control conditions
358 if (!ignoreCacheControl
) {
359 // XXX: check to see if the request headers alone were enough to prevent caching earlier
360 // (ie no-store request header) no need to check those all again here if so.
361 // for now we are not reliably doing that so we waste CPU re-checking request CC
363 // RFC 2616 section 14.9.2 - MUST NOT cache any response with request CC:no-store
364 if (request
&& request
->cache_control
&& request
->cache_control
->noStore() &&
365 !REFRESH_OVERRIDE(ignore_no_store
)) {
366 debugs(22, 3, HERE
<< "NO because client request Cache-Control:no-store");
370 // NP: request CC:no-cache only means cache READ is forbidden. STORE is permitted.
371 if (rep
->cache_control
&& rep
->cache_control
->hasNoCache() && rep
->cache_control
->noCache().size() > 0) {
372 /* TODO: we are allowed to cache when no-cache= has parameters.
373 * Provided we strip away any of the listed headers unless they are revalidated
374 * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
375 * That is a bit tricky for squid right now so we avoid caching entirely.
377 debugs(22, 3, HERE
<< "NO because server reply Cache-Control:no-cache has parameters");
381 // NP: request CC:private is undefined. We ignore.
382 // NP: other request CC flags are limiters on HIT/MISS. We don't care about here.
384 // RFC 2616 section 14.9.2 - MUST NOT cache any response with CC:no-store
385 if (rep
->cache_control
&& rep
->cache_control
->noStore() &&
386 !REFRESH_OVERRIDE(ignore_no_store
)) {
387 debugs(22, 3, HERE
<< "NO because server reply Cache-Control:no-store");
391 // RFC 2616 section 14.9.1 - MUST NOT cache any response with CC:private in a shared cache like Squid.
392 // CC:private overrides CC:public when both are present in a response.
393 // TODO: add a shared/private cache configuration possibility.
394 if (rep
->cache_control
&&
395 rep
->cache_control
->hasPrivate() &&
396 !REFRESH_OVERRIDE(ignore_private
)) {
397 /* TODO: we are allowed to cache when private= has parameters.
398 * Provided we strip away any of the listed headers unless they are revalidated
399 * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
400 * That is a bit tricky for squid right now so we avoid caching entirely.
402 debugs(22, 3, HERE
<< "NO because server reply Cache-Control:private");
407 // RFC 2068, sec 14.9.4 - MUST NOT cache any response with Authentication UNLESS certain CC controls are present
408 // allow HTTP violations to IGNORE those controls (ie re-block caching Auth)
409 if (request
&& (request
->flags
.auth
|| request
->flags
.authSent
)) {
410 if (!rep
->cache_control
) {
411 debugs(22, 3, HERE
<< "NO because Authenticated and server reply missing Cache-Control");
415 if (ignoreCacheControl
) {
416 debugs(22, 3, HERE
<< "NO because Authenticated and ignoring Cache-Control");
420 bool mayStore
= false;
421 // HTTPbis pt6 section 3.2: a response CC:public is present
422 if (rep
->cache_control
->Public()) {
423 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:public");
426 // HTTPbis pt6 section 3.2: a response CC:must-revalidate is present
427 } else if (rep
->cache_control
->mustRevalidate()) {
428 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:must-revalidate");
431 #if USE_HTTP_VIOLATIONS
432 // NP: given the must-revalidate exception we should also be able to exempt no-cache.
433 // HTTPbis WG verdict on this is that it is omitted from the spec due to being 'unexpected' by
434 // some. The caching+revalidate is not exactly unsafe though with Squids interpretation of no-cache
435 // (without parameters) as equivalent to must-revalidate in the reply.
436 } else if (rep
->cache_control
->hasNoCache() && rep
->cache_control
->noCache().size() == 0) {
437 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:no-cache (equivalent to must-revalidate)");
441 // HTTPbis pt6 section 3.2: a response CC:s-maxage is present
442 } else if (rep
->cache_control
->sMaxAge()) {
443 debugs(22, 3, HERE
<< "Authenticated but server reply Cache-Control:s-maxage");
448 debugs(22, 3, HERE
<< "NO because Authenticated transaction");
452 // NP: response CC:no-cache is equivalent to CC:must-revalidate,max-age=0. We MAY cache, and do so.
453 // NP: other request CC flags are limiters on HIT/MISS/REFRESH. We don't care about here.
456 /* HACK: The "multipart/x-mixed-replace" content type is used for
457 * continuous push replies. These are generally dynamic and
458 * probably should not be cachable
460 if ((v
= hdr
->getStr(Http::HdrType::CONTENT_TYPE
)))
461 if (!strncasecmp(v
, "multipart/x-mixed-replace", 25)) {
462 debugs(22, 3, HERE
<< "NO because Content-Type:multipart/x-mixed-replace");
466 switch (rep
->sline
.status()) {
467 /* Responses that are cacheable */
471 case Http::scNonAuthoritativeInformation
:
473 case Http::scMultipleChoices
:
475 case Http::scMovedPermanently
:
476 case Http::scPermanentRedirect
:
480 * Don't cache objects that need to be refreshed on next request,
481 * unless we know how to refresh it.
484 if (!refreshIsCachable(entry
) && !REFRESH_OVERRIDE(store_stale
)) {
485 debugs(22, 3, "NO because refreshIsCachable() returned non-cacheable..");
488 debugs(22, 3, HERE
<< "YES because HTTP status " << rep
->sline
.status());
494 /* Responses that only are cacheable if the server says so */
497 case Http::scTemporaryRedirect
:
498 if (rep
->date
<= 0) {
499 debugs(22, 3, HERE
<< "NO because HTTP status " << rep
->sline
.status() << " and Date missing/invalid");
502 if (rep
->expires
> rep
->date
) {
503 debugs(22, 3, HERE
<< "YES because HTTP status " << rep
->sline
.status() << " and Expires > Date");
506 debugs(22, 3, HERE
<< "NO because HTTP status " << rep
->sline
.status() << " and Expires <= Date");
512 /* Errors can be negatively cached */
514 case Http::scNoContent
:
516 case Http::scUseProxy
:
518 case Http::scBadRequest
:
520 case Http::scForbidden
:
522 case Http::scNotFound
:
524 case Http::scMethodNotAllowed
:
526 case Http::scUriTooLong
:
528 case Http::scInternalServerError
:
530 case Http::scNotImplemented
:
532 case Http::scBadGateway
:
534 case Http::scServiceUnavailable
:
536 case Http::scGatewayTimeout
:
537 case Http::scMisdirectedRequest
:
539 debugs(22, 3, "MAYBE because HTTP status " << rep
->sline
.status());
545 /* Some responses can never be cached */
547 case Http::scPartialContent
: /* Not yet supported */
549 case Http::scSeeOther
:
551 case Http::scNotModified
:
553 case Http::scUnauthorized
:
555 case Http::scProxyAuthenticationRequired
:
557 case Http::scInvalidHeader
: /* Squid header parsing error */
559 case Http::scHeaderTooLarge
:
561 case Http::scPaymentRequired
:
562 case Http::scNotAcceptable
:
563 case Http::scRequestTimeout
:
564 case Http::scConflict
:
565 case Http::scLengthRequired
:
566 case Http::scPreconditionFailed
:
567 case Http::scPayloadTooLarge
:
568 case Http::scUnsupportedMediaType
:
569 case Http::scUnprocessableEntity
:
571 case Http::scFailedDependency
:
572 case Http::scInsufficientStorage
:
573 case Http::scRequestedRangeNotSatisfied
:
574 case Http::scExpectationFailed
:
576 debugs(22, 3, HERE
<< "NO because HTTP status " << rep
->sline
.status());
580 /* RFC 2616 section 6.1.1: an unrecognized response MUST NOT be cached. */
581 debugs (11, 3, HERE
<< "NO because unknown HTTP status code " << rep
->sline
.status());
591 /// assemble a variant key (vary-mark) from the given Vary header and HTTP request
593 assembleVaryKey(String
&vary
, SBuf
&vstr
, const HttpRequest
&request
)
595 static const SBuf
asterisk("*");
596 const char *pos
= nullptr;
597 const char *item
= nullptr;
600 while (strListGetItem(&vary
, ',', &item
, &ilen
, &pos
)) {
601 SBuf
name(item
, ilen
);
602 if (name
== asterisk
) {
608 vstr
.append(", ", 2);
610 String
hdr(request
.header
.getByName(name
));
611 const char *value
= hdr
.termedBuf();
613 value
= rfc1738_escape_part(value
);
614 vstr
.append("=\"", 2);
616 vstr
.append("\"", 1);
624 * For Vary, store the relevant request headers as
625 * virtual headers in the reply
626 * Returns an empty SBuf if the variance cannot be stored
629 httpMakeVaryMark(HttpRequest
* request
, HttpReply
const * reply
)
634 vary
= reply
->header
.getList(Http::HdrType::VARY
);
635 assembleVaryKey(vary
, vstr
, *request
);
637 #if X_ACCELERATOR_VARY
639 vary
= reply
->header
.getList(Http::HdrType::HDR_X_ACCELERATOR_VARY
);
640 assembleVaryKey(vary
, vstr
, *request
);
648 HttpStateData::keepaliveAccounting(HttpReply
*reply
)
652 ++ _peer
->stats
.n_keepalives_sent
;
654 if (reply
->keep_alive
) {
656 ++ _peer
->stats
.n_keepalives_recv
;
658 if (Config
.onoff
.detect_broken_server_pconns
659 && reply
->bodySize(request
->method
) == -1 && !flags
.chunked
) {
660 debugs(11, DBG_IMPORTANT
, "keepaliveAccounting: Impossible keep-alive header from '" << entry
->url() << "'" );
661 // debugs(11, 2, "GOT HTTP REPLY HDR:\n---------\n" << readBuf->content() << "\n----------" );
662 flags
.keepalive_broken
= true;
668 HttpStateData::checkDateSkew(HttpReply
*reply
)
670 if (reply
->date
> -1 && !_peer
) {
671 int skew
= abs((int)(reply
->date
- squid_curtime
));
674 debugs(11, 3, "" << request
->url
.host() << "'s clock is skewed by " << skew
<< " seconds!");
679 * This creates the error page itself.. its likely
680 * that the forward ported reply header max size patch
681 * generates non http conformant error pages - in which
682 * case the errors where should be 'BAD_GATEWAY' etc
685 HttpStateData::processReplyHeader()
687 /** Creates a blank header. If this routine is made incremental, this will not do */
689 /* NP: all exit points to this function MUST call ctx_exit(ctx) */
690 Ctx ctx
= ctx_enter(entry
->mem_obj
->urlXXX());
692 debugs(11, 3, "processReplyHeader: key '" << entry
->getMD5Text() << "'");
694 assert(!flags
.headers_parsed
);
696 if (!inBuf
.length()) {
701 /* Attempt to parse the first line; this will define where the protocol, status, reason-phrase and header begin */
704 hp
= new Http1::ResponseParser
;
706 bool parsedOk
= hp
->parse(inBuf
);
708 // sync the buffers after parsing.
709 inBuf
= hp
->remaining();
711 if (hp
->needsMoreData()) {
712 if (eof
) { // no more data coming
713 /* Bug 2879: Replies may terminate with \r\n then EOF instead of \r\n\r\n.
714 * We also may receive truncated responses.
715 * Ensure here that we have at minimum two \r\n when EOF is seen.
717 inBuf
.append("\r\n\r\n", 4);
719 parsedOk
= hp
->parse(inBuf
);
720 // sync the buffers after parsing.
721 inBuf
= hp
->remaining();
723 debugs(33, 5, "Incomplete response, waiting for end of response headers");
730 // unrecoverable parsing error
731 // TODO: Use Raw! XXX: inBuf no longer has the [beginning of the] malformed header.
732 debugs(11, 3, "Non-HTTP-compliant header:\n---------\n" << inBuf
<< "\n----------");
733 flags
.headers_parsed
= true;
734 HttpReply
*newrep
= new HttpReply
;
735 newrep
->sline
.set(Http::ProtocolVersion(), hp
->parseStatusCode
);
736 setVirginReply(newrep
);
742 /* We know the whole response is in parser now */
743 debugs(11, 2, "HTTP Server " << serverConnection
);
744 debugs(11, 2, "HTTP Server RESPONSE:\n---------\n" <<
745 hp
->messageProtocol() << " " << hp
->messageStatus() << " " << hp
->reasonPhrase() << "\n" <<
749 // reset payload tracking to begin after message headers
750 payloadSeen
= inBuf
.length();
752 HttpReply
*newrep
= new HttpReply
;
753 // XXX: RFC 7230 indicates we MAY ignore the reason phrase,
754 // and use an empty string on unknown status.
755 // We do that now to avoid performance regression from using SBuf::c_str()
756 newrep
->sline
.set(Http::ProtocolVersion(1,1), hp
->messageStatus() /* , hp->reasonPhrase() */);
757 newrep
->sline
.protocol
= newrep
->sline
.version
.protocol
= hp
->messageProtocol().protocol
;
758 newrep
->sline
.version
.major
= hp
->messageProtocol().major
;
759 newrep
->sline
.version
.minor
= hp
->messageProtocol().minor
;
762 if (!newrep
->parseHeader(*hp
)) {
763 // XXX: when Http::ProtocolVersion is a function, remove this hack. just set with messageProtocol()
764 newrep
->sline
.set(Http::ProtocolVersion(), Http::scInvalidHeader
);
765 newrep
->sline
.version
.protocol
= hp
->messageProtocol().protocol
;
766 newrep
->sline
.version
.major
= hp
->messageProtocol().major
;
767 newrep
->sline
.version
.minor
= hp
->messageProtocol().minor
;
768 debugs(11, 2, "error parsing response headers mime block");
771 // done with Parser, now process using the HttpReply
774 newrep
->sources
|= request
->url
.getScheme() == AnyP::PROTO_HTTPS
? Http::Message::srcHttps
: Http::Message::srcHttp
;
776 newrep
->removeStaleWarnings();
778 if (newrep
->sline
.protocol
== AnyP::PROTO_HTTP
&& newrep
->sline
.status() >= 100 && newrep
->sline
.status() < 200) {
784 flags
.chunked
= false;
785 if (newrep
->sline
.protocol
== AnyP::PROTO_HTTP
&& newrep
->header
.chunked()) {
786 flags
.chunked
= true;
787 httpChunkDecoder
= new Http1::TeChunkedParser
;
790 if (!peerSupportsConnectionPinning())
791 request
->flags
.connectionAuthDisabled
= true;
793 HttpReply
*vrep
= setVirginReply(newrep
);
794 flags
.headers_parsed
= true;
796 keepaliveAccounting(vrep
);
800 processSurrogateControl (vrep
);
802 request
->hier
.peer_reply_status
= newrep
->sline
.status();
807 /// ignore or start forwarding the 1xx response (a.k.a., control message)
809 HttpStateData::handle1xx(HttpReply
*reply
)
811 HttpReply::Pointer
msg(reply
); // will destroy reply if unused
813 // one 1xx at a time: we must not be called while waiting for previous 1xx
814 Must(!flags
.handling1xx
);
815 flags
.handling1xx
= true;
817 if (!request
->canHandle1xx() || request
->forcedBodyContinuation
) {
818 debugs(11, 2, "ignoring 1xx because it is " << (request
->forcedBodyContinuation
? "already sent" : "not supported by client"));
823 #if USE_HTTP_VIOLATIONS
824 // check whether the 1xx response forwarding is allowed by squid.conf
825 if (Config
.accessList
.reply
) {
826 ACLFilledChecklist
ch(Config
.accessList
.reply
, originalRequest(), NULL
);
828 HTTPMSGLOCK(ch
.reply
);
829 if (ch
.fastCheck() != ACCESS_ALLOWED
) { // TODO: support slow lookups?
830 debugs(11, 3, HERE
<< "ignoring denied 1xx");
835 #endif // USE_HTTP_VIOLATIONS
837 debugs(11, 2, HERE
<< "forwarding 1xx to client");
839 // the Sink will use this to call us back after writing 1xx to the client
840 typedef NullaryMemFunT
<HttpStateData
> CbDialer
;
841 const AsyncCall::Pointer cb
= JobCallback(11, 3, CbDialer
, this,
842 HttpStateData::proceedAfter1xx
);
843 CallJobHere1(11, 4, request
->clientConnectionManager
, ConnStateData
,
844 ConnStateData::sendControlMsg
, HttpControlMsg(msg
, cb
));
845 // If the call is not fired, then the Sink is gone, and HttpStateData
846 // will terminate due to an aborted store entry or another similar error.
847 // If we get stuck, it is not handle1xx fault if we could get stuck
848 // for similar reasons without a 1xx response.
851 /// restores state and resumes processing after 1xx is ignored or forwarded
853 HttpStateData::proceedAfter1xx()
855 Must(flags
.handling1xx
);
856 debugs(11, 2, "continuing with " << payloadSeen
<< " bytes in buffer after 1xx");
857 CallJobHere(11, 3, this, HttpStateData
, HttpStateData::processReply
);
861 * returns true if the peer can support connection pinning
863 bool HttpStateData::peerSupportsConnectionPinning() const
865 const HttpReply
*rep
= entry
->mem_obj
->getReply();
866 const HttpHeader
*hdr
= &rep
->header
;
873 /*If this peer does not support connection pinning (authenticated
874 connections) return false
876 if (!_peer
->connection_auth
)
879 /*The peer supports connection pinning and the http reply status
880 is not unauthorized, so the related connection can be pinned
882 if (rep
->sline
.status() != Http::scUnauthorized
)
885 /*The server respond with Http::scUnauthorized and the peer configured
886 with "connection-auth=on" we know that the peer supports pinned
889 if (_peer
->connection_auth
== 1)
892 /*At this point peer has configured with "connection-auth=auto"
893 parameter so we need some extra checks to decide if we are going
894 to allow pinned connections or not
897 /*if the peer configured with originserver just allow connection
898 pinning (squid 2.6 behaviour)
900 if (_peer
->options
.originserver
)
903 /*if the connections it is already pinned it is OK*/
904 if (request
->flags
.pinned
)
907 /*Allow pinned connections only if the Proxy-support header exists in
908 reply and has in its list the "Session-Based-Authentication"
909 which means that the peer supports connection pinning.
911 if (!hdr
->has(Http::HdrType::PROXY_SUPPORT
))
914 header
= hdr
->getStrOrList(Http::HdrType::PROXY_SUPPORT
);
915 /* XXX This ought to be done in a case-insensitive manner */
916 rc
= (strstr(header
.termedBuf(), "Session-Based-Authentication") != NULL
);
921 // Called when we parsed (and possibly adapted) the headers but
922 // had not starting storing (a.k.a., sending) the body yet.
924 HttpStateData::haveParsedReplyHeaders()
926 Client::haveParsedReplyHeaders();
928 Ctx ctx
= ctx_enter(entry
->mem_obj
->urlXXX());
929 HttpReply
*rep
= finalReply();
931 entry
->timestampsSet();
933 /* Check if object is cacheable or not based on reply code */
934 debugs(11, 3, "HTTP CODE: " << rep
->sline
.status());
936 if (const StoreEntry
*oldEntry
= findPreviouslyCachedEntry(entry
))
937 sawDateGoBack
= rep
->olderThan(oldEntry
->getReply());
939 if (neighbors_do_private_keys
&& !sawDateGoBack
)
940 httpMaybeRemovePublic(entry
, rep
->sline
.status());
942 bool varyFailure
= false;
943 if (rep
->header
.has(Http::HdrType::VARY
)
944 #if X_ACCELERATOR_VARY
945 || rep
->header
.has(Http::HdrType::HDR_X_ACCELERATOR_VARY
)
948 const SBuf
vary(httpMakeVaryMark(request
, rep
));
950 if (vary
.isEmpty()) {
951 entry
->makePrivate();
952 if (!fwd
->reforwardableStatus(rep
->sline
.status()))
953 EBIT_CLR(entry
->flags
, ENTRY_FWD_HDR_WAIT
);
956 entry
->mem_obj
->vary_headers
= vary
;
958 // RFC 7231 section 7.1.4
959 // Vary:* can be cached, but has mandatory revalidation
960 static const SBuf
asterisk("*");
961 if (vary
== asterisk
)
962 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_ALWAYS
);
968 * If its not a reply that we will re-forward, then
969 * allow the client to get it.
971 if (!fwd
->reforwardableStatus(rep
->sline
.status()))
972 EBIT_CLR(entry
->flags
, ENTRY_FWD_HDR_WAIT
);
974 switch (cacheableReply()) {
981 entry
->makePrivate();
986 #if USE_HTTP_VIOLATIONS
987 if (Config
.negativeTtl
> 0)
988 entry
->cacheNegatively();
991 entry
->makePrivate();
1000 if (!ignoreCacheControl
) {
1001 if (rep
->cache_control
) {
1002 // We are required to revalidate on many conditions.
1003 // For security reasons we do so even if storage was caused by refresh_pattern ignore-* option
1005 // CC:must-revalidate or CC:proxy-revalidate
1006 const bool ccMustRevalidate
= (rep
->cache_control
->proxyRevalidate() || rep
->cache_control
->mustRevalidate());
1008 // CC:no-cache (only if there are no parameters)
1009 const bool ccNoCacheNoParams
= (rep
->cache_control
->hasNoCache() && rep
->cache_control
->noCache().size()==0);
1012 const bool ccSMaxAge
= rep
->cache_control
->hasSMaxAge();
1014 // CC:private (yes, these can sometimes be stored)
1015 const bool ccPrivate
= rep
->cache_control
->hasPrivate();
1017 if (ccNoCacheNoParams
|| ccPrivate
)
1018 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_ALWAYS
);
1019 else if (ccMustRevalidate
|| ccSMaxAge
)
1020 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_STALE
);
1022 #if USE_HTTP_VIOLATIONS // response header Pragma::no-cache is undefined in HTTP
1024 // Expensive calculation. So only do it IF the CC: header is not present.
1026 /* HACK: Pragma: no-cache in _replies_ is not documented in HTTP,
1027 * but servers like "Active Imaging Webcast/2.0" sure do use it */
1028 if (rep
->header
.has(Http::HdrType::PRAGMA
) &&
1029 rep
->header
.hasListMember(Http::HdrType::PRAGMA
,"no-cache",','))
1030 EBIT_SET(entry
->flags
, ENTRY_REVALIDATE_ALWAYS
);
1036 headersLog(1, 0, request
->method
, rep
);
1043 HttpStateData::ConnectionStatus
1044 HttpStateData::statusIfComplete() const
1046 const HttpReply
*rep
= virginReply();
1048 * If the reply wants to close the connection, it takes precedence */
1050 if (httpHeaderHasConnDir(&rep
->header
, "close"))
1051 return COMPLETE_NONPERSISTENT_MSG
;
1054 * If we didn't send a keep-alive request header, then this
1055 * can not be a persistent connection.
1057 if (!flags
.keepalive
)
1058 return COMPLETE_NONPERSISTENT_MSG
;
1061 * If we haven't sent the whole request then this can not be a persistent
1064 if (!flags
.request_sent
) {
1065 debugs(11, 2, "Request not yet fully sent " << request
->method
<< ' ' << entry
->url());
1066 return COMPLETE_NONPERSISTENT_MSG
;
1070 * What does the reply have to say about keep-alive?
1074 * If the origin server (HTTP/1.0) does not send a keep-alive
1075 * header, but keeps the connection open anyway, what happens?
1076 * We'll return here and http.c waits for an EOF before changing
1077 * store_status to STORE_OK. Combine this with ENTRY_FWD_HDR_WAIT
1078 * and an error status code, and we might have to wait until
1079 * the server times out the socket.
1081 if (!rep
->keep_alive
)
1082 return COMPLETE_NONPERSISTENT_MSG
;
1084 return COMPLETE_PERSISTENT_MSG
;
1087 HttpStateData::ConnectionStatus
1088 HttpStateData::persistentConnStatus() const
1090 debugs(11, 3, HERE
<< serverConnection
<< " eof=" << eof
);
1091 if (eof
) // already reached EOF
1092 return COMPLETE_NONPERSISTENT_MSG
;
1094 /* If server fd is closing (but we have not been notified yet), stop Comm
1095 I/O to avoid assertions. TODO: Change Comm API to handle callers that
1096 want more I/O after async closing (usually initiated by others). */
1097 // XXX: add canReceive or s/canSend/canTalkToServer/
1098 if (!Comm::IsConnOpen(serverConnection
))
1099 return COMPLETE_NONPERSISTENT_MSG
;
1102 * In chunked response we do not know the content length but we are absolutely
1103 * sure about the end of response, so we are calling the statusIfComplete to
1104 * decide if we can be persistant
1106 if (lastChunk
&& flags
.chunked
)
1107 return statusIfComplete();
1109 const HttpReply
*vrep
= virginReply();
1110 debugs(11, 5, "persistentConnStatus: content_length=" << vrep
->content_length
);
1112 const int64_t clen
= vrep
->bodySize(request
->method
);
1114 debugs(11, 5, "persistentConnStatus: clen=" << clen
);
1116 /* If the body size is unknown we must wait for EOF */
1118 return INCOMPLETE_MSG
;
1121 * If the body size is known, we must wait until we've gotten all of it. */
1123 debugs(11,5, "payloadSeen=" << payloadSeen
<< " content_length=" << vrep
->content_length
);
1125 if (payloadSeen
< vrep
->content_length
)
1126 return INCOMPLETE_MSG
;
1128 if (payloadTruncated
> 0) // already read more than needed
1129 return COMPLETE_NONPERSISTENT_MSG
; // disable pconns
1133 * If there is no message body or we got it all, we can be persistent */
1134 return statusIfComplete();
1139 readDelayed(void *context
, CommRead
const &)
1141 HttpStateData
*state
= static_cast<HttpStateData
*>(context
);
1142 state
->flags
.do_next_read
= true;
1143 state
->maybeReadVirginBody();
1148 HttpStateData::readReply(const CommIoCbParams
&io
)
1150 Must(!flags
.do_next_read
); // XXX: should have been set false by mayReadVirginBody()
1151 flags
.do_next_read
= false;
1153 debugs(11, 5, io
.conn
);
1155 // Bail out early on Comm::ERR_CLOSING - close handlers will tidy up for us
1156 if (io
.flag
== Comm::ERR_CLOSING
) {
1157 debugs(11, 3, "http socket closing");
1161 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
1162 abortTransaction("store entry aborted while reading reply");
1166 Must(Comm::IsConnOpen(serverConnection
));
1167 Must(io
.conn
->fd
== serverConnection
->fd
);
1170 * Don't reset the timeout value here. The value should be
1171 * counting Config.Timeout.request and applies to the request
1172 * as a whole, not individual read() calls.
1173 * Plus, it breaks our lame *HalfClosed() detection
1176 Must(maybeMakeSpaceAvailable(true));
1177 CommIoCbParams
rd(this); // will be expanded with ReadNow results
1179 rd
.size
= entry
->bytesWanted(Range
<size_t>(0, inBuf
.spaceSize()));
1182 assert(entry
->mem_obj
);
1184 /* read ahead limit */
1185 /* Perhaps these two calls should both live in MemObject */
1186 AsyncCall::Pointer nilCall
;
1187 if (!entry
->mem_obj
->readAheadPolicyCanRead()) {
1188 entry
->mem_obj
->delayRead(DeferredRead(readDelayed
, this, CommRead(io
.conn
, NULL
, 0, nilCall
)));
1192 /* delay id limit */
1193 entry
->mem_obj
->mostBytesAllowed().delayRead(DeferredRead(readDelayed
, this, CommRead(io
.conn
, NULL
, 0, nilCall
)));
1198 switch (Comm::ReadNow(rd
, inBuf
)) {
1199 case Comm::INPROGRESS
:
1200 if (inBuf
.isEmpty())
1201 debugs(33, 2, io
.conn
<< ": no data to process, " << xstrerr(rd
.xerrno
));
1202 flags
.do_next_read
= true;
1203 maybeReadVirginBody();
1208 payloadSeen
+= rd
.size
;
1210 DelayId delayId
= entry
->mem_obj
->mostBytesAllowed();
1211 delayId
.bytesIn(rd
.size
);
1214 statCounter
.server
.all
.kbytes_in
+= rd
.size
;
1215 statCounter
.server
.http
.kbytes_in
+= rd
.size
;
1216 ++ IOStats
.Http
.reads
;
1219 for (int clen
= rd
.size
- 1; clen
; ++bin
)
1222 ++ IOStats
.Http
.read_hist
[bin
];
1224 // update peer response time stats (%<pt)
1225 const timeval
&sent
= request
->hier
.peer_http_request_sent
;
1227 tvSub(request
->hier
.peer_response_time
, sent
, current_time
);
1229 request
->hier
.peer_response_time
.tv_sec
= -1;
1232 /* Continue to process previously read data */
1235 case Comm::ENDFILE
: // close detected by 0-byte read
1237 flags
.do_next_read
= false;
1239 /* Continue to process previously read data */
1242 // case Comm::COMM_ERROR:
1243 default: // no other flags should ever occur
1244 debugs(11, 2, io
.conn
<< ": read failure: " << xstrerr(rd
.xerrno
));
1245 ErrorState
*err
= new ErrorState(ERR_READ_ERROR
, Http::scBadGateway
, fwd
->request
);
1246 err
->xerrno
= rd
.xerrno
;
1248 flags
.do_next_read
= false;
1250 mustStop("HttpStateData::readReply");
1254 /* Process next response from buffer */
1258 /// processes the already read and buffered response data, possibly after
1259 /// waiting for asynchronous 1xx control message processing
1261 HttpStateData::processReply()
1264 if (flags
.handling1xx
) { // we came back after handling a 1xx response
1265 debugs(11, 5, HERE
<< "done with 1xx handling");
1266 flags
.handling1xx
= false;
1267 Must(!flags
.headers_parsed
);
1270 if (!flags
.headers_parsed
) { // have not parsed headers yet?
1271 PROF_start(HttpStateData_processReplyHeader
);
1272 processReplyHeader();
1273 PROF_stop(HttpStateData_processReplyHeader
);
1275 if (!continueAfterParsingHeader()) // parsing error or need more data
1276 return; // TODO: send errors to ICAP
1278 adaptOrFinalizeReply(); // may write to, abort, or "close" the entry
1281 // kick more reads if needed and/or process the response body, if any
1282 PROF_start(HttpStateData_processReplyBody
);
1283 processReplyBody(); // may call serverComplete()
1284 PROF_stop(HttpStateData_processReplyBody
);
1288 \retval true if we can continue with processing the body or doing ICAP.
1291 HttpStateData::continueAfterParsingHeader()
1293 if (flags
.handling1xx
) {
1294 debugs(11, 5, HERE
<< "wait for 1xx handling");
1295 Must(!flags
.headers_parsed
);
1299 if (!flags
.headers_parsed
&& !eof
) {
1300 debugs(11, 9, "needs more at " << inBuf
.length());
1301 flags
.do_next_read
= true;
1302 /** \retval false If we have not finished parsing the headers and may get more data.
1303 * Schedules more reads to retrieve the missing data.
1305 maybeReadVirginBody(); // schedules all kinds of reads; TODO: rename
1309 /** If we are done with parsing, check for errors */
1311 err_type error
= ERR_NONE
;
1313 if (flags
.headers_parsed
) { // parsed headers, possibly with errors
1314 // check for header parsing errors
1315 if (HttpReply
*vrep
= virginReply()) {
1316 const Http::StatusCode s
= vrep
->sline
.status();
1317 const AnyP::ProtocolVersion
&v
= vrep
->sline
.version
;
1318 if (s
== Http::scInvalidHeader
&& v
!= Http::ProtocolVersion(0,9)) {
1319 debugs(11, DBG_IMPORTANT
, "WARNING: HTTP: Invalid Response: Bad header encountered from " << entry
->url() << " AKA " << request
->url
);
1320 error
= ERR_INVALID_RESP
;
1321 } else if (s
== Http::scHeaderTooLarge
) {
1322 fwd
->dontRetry(true);
1323 error
= ERR_TOO_BIG
;
1324 } else if (vrep
->header
.conflictingContentLength()) {
1325 fwd
->dontRetry(true);
1326 error
= ERR_INVALID_RESP
;
1328 return true; // done parsing, got reply, and no error
1331 // parsed headers but got no reply
1332 debugs(11, DBG_IMPORTANT
, "WARNING: HTTP: Invalid Response: No reply at all for " << entry
->url() << " AKA " << request
->url
);
1333 error
= ERR_INVALID_RESP
;
1337 if (inBuf
.length()) {
1338 error
= ERR_INVALID_RESP
;
1339 debugs(11, DBG_IMPORTANT
, "WARNING: HTTP: Invalid Response: Headers did not parse at all for " << entry
->url() << " AKA " << request
->url
);
1341 error
= ERR_ZERO_SIZE_OBJECT
;
1342 debugs(11, (request
->flags
.accelerated
?DBG_IMPORTANT
:2), "WARNING: HTTP: Invalid Response: No object data received for " << entry
->url() << " AKA " << request
->url
);
1346 assert(error
!= ERR_NONE
);
1348 fwd
->fail(new ErrorState(error
, Http::scBadGateway
, fwd
->request
));
1349 flags
.do_next_read
= false;
1351 mustStop("HttpStateData::continueAfterParsingHeader");
1352 return false; // quit on error
1355 /** truncate what we read if we read too much so that writeReplyBody()
1356 writes no more than what we should have read */
1358 HttpStateData::truncateVirginBody()
1360 assert(flags
.headers_parsed
);
1362 HttpReply
*vrep
= virginReply();
1364 if (!vrep
->expectingBody(request
->method
, clen
) || clen
< 0)
1365 return; // no body or a body of unknown size, including chunked
1367 if (payloadSeen
- payloadTruncated
<= clen
)
1368 return; // we did not read too much or already took care of the extras
1370 if (const int64_t extras
= payloadSeen
- payloadTruncated
- clen
) {
1371 // server sent more that the advertised content length
1372 debugs(11, 5, "payloadSeen=" << payloadSeen
<<
1373 " clen=" << clen
<< '/' << vrep
->content_length
<<
1374 " trucated=" << payloadTruncated
<< '+' << extras
);
1376 inBuf
.chop(0, inBuf
.length() - extras
);
1377 payloadTruncated
+= extras
;
1382 * Call this when there is data from the origin server
1383 * which should be sent to either StoreEntry, or to ICAP...
1386 HttpStateData::writeReplyBody()
1388 truncateVirginBody(); // if needed
1389 const char *data
= inBuf
.rawContent();
1390 int len
= inBuf
.length();
1391 addVirginReplyBody(data
, len
);
1396 HttpStateData::decodeAndWriteReplyBody()
1398 const char *data
= NULL
;
1400 bool wasThereAnException
= false;
1401 assert(flags
.chunked
);
1402 assert(httpChunkDecoder
);
1403 SQUID_ENTER_THROWING_CODE();
1406 httpChunkDecoder
->setPayloadBuffer(&decodedData
);
1407 const bool doneParsing
= httpChunkDecoder
->parse(inBuf
);
1408 inBuf
= httpChunkDecoder
->remaining(); // sync buffers after parse
1409 len
= decodedData
.contentSize();
1410 data
=decodedData
.content();
1411 addVirginReplyBody(data
, len
);
1414 flags
.do_next_read
= false;
1416 SQUID_EXIT_THROWING_CODE(wasThereAnException
);
1417 return wasThereAnException
;
1421 * processReplyBody has two purposes:
1422 * 1 - take the reply body data, if any, and put it into either
1423 * the StoreEntry, or give it over to ICAP.
1424 * 2 - see if we made it to the end of the response (persistent
1425 * connections and such)
1428 HttpStateData::processReplyBody()
1430 Ip::Address client_addr
;
1431 bool ispinned
= false;
1433 if (!flags
.headers_parsed
) {
1434 flags
.do_next_read
= true;
1435 maybeReadVirginBody();
1440 debugs(11,5, HERE
<< "adaptationAccessCheckPending=" << adaptationAccessCheckPending
);
1441 if (adaptationAccessCheckPending
)
1447 * At this point the reply headers have been parsed and consumed.
1448 * That means header content has been removed from readBuf and
1449 * it contains only body data.
1451 if (entry
->isAccepting()) {
1452 if (flags
.chunked
) {
1453 if (!decodeAndWriteReplyBody()) {
1454 flags
.do_next_read
= false;
1462 // storing/sending methods like earlier adaptOrFinalizeReply() or
1463 // above writeReplyBody() may release/abort the store entry.
1464 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
1465 // TODO: In some cases (e.g., 304), we should keep persistent conn open.
1466 // Detect end-of-reply (and, hence, pool our idle pconn) earlier (ASAP).
1467 abortTransaction("store entry aborted while storing reply");
1470 switch (persistentConnStatus()) {
1471 case INCOMPLETE_MSG
: {
1472 debugs(11, 5, "processReplyBody: INCOMPLETE_MSG from " << serverConnection
);
1473 /* Wait for more data or EOF condition */
1474 AsyncCall::Pointer nil
;
1475 if (flags
.keepalive_broken
) {
1476 commSetConnTimeout(serverConnection
, 10, nil
);
1478 commSetConnTimeout(serverConnection
, Config
.Timeout
.read
, nil
);
1481 flags
.do_next_read
= true;
1485 case COMPLETE_PERSISTENT_MSG
:
1486 debugs(11, 5, "processReplyBody: COMPLETE_PERSISTENT_MSG from " << serverConnection
);
1487 /* yes we have to clear all these! */
1488 commUnsetConnTimeout(serverConnection
);
1489 flags
.do_next_read
= false;
1491 comm_remove_close_handler(serverConnection
->fd
, closeHandler
);
1492 closeHandler
= NULL
;
1493 fwd
->unregister(serverConnection
);
1495 if (request
->flags
.spoofClientIp
)
1496 client_addr
= request
->client_addr
;
1498 if (request
->flags
.pinned
) {
1500 } else if (request
->flags
.connectionAuth
&& request
->flags
.authSent
) {
1504 if (ispinned
&& request
->clientConnectionManager
.valid()) {
1505 request
->clientConnectionManager
->pinConnection(serverConnection
, request
, _peer
,
1506 (request
->flags
.connectionAuth
));
1508 fwd
->pconnPush(serverConnection
, request
->url
.host());
1511 serverConnection
= NULL
;
1515 case COMPLETE_NONPERSISTENT_MSG
:
1516 debugs(11, 5, "processReplyBody: COMPLETE_NONPERSISTENT_MSG from " << serverConnection
);
1521 maybeReadVirginBody();
1525 HttpStateData::mayReadVirginReplyBody() const
1527 // TODO: Be more precise here. For example, if/when reading trailer, we may
1528 // not be doneWithServer() yet, but we should return false. Similarly, we
1529 // could still be writing the request body after receiving the whole reply.
1530 return !doneWithServer();
1534 HttpStateData::maybeReadVirginBody()
1537 if (!Comm::IsConnOpen(serverConnection
) || fd_table
[serverConnection
->fd
].closing())
1540 if (!maybeMakeSpaceAvailable(false))
1543 // XXX: get rid of the do_next_read flag
1544 // check for the proper reasons preventing read(2)
1545 if (!flags
.do_next_read
)
1548 flags
.do_next_read
= false;
1550 // must not already be waiting for read(2) ...
1551 assert(!Comm::MonitorsRead(serverConnection
->fd
));
1553 // wait for read(2) to be possible.
1554 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
1555 AsyncCall::Pointer call
= JobCallback(11, 5, Dialer
, this, HttpStateData::readReply
);
1556 Comm::Read(serverConnection
, call
);
1560 HttpStateData::maybeMakeSpaceAvailable(bool doGrow
)
1562 // how much we are allowed to buffer
1563 const int limitBuffer
= (flags
.headers_parsed
? Config
.readAheadGap
: Config
.maxReplyHeaderSize
);
1565 if (limitBuffer
< 0 || inBuf
.length() >= (SBuf::size_type
)limitBuffer
) {
1566 // when buffer is at or over limit already
1567 debugs(11, 7, "wont read up to " << limitBuffer
<< ". buffer has (" << inBuf
.length() << "/" << inBuf
.spaceSize() << ") from " << serverConnection
);
1568 debugs(11, DBG_DATA
, "buffer has {" << inBuf
<< "}");
1569 // Process next response from buffer
1574 // how much we want to read
1575 const size_t read_size
= calcBufferSpaceToReserve(inBuf
.spaceSize(), (limitBuffer
- inBuf
.length()));
1578 debugs(11, 7, "wont read up to " << read_size
<< " into buffer (" << inBuf
.length() << "/" << inBuf
.spaceSize() << ") from " << serverConnection
);
1582 // just report whether we could grow or not, dont actually do it
1584 return (read_size
>= 2);
1586 // we may need to grow the buffer
1587 inBuf
.reserveSpace(read_size
);
1588 debugs(11, 8, (!flags
.do_next_read
? "wont" : "may") <<
1589 " read up to " << read_size
<< " bytes info buf(" << inBuf
.length() << "/" << inBuf
.spaceSize() <<
1590 ") from " << serverConnection
);
1592 return (inBuf
.spaceSize() >= 2); // only read if there is 1+ bytes of space available
1595 /// called after writing the very last request byte (body, last-chunk, etc)
1597 HttpStateData::wroteLast(const CommIoCbParams
&io
)
1599 debugs(11, 5, HERE
<< serverConnection
<< ": size " << io
.size
<< ": errflag " << io
.flag
<< ".");
1600 #if URL_CHECKSUM_DEBUG
1602 entry
->mem_obj
->checkUrlChecksum();
1606 fd_bytes(io
.fd
, io
.size
, FD_WRITE
);
1607 statCounter
.server
.all
.kbytes_out
+= io
.size
;
1608 statCounter
.server
.http
.kbytes_out
+= io
.size
;
1611 if (io
.flag
== Comm::ERR_CLOSING
)
1615 ErrorState
*err
= new ErrorState(ERR_WRITE_ERROR
, Http::scBadGateway
, fwd
->request
);
1616 err
->xerrno
= io
.xerrno
;
1619 mustStop("HttpStateData::wroteLast");
1626 /// successfully wrote the entire request (including body, last-chunk, etc.)
1628 HttpStateData::sendComplete()
1631 * Set the read timeout here because it hasn't been set yet.
1632 * We only set the read timeout after the request has been
1633 * fully written to the peer. If we start the timeout
1634 * after connection establishment, then we are likely to hit
1635 * the timeout for POST/PUT requests that have very large
1638 typedef CommCbMemFunT
<HttpStateData
, CommTimeoutCbParams
> TimeoutDialer
;
1639 AsyncCall::Pointer timeoutCall
= JobCallback(11, 5,
1640 TimeoutDialer
, this, HttpStateData::httpTimeout
);
1642 commSetConnTimeout(serverConnection
, Config
.Timeout
.read
, timeoutCall
);
1643 flags
.request_sent
= true;
1644 request
->hier
.peer_http_request_sent
= current_time
;
1648 HttpStateData::closeServer()
1650 debugs(11,5, HERE
<< "closing HTTP server " << serverConnection
<< " this " << this);
1652 if (Comm::IsConnOpen(serverConnection
)) {
1653 fwd
->unregister(serverConnection
);
1654 comm_remove_close_handler(serverConnection
->fd
, closeHandler
);
1655 closeHandler
= NULL
;
1656 serverConnection
->close();
1661 HttpStateData::doneWithServer() const
1663 return !Comm::IsConnOpen(serverConnection
);
1667 * Fixup authentication request headers for special cases
1670 httpFixupAuthentication(HttpRequest
* request
, const HttpHeader
* hdr_in
, HttpHeader
* hdr_out
, const Http::StateFlags
&flags
)
1672 Http::HdrType header
= flags
.originpeer
? Http::HdrType::AUTHORIZATION
: Http::HdrType::PROXY_AUTHORIZATION
;
1674 /* Nothing to do unless we are forwarding to a peer */
1675 if (!request
->flags
.proxying
)
1678 /* Needs to be explicitly enabled */
1679 if (!request
->peer_login
)
1682 /* Maybe already dealt with? */
1683 if (hdr_out
->has(header
))
1686 /* Nothing to do here for PASSTHRU */
1687 if (strcmp(request
->peer_login
, "PASSTHRU") == 0)
1690 /* PROXYPASS is a special case, single-signon to servers with the proxy password (basic only) */
1691 if (flags
.originpeer
&& strcmp(request
->peer_login
, "PROXYPASS") == 0 && hdr_in
->has(Http::HdrType::PROXY_AUTHORIZATION
)) {
1692 const char *auth
= hdr_in
->getStr(Http::HdrType::PROXY_AUTHORIZATION
);
1694 if (auth
&& strncasecmp(auth
, "basic ", 6) == 0) {
1695 hdr_out
->putStr(header
, auth
);
1700 uint8_t loginbuf
[base64_encode_len(MAX_LOGIN_SZ
)];
1702 struct base64_encode_ctx ctx
;
1703 base64_encode_init(&ctx
);
1705 /* Special mode to pass the username to the upstream cache */
1706 if (*request
->peer_login
== '*') {
1707 const char *username
= "-";
1709 if (request
->extacl_user
.size())
1710 username
= request
->extacl_user
.termedBuf();
1712 else if (request
->auth_user_request
!= NULL
)
1713 username
= request
->auth_user_request
->username();
1716 blen
= base64_encode_update(&ctx
, loginbuf
, strlen(username
), reinterpret_cast<const uint8_t*>(username
));
1717 blen
+= base64_encode_update(&ctx
, loginbuf
+blen
, strlen(request
->peer_login
+1), reinterpret_cast<const uint8_t*>(request
->peer_login
+1));
1718 blen
+= base64_encode_final(&ctx
, loginbuf
+blen
);
1719 httpHeaderPutStrf(hdr_out
, header
, "Basic %.*s", (int)blen
, loginbuf
);
1723 /* external_acl provided credentials */
1724 if (request
->extacl_user
.size() && request
->extacl_passwd
.size() &&
1725 (strcmp(request
->peer_login
, "PASS") == 0 ||
1726 strcmp(request
->peer_login
, "PROXYPASS") == 0)) {
1728 blen
= base64_encode_update(&ctx
, loginbuf
, request
->extacl_user
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_user
.rawBuf()));
1729 blen
+= base64_encode_update(&ctx
, loginbuf
+blen
, 1, reinterpret_cast<const uint8_t*>(":"));
1730 blen
+= base64_encode_update(&ctx
, loginbuf
+blen
, request
->extacl_passwd
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_passwd
.rawBuf()));
1731 blen
+= base64_encode_final(&ctx
, loginbuf
+blen
);
1732 httpHeaderPutStrf(hdr_out
, header
, "Basic %.*s", (int)blen
, loginbuf
);
1735 // if no external user credentials are available to fake authentication with PASS acts like PASSTHRU
1736 if (strcmp(request
->peer_login
, "PASS") == 0)
1739 /* Kerberos login to peer */
1740 #if HAVE_AUTH_MODULE_NEGOTIATE && HAVE_KRB5 && HAVE_GSSAPI
1741 if (strncmp(request
->peer_login
, "NEGOTIATE",strlen("NEGOTIATE")) == 0) {
1743 char *PrincipalName
=NULL
,*p
;
1744 int negotiate_flags
= 0;
1746 if ((p
=strchr(request
->peer_login
,':')) != NULL
) {
1749 if (request
->flags
.auth_no_keytab
) {
1750 negotiate_flags
|= PEER_PROXY_NEGOTIATE_NOKEYTAB
;
1752 Token
= peer_proxy_negotiate_auth(PrincipalName
, request
->peer_host
, negotiate_flags
);
1754 httpHeaderPutStrf(hdr_out
, header
, "Negotiate %s",Token
);
1758 #endif /* HAVE_KRB5 && HAVE_GSSAPI */
1760 blen
= base64_encode_update(&ctx
, loginbuf
, strlen(request
->peer_login
), reinterpret_cast<const uint8_t*>(request
->peer_login
));
1761 blen
+= base64_encode_final(&ctx
, loginbuf
+blen
);
1762 httpHeaderPutStrf(hdr_out
, header
, "Basic %.*s", (int)blen
, loginbuf
);
1767 * build request headers and append them to a given MemBuf
1768 * used by buildRequestPrefix()
1769 * note: initialised the HttpHeader, the caller is responsible for Clean()-ing
1772 HttpStateData::httpBuildRequestHeader(HttpRequest
* request
,
1774 const AccessLogEntryPointer
&al
,
1775 HttpHeader
* hdr_out
,
1776 const Http::StateFlags
&flags
)
1778 /* building buffer for complex strings */
1779 #define BBUF_SZ (MAX_URL+32)
1780 LOCAL_ARRAY(char, bbuf
, BBUF_SZ
);
1781 LOCAL_ARRAY(char, ntoabuf
, MAX_IPSTRLEN
);
1782 const HttpHeader
*hdr_in
= &request
->header
;
1783 const HttpHeaderEntry
*e
= NULL
;
1784 HttpHeaderPos pos
= HttpHeaderInitPos
;
1785 assert (hdr_out
->owner
== hoRequest
);
1787 /* use our IMS header if the cached entry has Last-Modified time */
1788 if (request
->lastmod
> -1)
1789 hdr_out
->putTime(Http::HdrType::IF_MODIFIED_SINCE
, request
->lastmod
);
1791 // Add our own If-None-Match field if the cached entry has a strong ETag.
1792 // copyOneHeaderFromClientsideRequestToUpstreamRequest() adds client ones.
1793 if (request
->etag
.size() > 0) {
1794 hdr_out
->addEntry(new HttpHeaderEntry(Http::HdrType::IF_NONE_MATCH
, NULL
,
1795 request
->etag
.termedBuf()));
1798 bool we_do_ranges
= decideIfWeDoRanges (request
);
1800 String
strConnection (hdr_in
->getList(Http::HdrType::CONNECTION
));
1802 while ((e
= hdr_in
->getEntry(&pos
)))
1803 copyOneHeaderFromClientsideRequestToUpstreamRequest(e
, strConnection
, request
, hdr_out
, we_do_ranges
, flags
);
1805 /* Abstraction break: We should interpret multipart/byterange responses
1806 * into offset-length data, and this works around our inability to do so.
1808 if (!we_do_ranges
&& request
->multipartRangeRequest()) {
1809 /* don't cache the result */
1810 request
->flags
.cachable
= false;
1811 /* pretend it's not a range request */
1812 request
->ignoreRange("want to request the whole object");
1813 request
->flags
.isRanged
= false;
1817 if (Config
.onoff
.via
) {
1819 strVia
= hdr_in
->getList(Http::HdrType::VIA
);
1820 snprintf(bbuf
, BBUF_SZ
, "%d.%d %s",
1821 request
->http_ver
.major
,
1822 request
->http_ver
.minor
, ThisCache
);
1823 strListAdd(&strVia
, bbuf
, ',');
1824 hdr_out
->putStr(Http::HdrType::VIA
, strVia
.termedBuf());
1828 if (request
->flags
.accelerated
) {
1829 /* Append Surrogate-Capabilities */
1830 String
strSurrogate(hdr_in
->getList(Http::HdrType::SURROGATE_CAPABILITY
));
1832 snprintf(bbuf
, BBUF_SZ
, "%s=\"Surrogate/1.0 ESI/1.0\"", Config
.Accel
.surrogate_id
);
1834 snprintf(bbuf
, BBUF_SZ
, "%s=\"Surrogate/1.0\"", Config
.Accel
.surrogate_id
);
1836 strListAdd(&strSurrogate
, bbuf
, ',');
1837 hdr_out
->putStr(Http::HdrType::SURROGATE_CAPABILITY
, strSurrogate
.termedBuf());
1840 /** \pre Handle X-Forwarded-For */
1841 if (strcmp(opt_forwarded_for
, "delete") != 0) {
1843 String strFwd
= hdr_in
->getList(Http::HdrType::X_FORWARDED_FOR
);
1845 // if we cannot double strFwd size, then it grew past 50% of the limit
1846 if (!strFwd
.canGrowBy(strFwd
.size())) {
1847 // There is probably a forwarding loop with Via detection disabled.
1848 // If we do nothing, String will assert on overflow soon.
1849 // TODO: Terminate all transactions with huge XFF?
1852 static int warnedCount
= 0;
1853 if (warnedCount
++ < 100) {
1854 const SBuf
url(entry
? SBuf(entry
->url()) : request
->effectiveRequestUri());
1855 debugs(11, DBG_IMPORTANT
, "Warning: likely forwarding loop with " << url
);
1859 if (strcmp(opt_forwarded_for
, "on") == 0) {
1860 /** If set to ON - append client IP or 'unknown'. */
1861 if ( request
->client_addr
.isNoAddr() )
1862 strListAdd(&strFwd
, "unknown", ',');
1864 strListAdd(&strFwd
, request
->client_addr
.toStr(ntoabuf
, MAX_IPSTRLEN
), ',');
1865 } else if (strcmp(opt_forwarded_for
, "off") == 0) {
1866 /** If set to OFF - append 'unknown'. */
1867 strListAdd(&strFwd
, "unknown", ',');
1868 } else if (strcmp(opt_forwarded_for
, "transparent") == 0) {
1869 /** If set to TRANSPARENT - pass through unchanged. */
1870 } else if (strcmp(opt_forwarded_for
, "truncate") == 0) {
1871 /** If set to TRUNCATE - drop existing list and replace with client IP or 'unknown'. */
1872 if ( request
->client_addr
.isNoAddr() )
1875 strFwd
= request
->client_addr
.toStr(ntoabuf
, MAX_IPSTRLEN
);
1877 if (strFwd
.size() > 0)
1878 hdr_out
->putStr(Http::HdrType::X_FORWARDED_FOR
, strFwd
.termedBuf());
1880 /** If set to DELETE - do not copy through. */
1882 /* append Host if not there already */
1883 if (!hdr_out
->has(Http::HdrType::HOST
)) {
1884 if (request
->peer_domain
) {
1885 hdr_out
->putStr(Http::HdrType::HOST
, request
->peer_domain
);
1887 SBuf authority
= request
->url
.authority();
1888 hdr_out
->putStr(Http::HdrType::HOST
, authority
.c_str());
1892 /* append Authorization if known in URL, not in header and going direct */
1893 if (!hdr_out
->has(Http::HdrType::AUTHORIZATION
)) {
1894 if (!request
->flags
.proxying
&& !request
->url
.userInfo().isEmpty()) {
1895 static uint8_t result
[base64_encode_len(MAX_URL
*2)]; // should be big enough for a single URI segment
1896 struct base64_encode_ctx ctx
;
1897 base64_encode_init(&ctx
);
1898 size_t blen
= base64_encode_update(&ctx
, result
, request
->url
.userInfo().length(), reinterpret_cast<const uint8_t*>(request
->url
.userInfo().rawContent()));
1899 blen
+= base64_encode_final(&ctx
, result
+blen
);
1900 result
[blen
] = '\0';
1902 httpHeaderPutStrf(hdr_out
, Http::HdrType::AUTHORIZATION
, "Basic %.*s", (int)blen
, result
);
1906 /* Fixup (Proxy-)Authorization special cases. Plain relaying dealt with above */
1907 httpFixupAuthentication(request
, hdr_in
, hdr_out
, flags
);
1909 /* append Cache-Control, add max-age if not there already */
1911 HttpHdrCc
*cc
= hdr_in
->getCc();
1914 cc
= new HttpHdrCc();
1916 #if 0 /* see bug 2330 */
1917 /* Set no-cache if determined needed but not found */
1918 if (request
->flags
.nocache
)
1919 EBIT_SET(cc
->mask
, HttpHdrCcType::CC_NO_CACHE
);
1922 /* Add max-age only without no-cache */
1923 if (!cc
->hasMaxAge() && !cc
->hasNoCache()) {
1924 // XXX: performance regression. c_str() reallocates
1925 SBuf
tmp(request
->effectiveRequestUri());
1926 cc
->maxAge(getMaxAge(entry
? entry
->url() : tmp
.c_str()));
1929 /* Enforce sibling relations */
1930 if (flags
.only_if_cached
)
1931 cc
->onlyIfCached(true);
1938 /* maybe append Connection: keep-alive */
1939 if (flags
.keepalive
) {
1940 hdr_out
->putStr(Http::HdrType::CONNECTION
, "keep-alive");
1943 /* append Front-End-Https */
1944 if (flags
.front_end_https
) {
1945 if (flags
.front_end_https
== 1 || request
->url
.getScheme() == AnyP::PROTO_HTTPS
)
1946 hdr_out
->putStr(Http::HdrType::FRONT_END_HTTPS
, "On");
1949 if (flags
.chunked_request
) {
1950 // Do not just copy the original value so that if the client-side
1951 // starts decode other encodings, this code may remain valid.
1952 hdr_out
->putStr(Http::HdrType::TRANSFER_ENCODING
, "chunked");
1955 /* Now mangle the headers. */
1956 httpHdrMangleList(hdr_out
, request
, al
, ROR_REQUEST
);
1958 strConnection
.clean();
1962 * Decides whether a particular header may be cloned from the received Clients request
1963 * to our outgoing fetch request.
1966 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry
*e
, const String strConnection
, const HttpRequest
* request
, HttpHeader
* hdr_out
, const int we_do_ranges
, const Http::StateFlags
&flags
)
1968 debugs(11, 5, "httpBuildRequestHeader: " << e
->name
<< ": " << e
->value
);
1972 /** \par RFC 2616 sect 13.5.1 - Hop-by-Hop headers which Squid should not pass on. */
1974 case Http::HdrType::PROXY_AUTHORIZATION
:
1975 /** \par Proxy-Authorization:
1976 * Only pass on proxy authentication to peers for which
1977 * authentication forwarding is explicitly enabled
1979 if (!flags
.originpeer
&& flags
.proxying
&& request
->peer_login
&&
1980 (strcmp(request
->peer_login
, "PASS") == 0 ||
1981 strcmp(request
->peer_login
, "PROXYPASS") == 0 ||
1982 strcmp(request
->peer_login
, "PASSTHRU") == 0)) {
1983 hdr_out
->addEntry(e
->clone());
1987 /** \par RFC 2616 sect 13.5.1 - Hop-by-Hop headers which Squid does not pass on. */
1989 case Http::HdrType::CONNECTION
: /** \par Connection: */
1990 case Http::HdrType::TE
: /** \par TE: */
1991 case Http::HdrType::KEEP_ALIVE
: /** \par Keep-Alive: */
1992 case Http::HdrType::PROXY_AUTHENTICATE
: /** \par Proxy-Authenticate: */
1993 case Http::HdrType::TRAILER
: /** \par Trailer: */
1994 case Http::HdrType::UPGRADE
: /** \par Upgrade: */
1995 case Http::HdrType::TRANSFER_ENCODING
: /** \par Transfer-Encoding: */
1998 /** \par OTHER headers I haven't bothered to track down yet. */
2000 case Http::HdrType::AUTHORIZATION
:
2001 /** \par WWW-Authorization:
2002 * Pass on WWW authentication */
2004 if (!flags
.originpeer
) {
2005 hdr_out
->addEntry(e
->clone());
2007 /** \note In accelerators, only forward authentication if enabled
2008 * (see also httpFixupAuthentication for special cases)
2010 if (request
->peer_login
&&
2011 (strcmp(request
->peer_login
, "PASS") == 0 ||
2012 strcmp(request
->peer_login
, "PASSTHRU") == 0 ||
2013 strcmp(request
->peer_login
, "PROXYPASS") == 0)) {
2014 hdr_out
->addEntry(e
->clone());
2020 case Http::HdrType::HOST
:
2022 * Normally Squid rewrites the Host: header.
2023 * However, there is one case when we don't: If the URL
2024 * went through our redirector and the admin configured
2025 * 'redir_rewrites_host' to be off.
2027 if (request
->peer_domain
)
2028 hdr_out
->putStr(Http::HdrType::HOST
, request
->peer_domain
);
2029 else if (request
->flags
.redirected
&& !Config
.onoff
.redir_rewrites_host
)
2030 hdr_out
->addEntry(e
->clone());
2032 SBuf authority
= request
->url
.authority();
2033 hdr_out
->putStr(Http::HdrType::HOST
, authority
.c_str());
2038 case Http::HdrType::IF_MODIFIED_SINCE
:
2039 /** \par If-Modified-Since:
2040 * append unless we added our own,
2041 * but only if cache_miss_revalidate is enabled, or
2042 * the request is not cacheable, or
2043 * the request contains authentication credentials.
2044 * \note at most one client's If-Modified-Since header can pass through
2046 // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2047 if (hdr_out
->has(Http::HdrType::IF_MODIFIED_SINCE
))
2049 else if (Config
.onoff
.cache_miss_revalidate
|| !request
->flags
.cachable
|| request
->flags
.auth
)
2050 hdr_out
->addEntry(e
->clone());
2053 case Http::HdrType::IF_NONE_MATCH
:
2054 /** \par If-None-Match:
2055 * append if the wildcard '*' special case value is present, or
2056 * cache_miss_revalidate is disabled, or
2057 * the request is not cacheable in this proxy, or
2058 * the request contains authentication credentials.
2059 * \note this header lists a set of responses for the server to elide sending. Squid added values are extending that set.
2061 // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2062 if (hdr_out
->hasListMember(Http::HdrType::IF_MATCH
, "*", ',') || Config
.onoff
.cache_miss_revalidate
|| !request
->flags
.cachable
|| request
->flags
.auth
)
2063 hdr_out
->addEntry(e
->clone());
2066 case Http::HdrType::MAX_FORWARDS
:
2067 /** \par Max-Forwards:
2068 * pass only on TRACE or OPTIONS requests */
2069 if (request
->method
== Http::METHOD_TRACE
|| request
->method
== Http::METHOD_OPTIONS
) {
2070 const int64_t hops
= e
->getInt64();
2073 hdr_out
->putInt64(Http::HdrType::MAX_FORWARDS
, hops
- 1);
2078 case Http::HdrType::VIA
:
2080 * If Via is disabled then forward any received header as-is.
2081 * Otherwise leave for explicit updated addition later. */
2083 if (!Config
.onoff
.via
)
2084 hdr_out
->addEntry(e
->clone());
2088 case Http::HdrType::RANGE
:
2090 case Http::HdrType::IF_RANGE
:
2092 case Http::HdrType::REQUEST_RANGE
:
2093 /** \par Range:, If-Range:, Request-Range:
2094 * Only pass if we accept ranges */
2096 hdr_out
->addEntry(e
->clone());
2100 case Http::HdrType::PROXY_CONNECTION
: // SHOULD ignore. But doing so breaks things.
2103 case Http::HdrType::CONTENT_LENGTH
:
2104 // pass through unless we chunk; also, keeping this away from default
2105 // prevents request smuggling via Connection: Content-Length tricks
2106 if (!flags
.chunked_request
)
2107 hdr_out
->addEntry(e
->clone());
2110 case Http::HdrType::X_FORWARDED_FOR
:
2112 case Http::HdrType::CACHE_CONTROL
:
2113 /** \par X-Forwarded-For:, Cache-Control:
2114 * handled specially by Squid, so leave off for now.
2115 * append these after the loop if needed */
2118 case Http::HdrType::FRONT_END_HTTPS
:
2119 /** \par Front-End-Https:
2120 * Pass thru only if peer is configured with front-end-https */
2121 if (!flags
.front_end_https
)
2122 hdr_out
->addEntry(e
->clone());
2128 * pass on all other header fields
2129 * which are NOT listed by the special Connection: header. */
2131 if (strConnection
.size()>0 && strListIsMember(&strConnection
, e
->name
.termedBuf(), ',')) {
2132 debugs(11, 2, "'" << e
->name
<< "' header cropped by Connection: definition");
2136 hdr_out
->addEntry(e
->clone());
2141 HttpStateData::decideIfWeDoRanges (HttpRequest
* request
)
2144 /* decide if we want to do Ranges ourselves
2145 * and fetch the whole object now)
2146 * We want to handle Ranges ourselves iff
2147 * - we can actually parse client Range specs
2148 * - the specs are expected to be simple enough (e.g. no out-of-order ranges)
2149 * - reply will be cachable
2150 * (If the reply will be uncachable we have to throw it away after
2151 * serving this request, so it is better to forward ranges to
2152 * the server and fetch only the requested content)
2155 int64_t roffLimit
= request
->getRangeOffsetLimit();
2157 if (NULL
== request
->range
|| !request
->flags
.cachable
2158 || request
->range
->offsetLimitExceeded(roffLimit
) || request
->flags
.connectionAuth
)
2161 debugs(11, 8, "decideIfWeDoRanges: range specs: " <<
2162 request
->range
<< ", cachable: " <<
2163 request
->flags
.cachable
<< "; we_do_ranges: " << result
);
2168 /* build request prefix and append it to a given MemBuf;
2169 * return the length of the prefix */
2171 HttpStateData::buildRequestPrefix(MemBuf
* mb
)
2173 const int offset
= mb
->size
;
2174 /* Uses a local httpver variable to print the HTTP label
2175 * since the HttpRequest may have an older version label.
2176 * XXX: This could create protocol bugs as the headers sent and
2177 * flow control should all be based on the HttpRequest version
2178 * not the one we are sending. Needs checking.
2180 const AnyP::ProtocolVersion httpver
= Http::ProtocolVersion();
2181 const SBuf
url(_peer
&& !_peer
->options
.originserver
? request
->effectiveRequestUri() : request
->url
.path());
2182 mb
->appendf(SQUIDSBUFPH
" " SQUIDSBUFPH
" %s/%d.%d\r\n",
2183 SQUIDSBUFPRINT(request
->method
.image()),
2184 SQUIDSBUFPRINT(url
),
2185 AnyP::ProtocolType_str
[httpver
.protocol
],
2186 httpver
.major
,httpver
.minor
);
2187 /* build and pack headers */
2189 HttpHeader
hdr(hoRequest
);
2190 httpBuildRequestHeader(request
, entry
, fwd
->al
, &hdr
, flags
);
2192 if (request
->flags
.pinned
&& request
->flags
.connectionAuth
)
2193 request
->flags
.authSent
= true;
2194 else if (hdr
.has(Http::HdrType::AUTHORIZATION
))
2195 request
->flags
.authSent
= true;
2200 /* append header terminator */
2201 mb
->append(crlf
, 2);
2202 return mb
->size
- offset
;
2205 /* This will be called when connect completes. Write request. */
2207 HttpStateData::sendRequest()
2211 debugs(11, 5, HERE
<< serverConnection
<< ", request " << request
<< ", this " << this << ".");
2213 if (!Comm::IsConnOpen(serverConnection
)) {
2214 debugs(11,3, HERE
<< "cannot send request to closing " << serverConnection
);
2215 assert(closeHandler
!= NULL
);
2219 typedef CommCbMemFunT
<HttpStateData
, CommTimeoutCbParams
> TimeoutDialer
;
2220 AsyncCall::Pointer timeoutCall
= JobCallback(11, 5,
2221 TimeoutDialer
, this, HttpStateData::httpTimeout
);
2222 commSetConnTimeout(serverConnection
, Config
.Timeout
.lifetime
, timeoutCall
);
2223 flags
.do_next_read
= true;
2224 maybeReadVirginBody();
2226 if (request
->body_pipe
!= NULL
) {
2227 if (!startRequestBodyFlow()) // register to receive body data
2229 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2230 requestSender
= JobCallback(11,5,
2231 Dialer
, this, HttpStateData::sentRequestBody
);
2233 Must(!flags
.chunked_request
);
2234 // use chunked encoding if we do not know the length
2235 if (request
->content_length
< 0)
2236 flags
.chunked_request
= true;
2238 assert(!requestBodySource
);
2239 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2240 requestSender
= JobCallback(11,5,
2241 Dialer
, this, HttpStateData::wroteLast
);
2244 flags
.originpeer
= (_peer
!= NULL
&& _peer
->options
.originserver
);
2245 flags
.proxying
= (_peer
!= NULL
&& !flags
.originpeer
);
2248 * Is keep-alive okay for all request methods?
2250 if (request
->flags
.mustKeepalive
)
2251 flags
.keepalive
= true;
2252 else if (request
->flags
.pinned
)
2253 flags
.keepalive
= request
->persistent();
2254 else if (!Config
.onoff
.server_pconns
)
2255 flags
.keepalive
= false;
2256 else if (_peer
== NULL
)
2257 flags
.keepalive
= true;
2258 else if (_peer
->stats
.n_keepalives_sent
< 10)
2259 flags
.keepalive
= true;
2260 else if ((double) _peer
->stats
.n_keepalives_recv
/
2261 (double) _peer
->stats
.n_keepalives_sent
> 0.50)
2262 flags
.keepalive
= true;
2265 /*The old code here was
2266 if (neighborType(_peer, request->url) == PEER_SIBLING && ...
2267 which is equivalent to:
2268 if (neighborType(_peer, URL()) == PEER_SIBLING && ...
2270 if (((_peer->type == PEER_MULTICAST && p->options.mcast_siblings) ||
2271 _peer->type == PEER_SIBLINGS ) && _peer->options.allow_miss)
2272 flags.only_if_cached = 1;
2274 But I suppose it was a bug
2276 if (neighborType(_peer
, request
->url
) == PEER_SIBLING
&& !_peer
->options
.allow_miss
)
2277 flags
.only_if_cached
= true;
2279 flags
.front_end_https
= _peer
->front_end_https
;
2283 request
->peer_host
=_peer
?_peer
->host
:NULL
;
2284 buildRequestPrefix(&mb
);
2286 debugs(11, 2, "HTTP Server " << serverConnection
);
2287 debugs(11, 2, "HTTP Server REQUEST:\n---------\n" << mb
.buf
<< "\n----------");
2289 Comm::Write(serverConnection
, &mb
, requestSender
);
2294 HttpStateData::getMoreRequestBody(MemBuf
&buf
)
2296 // parent's implementation can handle the no-encoding case
2297 if (!flags
.chunked_request
)
2298 return Client::getMoreRequestBody(buf
);
2302 Must(requestBodySource
!= NULL
);
2303 if (!requestBodySource
->getMoreData(raw
))
2304 return false; // no request body bytes to chunk yet
2306 // optimization: pre-allocate buffer size that should be enough
2307 const mb_size_t rawDataSize
= raw
.contentSize();
2308 // we may need to send: hex-chunk-size CRLF raw-data CRLF last-chunk
2309 buf
.init(16 + 2 + rawDataSize
+ 2 + 5, raw
.max_capacity
);
2311 buf
.appendf("%x\r\n", static_cast<unsigned int>(rawDataSize
));
2312 buf
.append(raw
.content(), rawDataSize
);
2313 buf
.append("\r\n", 2);
2315 Must(rawDataSize
> 0); // we did not accidently created last-chunk above
2317 // Do not send last-chunk unless we successfully received everything
2318 if (receivedWholeRequestBody
) {
2319 Must(!flags
.sentLastChunk
);
2320 flags
.sentLastChunk
= true;
2321 buf
.append("0\r\n\r\n", 5);
2328 httpStart(FwdState
*fwd
)
2330 debugs(11, 3, fwd
->request
->method
<< ' ' << fwd
->entry
->url());
2331 AsyncJob::Start(new HttpStateData(fwd
));
2335 HttpStateData::start()
2337 if (!sendRequest()) {
2338 debugs(11, 3, "httpStart: aborted");
2339 mustStop("HttpStateData::start failed");
2343 ++ statCounter
.server
.all
.requests
;
2344 ++ statCounter
.server
.http
.requests
;
2347 * We used to set the read timeout here, but not any more.
2348 * Now its set in httpSendComplete() after the full request,
2349 * including request body, has been written to the server.
2353 /// if broken posts are enabled for the request, try to fix and return true
2355 HttpStateData::finishingBrokenPost()
2357 #if USE_HTTP_VIOLATIONS
2358 if (!Config
.accessList
.brokenPosts
) {
2359 debugs(11, 5, HERE
<< "No brokenPosts list");
2363 ACLFilledChecklist
ch(Config
.accessList
.brokenPosts
, originalRequest(), NULL
);
2364 if (ch
.fastCheck() != ACCESS_ALLOWED
) {
2365 debugs(11, 5, HERE
<< "didn't match brokenPosts");
2369 if (!Comm::IsConnOpen(serverConnection
)) {
2370 debugs(11, 3, HERE
<< "ignoring broken POST for closed " << serverConnection
);
2371 assert(closeHandler
!= NULL
);
2372 return true; // prevent caller from proceeding as if nothing happened
2375 debugs(11, 3, "finishingBrokenPost: fixing broken POST");
2376 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2377 requestSender
= JobCallback(11,5,
2378 Dialer
, this, HttpStateData::wroteLast
);
2379 Comm::Write(serverConnection
, "\r\n", 2, requestSender
, NULL
);
2383 #endif /* USE_HTTP_VIOLATIONS */
2386 /// if needed, write last-chunk to end the request body and return true
2388 HttpStateData::finishingChunkedRequest()
2390 if (flags
.sentLastChunk
) {
2391 debugs(11, 5, HERE
<< "already sent last-chunk");
2395 Must(receivedWholeRequestBody
); // or we should not be sending last-chunk
2396 flags
.sentLastChunk
= true;
2398 typedef CommCbMemFunT
<HttpStateData
, CommIoCbParams
> Dialer
;
2399 requestSender
= JobCallback(11,5, Dialer
, this, HttpStateData::wroteLast
);
2400 Comm::Write(serverConnection
, "0\r\n\r\n", 5, requestSender
, NULL
);
2405 HttpStateData::doneSendingRequestBody()
2407 Client::doneSendingRequestBody();
2408 debugs(11,5, HERE
<< serverConnection
);
2410 // do we need to write something after the last body byte?
2411 if (flags
.chunked_request
&& finishingChunkedRequest())
2413 if (!flags
.chunked_request
&& finishingBrokenPost())
2419 // more origin request body data is available
2421 HttpStateData::handleMoreRequestBodyAvailable()
2423 if (eof
|| !Comm::IsConnOpen(serverConnection
)) {
2424 // XXX: we should check this condition in other callbacks then!
2425 // TODO: Check whether this can actually happen: We should unsubscribe
2426 // as a body consumer when the above condition(s) are detected.
2427 debugs(11, DBG_IMPORTANT
, HERE
<< "Transaction aborted while reading HTTP body");
2431 assert(requestBodySource
!= NULL
);
2433 if (requestBodySource
->buf().hasContent()) {
2434 // XXX: why does not this trigger a debug message on every request?
2436 if (flags
.headers_parsed
&& !flags
.abuse_detected
) {
2437 flags
.abuse_detected
= true;
2438 debugs(11, DBG_IMPORTANT
, "http handleMoreRequestBodyAvailable: Likely proxy abuse detected '" << request
->client_addr
<< "' -> '" << entry
->url() << "'" );
2440 if (virginReply()->sline
.status() == Http::scInvalidHeader
) {
2442 mustStop("HttpStateData::handleMoreRequestBodyAvailable");
2448 HttpStateData::handleMoreRequestBodyAvailable();
2451 // premature end of the request body
2453 HttpStateData::handleRequestBodyProducerAborted()
2455 Client::handleRequestBodyProducerAborted();
2456 if (entry
->isEmpty()) {
2457 debugs(11, 3, "request body aborted: " << serverConnection
);
2458 // We usually get here when ICAP REQMOD aborts during body processing.
2459 // We might also get here if client-side aborts, but then our response
2460 // should not matter because either client-side will provide its own or
2461 // there will be no response at all (e.g., if the the client has left).
2462 ErrorState
*err
= new ErrorState(ERR_ICAP_FAILURE
, Http::scInternalServerError
, fwd
->request
);
2463 err
->detailError(ERR_DETAIL_SRV_REQMOD_REQ_BODY
);
2467 abortTransaction("request body producer aborted");
2470 // called when we wrote request headers(!) or a part of the body
2472 HttpStateData::sentRequestBody(const CommIoCbParams
&io
)
2475 statCounter
.server
.http
.kbytes_out
+= io
.size
;
2477 Client::sentRequestBody(io
);
2481 HttpStateData::abortAll(const char *reason
)
2483 debugs(11,5, HERE
<< "aborting transaction for " << reason
<<
2484 "; " << serverConnection
<< ", this " << this);