2 * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 85 Client-side Request Routines */
12 * General logic of request processing:
14 * We run a series of tests to determine if access will be permitted, and to do
15 * any redirection. Then we call into the result clientStream to retrieve data.
16 * From that point on it's up to reply management.
20 #include "acl/FilledChecklist.h"
21 #include "acl/Gadgets.h"
22 #include "anyp/PortCfg.h"
23 #include "base/AsyncJobCalls.h"
24 #include "client_side.h"
25 #include "client_side_reply.h"
26 #include "client_side_request.h"
27 #include "ClientRequestContext.h"
28 #include "clientStream.h"
29 #include "comm/Connection.h"
30 #include "comm/Write.h"
31 #include "err_detail_type.h"
32 #include "errorpage.h"
35 #include "format/Token.h"
38 #include "helper/Reply.h"
40 #include "HttpHdrCc.h"
41 #include "HttpReply.h"
42 #include "HttpRequest.h"
43 #include "ip/QosConfig.h"
45 #include "log/access_log.h"
46 #include "MemObject.h"
48 #include "profiler/Profiler.h"
50 #include "SquidConfig.h"
51 #include "SquidTime.h"
58 #include "auth/UserRequest.h"
61 #include "adaptation/AccessCheck.h"
62 #include "adaptation/Answer.h"
63 #include "adaptation/Iterator.h"
64 #include "adaptation/Service.h"
66 #include "adaptation/icap/History.h"
70 #include "ssl/ServerBump.h"
71 #include "ssl/support.h"
75 #define comm_close comm_lingering_close
78 static const char *const crlf
= "\r\n";
80 #if FOLLOW_X_FORWARDED_FOR
81 static void clientFollowXForwardedForCheck(allow_t answer
, void *data
);
82 #endif /* FOLLOW_X_FORWARDED_FOR */
84 ErrorState
*clientBuildError(err_type
, Http::StatusCode
, char const *url
, Ip::Address
&, HttpRequest
*);
86 CBDATA_CLASS_INIT(ClientRequestContext
);
90 static void clientAccessCheckDoneWrapper(allow_t
, void *);
92 static void sslBumpAccessCheckDoneWrapper(allow_t
, void *);
94 static int clientHierarchical(ClientHttpRequest
* http
);
95 static void clientInterpretRequestHeaders(ClientHttpRequest
* http
);
96 static HLPCB clientRedirectDoneWrapper
;
97 static HLPCB clientStoreIdDoneWrapper
;
98 static void checkNoCacheDoneWrapper(allow_t
, void *);
99 SQUIDCEXTERN CSR clientGetMoreData
;
100 SQUIDCEXTERN CSS clientReplyStatus
;
101 SQUIDCEXTERN CSD clientReplyDetach
;
102 static void checkFailureRatio(err_type
, hier_code
);
104 ClientRequestContext::~ClientRequestContext()
107 * Release our "lock" on our parent, ClientHttpRequest, if we
112 cbdataReferenceDone(http
);
115 debugs(85,3, "ClientRequestContext destructed, this=" << this);
118 ClientRequestContext::ClientRequestContext(ClientHttpRequest
*anHttp
) :
119 http(cbdataReference(anHttp
)),
121 redirect_state(REDIRECT_NONE
),
122 store_id_state(REDIRECT_NONE
),
123 host_header_verify_done(false),
124 http_access_done(false),
125 adapted_http_access_done(false),
127 adaptation_acl_check_done(false),
129 redirect_done(false),
130 store_id_done(false),
131 no_cache_done(false),
132 interpreted_req_hdrs(false),
133 tosToClientDone(false),
134 nfmarkToClientDone(false),
136 sslBumpCheckDone(false),
139 readNextRequest(false)
141 debugs(85, 3, "ClientRequestContext constructed, this=" << this);
144 CBDATA_CLASS_INIT(ClientHttpRequest
);
146 ClientHttpRequest::ClientHttpRequest(ConnStateData
* aConn
) :
148 AsyncJob("ClientHttpRequest"),
154 logType(LOG_TAG_NONE
),
155 calloutContext(NULL
),
156 maxReplyBodySize_(0),
161 , sslBumpNeed_(Ssl::bumpEnd
)
164 , request_satisfaction_mode(false)
165 , request_satisfaction_offset(0)
169 al
= new AccessLogEntry
;
170 al
->cache
.start_time
= current_time
;
171 al
->tcpClient
= clientConnection
= aConn
->clientConnection
;
172 al
->cache
.port
= aConn
->port
;
173 al
->cache
.caddr
= aConn
->log_addr
;
176 if (aConn
->clientConnection
!= NULL
&& aConn
->clientConnection
->isOpen()) {
177 if (auto ssl
= fd_table
[aConn
->clientConnection
->fd
].ssl
)
178 al
->cache
.sslClientCert
.reset(SSL_get_peer_certificate(ssl
));
181 dlinkAdd(this, &active
, &ClientActiveRequests
);
185 * returns true if client specified that the object must come from the cache
186 * without contacting origin server
189 ClientHttpRequest::onlyIfCached()const
192 return request
->cache_control
&&
193 request
->cache_control
->onlyIfCached();
197 * This function is designed to serve a fairly specific purpose.
198 * Occasionally our vBNS-connected caches can talk to each other, but not
199 * the rest of the world. Here we try to detect frequent failures which
200 * make the cache unusable (e.g. DNS lookup and connect() failures). If
201 * the failure:success ratio goes above 1.0 then we go into "hit only"
202 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors
203 * will only fetch HITs from us if they are using the ICP protocol. We
204 * stay in this mode for 5 minutes.
206 * Duane W., Sept 16, 1996
209 checkFailureRatio(err_type etype
, hier_code hcode
)
211 // Can be set at compile time with -D compiler flag
212 #ifndef FAILURE_MODE_TIME
213 #define FAILURE_MODE_TIME 300
216 if (hcode
== HIER_NONE
)
219 // don't bother when ICP is disabled.
220 if (Config
.Port
.icp
<= 0)
223 static double magic_factor
= 100.0;
227 n_good
= magic_factor
/ (1.0 + request_failure_ratio
);
229 n_bad
= magic_factor
- n_good
;
235 case ERR_CONNECT_FAIL
:
236 case ERR_SECURE_CONNECT_FAIL
:
246 request_failure_ratio
= n_bad
/ n_good
;
248 if (hit_only_mode_until
> squid_curtime
)
251 if (request_failure_ratio
< 1.0)
254 debugs(33, DBG_CRITICAL
, "WARNING: Failure Ratio at "<< std::setw(4)<<
255 std::setprecision(3) << request_failure_ratio
);
257 debugs(33, DBG_CRITICAL
, "WARNING: ICP going into HIT-only mode for " <<
258 FAILURE_MODE_TIME
/ 60 << " minutes...");
260 hit_only_mode_until
= squid_curtime
+ FAILURE_MODE_TIME
;
262 request_failure_ratio
= 0.8; /* reset to something less than 1.0 */
265 ClientHttpRequest::~ClientHttpRequest()
267 debugs(33, 3, "httpRequestFree: " << uri
);
268 PROF_start(httpRequestFree
);
270 // Even though freeResources() below may destroy the request,
271 // we no longer set request->body_pipe to NULL here
272 // because we did not initiate that pipe (ConnStateData did)
274 /* the ICP check here was erroneous
275 * - StoreEntry::releaseRequest was always called if entry was valid
283 checkFailureRatio(request
->errType
, al
->hier
.code
);
288 announceInitiatorAbort(virginHeadSource
);
290 if (adaptedBodySource
!= NULL
)
291 stopConsumingFrom(adaptedBodySource
);
295 delete calloutContext
;
297 clientConnection
= NULL
;
300 cbdataReferenceDone(conn_
);
302 /* moving to the next connection is handled by the context free */
303 dlinkDelete(&active
, &ClientActiveRequests
);
305 PROF_stop(httpRequestFree
);
309 * Create a request and kick it off
314 * TODO: Pass in the buffers to be used in the inital Read request, as they are
315 * determined by the user
318 clientBeginRequest(const HttpRequestMethod
& method
, char const *url
, CSCB
* streamcallback
,
319 CSD
* streamdetach
, ClientStreamData streamdata
, HttpHeader
const *header
,
320 char *tailbuf
, size_t taillen
)
323 ClientHttpRequest
*http
= new ClientHttpRequest(NULL
);
324 HttpRequest
*request
;
325 StoreIOBuffer tempBuffer
;
326 if (http
->al
!= NULL
)
327 http
->al
->cache
.start_time
= current_time
;
328 /* this is only used to adjust the connection offset in client_side.c */
330 tempBuffer
.length
= taillen
;
331 tempBuffer
.data
= tailbuf
;
332 /* client stream setup */
333 clientStreamInit(&http
->client_stream
, clientGetMoreData
, clientReplyDetach
,
334 clientReplyStatus
, new clientReplyContext(http
), streamcallback
,
335 streamdetach
, streamdata
, tempBuffer
);
336 /* make it visible in the 'current acctive requests list' */
338 /* internal requests only makes sense in an
339 * accelerator today. TODO: accept flags ? */
340 http
->flags
.accel
= true;
341 /* allow size for url rewriting */
342 url_sz
= strlen(url
) + Config
.appendDomainLen
+ 5;
343 http
->uri
= (char *)xcalloc(url_sz
, 1);
344 strcpy(http
->uri
, url
);
346 if ((request
= HttpRequest::CreateFromUrlAndMethod(http
->uri
, method
)) == NULL
) {
347 debugs(85, 5, "Invalid URL: " << http
->uri
);
352 * now update the headers in request with our supplied headers. urlParse
353 * should return a blank header set, but we use Update to be sure of
357 request
->header
.update(header
, NULL
);
359 http
->log_uri
= xstrdup(urlCanonicalClean(request
));
361 /* http struct now ready */
364 * build new header list *? TODO
366 request
->flags
.accelerated
= http
->flags
.accel
;
368 request
->flags
.internalClient
= true;
370 /* this is an internally created
371 * request, not subject to acceleration
372 * target overrides */
374 * FIXME? Do we want to detect and handle internal requests of internal
378 /* Internally created requests cannot have bodies today */
379 request
->content_length
= 0;
381 request
->client_addr
.setNoAddr();
383 #if FOLLOW_X_FORWARDED_FOR
384 request
->indirect_client_addr
.setNoAddr();
385 #endif /* FOLLOW_X_FORWARDED_FOR */
387 request
->my_addr
.setNoAddr(); /* undefined for internal requests */
389 request
->my_addr
.port(0);
391 request
->http_ver
= Http::ProtocolVersion();
393 http
->request
= request
;
394 HTTPMSGLOCK(http
->request
);
396 /* optional - skip the access check ? */
397 http
->calloutContext
= new ClientRequestContext(http
);
399 http
->calloutContext
->http_access_done
= false;
401 http
->calloutContext
->redirect_done
= true;
403 http
->calloutContext
->no_cache_done
= true;
411 ClientRequestContext::httpStateIsValid()
413 ClientHttpRequest
*http_
= http
;
415 if (cbdataReferenceValid(http_
))
420 cbdataReferenceDone(http_
);
425 #if FOLLOW_X_FORWARDED_FOR
427 * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
428 * against the followXFF ACL, or cleans up and passes control to
429 * clientAccessCheck().
431 * The trust model here is a little ambiguous. So to clarify the logic:
432 * - we may always use the direct client address as the client IP.
433 * - these trust tests merey tell whether we trust given IP enough to believe the
434 * IP string which it appended to the X-Forwarded-For: header.
435 * - if at any point we don't trust what an IP adds we stop looking.
436 * - at that point the current contents of indirect_client_addr are the value set
437 * by the last previously trusted IP.
438 * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
441 clientFollowXForwardedForCheck(allow_t answer
, void *data
)
443 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
445 if (!calloutContext
->httpStateIsValid())
448 ClientHttpRequest
*http
= calloutContext
->http
;
449 HttpRequest
*request
= http
->request
;
452 * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
453 * called as a result of ACL checks, or -1 if we are called when
454 * there's nothing left to do.
456 if (answer
== ACCESS_ALLOWED
&&
457 request
->x_forwarded_for_iterator
.size () != 0) {
460 * Remove the last comma-delimited element from the
461 * x_forwarded_for_iterator and use it to repeat the cycle.
464 const char *asciiaddr
;
467 p
= request
->x_forwarded_for_iterator
.termedBuf();
468 l
= request
->x_forwarded_for_iterator
.size();
471 * XXX x_forwarded_for_iterator should really be a list of
472 * IP addresses, but it's a String instead. We have to
473 * walk backwards through the String, biting off the last
474 * comma-delimited part each time. As long as the data is in
475 * a String, we should probably implement and use a variant of
476 * strListGetItem() that walks backwards instead of forwards
477 * through a comma-separated list. But we don't even do that;
478 * we just do the work in-line here.
480 /* skip trailing space and commas */
481 while (l
> 0 && (p
[l
-1] == ',' || xisspace(p
[l
-1])))
483 request
->x_forwarded_for_iterator
.cut(l
);
484 /* look for start of last item in list */
485 while (l
> 0 && ! (p
[l
-1] == ',' || xisspace(p
[l
-1])))
488 if ((addr
= asciiaddr
)) {
489 request
->indirect_client_addr
= addr
;
490 request
->x_forwarded_for_iterator
.cut(l
);
491 calloutContext
->acl_checklist
= clientAclChecklistCreate(Config
.accessList
.followXFF
, http
);
492 if (!Config
.onoff
.acl_uses_indirect_client
) {
493 /* override the default src_addr tested if we have to go deeper than one level into XFF */
494 Filled(calloutContext
->acl_checklist
)->src_addr
= request
->indirect_client_addr
;
496 calloutContext
->acl_checklist
->nonBlockingCheck(clientFollowXForwardedForCheck
, data
);
499 } /*if (answer == ACCESS_ALLOWED &&
500 request->x_forwarded_for_iterator.size () != 0)*/
502 /* clean up, and pass control to clientAccessCheck */
503 if (Config
.onoff
.log_uses_indirect_client
) {
505 * Ensure that the access log shows the indirect client
506 * instead of the direct client.
508 ConnStateData
*conn
= http
->getConn();
509 conn
->log_addr
= request
->indirect_client_addr
;
510 http
->al
->cache
.caddr
= conn
->log_addr
;
512 request
->x_forwarded_for_iterator
.clean();
513 request
->flags
.done_follow_x_forwarded_for
= true;
515 if (answer
!= ACCESS_ALLOWED
&& answer
!= ACCESS_DENIED
) {
516 debugs(28, DBG_CRITICAL
, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request
->indirect_client_addr
);
519 /* process actual access ACL as normal. */
520 calloutContext
->clientAccessCheck();
522 #endif /* FOLLOW_X_FORWARDED_FOR */
525 hostHeaderIpVerifyWrapper(const ipcache_addrs
* ia
, const Dns::LookupDetails
&dns
, void *data
)
527 ClientRequestContext
*c
= static_cast<ClientRequestContext
*>(data
);
528 c
->hostHeaderIpVerify(ia
, dns
);
532 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs
* ia
, const Dns::LookupDetails
&dns
)
534 Comm::ConnectionPointer clientConn
= http
->getConn()->clientConnection
;
536 // note the DNS details for the transaction stats.
537 http
->request
->recordLookup(dns
);
539 if (ia
!= NULL
&& ia
->count
> 0) {
540 // Is the NAT destination IP in DNS?
541 for (int i
= 0; i
< ia
->count
; ++i
) {
542 if (clientConn
->local
.matchIPAddr(ia
->in_addrs
[i
]) == 0) {
543 debugs(85, 3, HERE
<< "validate IP " << clientConn
->local
<< " possible from Host:");
544 http
->request
->flags
.hostVerified
= true;
548 debugs(85, 3, HERE
<< "validate IP " << clientConn
->local
<< " non-match from Host: IP " << ia
->in_addrs
[i
]);
551 debugs(85, 3, HERE
<< "FAIL: validate IP " << clientConn
->local
<< " possible from Host:");
552 hostHeaderVerifyFailed("local IP", "any domain IP");
556 ClientRequestContext::hostHeaderVerifyFailed(const char *A
, const char *B
)
558 // IP address validation for Host: failed. Admin wants to ignore them.
559 // NP: we do not yet handle CONNECT tunnels well, so ignore for them
560 if (!Config
.onoff
.hostStrictVerify
&& http
->request
->method
!= Http::METHOD_CONNECT
) {
561 debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http
->getConn()->clientConnection
<<
562 " (" << A
<< " does not match " << B
<< ") on URL: " << http
->request
->effectiveRequestUri());
564 // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
565 // The problems here are about WRITE for new cache content, which means flags.cachable
566 http
->request
->flags
.cachable
= false; // MUST NOT cache (for now)
567 // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
568 http
->request
->flags
.hierarchical
= false; // MUST NOT pass to peers (for now)
569 // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
574 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: Host header forgery detected on " <<
575 http
->getConn()->clientConnection
<< " (" << A
<< " does not match " << B
<< ")");
576 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: By user agent: " << http
->request
->header
.getStr(HDR_USER_AGENT
));
577 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: on URL: " << http
->request
->effectiveRequestUri());
579 // IP address validation for Host: failed. reject the connection.
580 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->prev
->data
;
581 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
583 repContext
->setReplyToError(ERR_CONFLICT_HOST
, Http::scConflict
,
584 http
->request
->method
, NULL
,
585 http
->getConn()->clientConnection
->remote
,
589 http
->getConn() != NULL
&& http
->getConn()->getAuth() != NULL
?
590 http
->getConn()->getAuth() : http
->request
->auth_user_request
);
594 node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
595 clientStreamRead(node
, http
, node
->readBuffer
);
599 ClientRequestContext::hostHeaderVerify()
601 // Require a Host: header.
602 const char *host
= http
->request
->header
.getStr(HDR_HOST
);
605 // TODO: dump out the HTTP/1.1 error about missing host header.
606 // otherwise this is fine, can't forge a header value when its not even set.
607 debugs(85, 3, HERE
<< "validate skipped with no Host: header present.");
612 if (http
->request
->flags
.internal
) {
613 // TODO: kill this when URL handling allows partial URLs out of accel mode
614 // and we no longer screw with the URL just to add our internal host there
615 debugs(85, 6, HERE
<< "validate skipped due to internal composite URL.");
620 // Locate if there is a port attached, strip ready for IP lookup
621 char *portStr
= NULL
;
622 char *hostB
= xstrdup(host
);
624 if (host
[0] == '[') {
626 portStr
= strchr(hostB
, ']');
627 if (portStr
&& *(++portStr
) != ':') {
631 // Domain or IPv4 literal with port
632 portStr
= strrchr(hostB
, ':');
637 *portStr
= '\0'; // strip the ':'
638 if (*(++portStr
) != '\0') {
640 int64_t ret
= strtoll(portStr
, &end
, 10);
641 if (end
== portStr
|| *end
!= '\0' || ret
< 1 || ret
> 0xFFFF) {
642 // invalid port details. Replace the ':'
646 port
= (ret
& 0xFFFF);
650 debugs(85, 3, "validate host=" << host
<< ", port=" << port
<< ", portStr=" << (portStr
?portStr
:"NULL"));
651 if (http
->request
->flags
.intercepted
|| http
->request
->flags
.interceptTproxy
) {
652 // verify the Host: port (if any) matches the apparent destination
653 if (portStr
&& port
!= http
->getConn()->clientConnection
->local
.port()) {
654 debugs(85, 3, "FAIL on validate port " << http
->getConn()->clientConnection
->local
.port() <<
655 " matches Host: port " << port
<< " (" << portStr
<< ")");
656 hostHeaderVerifyFailed("intercepted port", portStr
);
658 // XXX: match the scheme default port against the apparent destination
660 // verify the destination DNS is one of the Host: headers IPs
661 ipcache_nbgethostbyname(host
, hostHeaderIpVerifyWrapper
, this);
663 } else if (!Config
.onoff
.hostStrictVerify
) {
664 debugs(85, 3, "validate skipped.");
666 } else if (strlen(host
) != strlen(http
->request
->url
.host())) {
667 // Verify forward-proxy requested URL domain matches the Host: header
668 debugs(85, 3, "FAIL on validate URL domain length " << http
->request
->url
.host() << " matches Host: " << host
);
669 hostHeaderVerifyFailed(host
, http
->request
->url
.host());
670 } else if (matchDomainName(host
, http
->request
->url
.host()) != 0) {
671 // Verify forward-proxy requested URL domain matches the Host: header
672 debugs(85, 3, "FAIL on validate URL domain " << http
->request
->url
.host() << " matches Host: " << host
);
673 hostHeaderVerifyFailed(host
, http
->request
->url
.host());
674 } else if (portStr
&& port
!= http
->request
->url
.port()) {
675 // Verify forward-proxy requested URL domain matches the Host: header
676 debugs(85, 3, "FAIL on validate URL port " << http
->request
->url
.port() << " matches Host: port " << portStr
);
677 hostHeaderVerifyFailed("URL port", portStr
);
678 } else if (!portStr
&& http
->request
->method
!= Http::METHOD_CONNECT
&& http
->request
->url
.port() != http
->request
->url
.getScheme().defaultPort()) {
679 // Verify forward-proxy requested URL domain matches the Host: header
680 // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
681 debugs(85, 3, "FAIL on validate URL port " << http
->request
->url
.port() << " matches Host: default port " << http
->request
->url
.getScheme().defaultPort());
682 hostHeaderVerifyFailed("URL port", "default port");
685 debugs(85, 3, "validate passed.");
686 http
->request
->flags
.hostVerified
= true;
692 /* This is the entry point for external users of the client_side routines */
694 ClientRequestContext::clientAccessCheck()
696 #if FOLLOW_X_FORWARDED_FOR
697 if (!http
->request
->flags
.doneFollowXff() &&
698 Config
.accessList
.followXFF
&&
699 http
->request
->header
.has(HDR_X_FORWARDED_FOR
)) {
701 /* we always trust the direct client address for actual use */
702 http
->request
->indirect_client_addr
= http
->request
->client_addr
;
703 http
->request
->indirect_client_addr
.port(0);
705 /* setup the XFF iterator for processing */
706 http
->request
->x_forwarded_for_iterator
= http
->request
->header
.getList(HDR_X_FORWARDED_FOR
);
708 /* begin by checking to see if we trust direct client enough to walk XFF */
709 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.followXFF
, http
);
710 acl_checklist
->nonBlockingCheck(clientFollowXForwardedForCheck
, this);
715 if (Config
.accessList
.http
) {
716 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.http
, http
);
717 acl_checklist
->nonBlockingCheck(clientAccessCheckDoneWrapper
, this);
719 debugs(0, DBG_CRITICAL
, "No http_access configuration found. This will block ALL traffic");
720 clientAccessCheckDone(ACCESS_DENIED
);
725 * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
726 * The default here is to allow all. Since the earlier http_access should do a default deny all.
727 * This check is just for a last-minute denial based on adapted request headers.
730 ClientRequestContext::clientAccessCheck2()
732 if (Config
.accessList
.adapted_http
) {
733 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.adapted_http
, http
);
734 acl_checklist
->nonBlockingCheck(clientAccessCheckDoneWrapper
, this);
736 debugs(85, 2, HERE
<< "No adapted_http_access configuration. default: ALLOW");
737 clientAccessCheckDone(ACCESS_ALLOWED
);
742 clientAccessCheckDoneWrapper(allow_t answer
, void *data
)
744 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
746 if (!calloutContext
->httpStateIsValid())
749 calloutContext
->clientAccessCheckDone(answer
);
753 ClientRequestContext::clientAccessCheckDone(const allow_t
&answer
)
755 acl_checklist
= NULL
;
757 Http::StatusCode status
;
758 debugs(85, 2, "The request " << http
->request
->method
<< ' ' <<
759 http
->uri
<< " is " << answer
<<
760 "; last ACL checked: " << (AclMatchedName
? AclMatchedName
: "[none]"));
763 char const *proxy_auth_msg
= "<null>";
764 if (http
->getConn() != NULL
&& http
->getConn()->getAuth() != NULL
)
765 proxy_auth_msg
= http
->getConn()->getAuth()->denyMessage("<null>");
766 else if (http
->request
->auth_user_request
!= NULL
)
767 proxy_auth_msg
= http
->request
->auth_user_request
->denyMessage("<null>");
770 if (answer
!= ACCESS_ALLOWED
) {
771 // auth has a grace period where credentials can be expired but okay not to challenge.
773 /* Send an auth challenge or error */
774 // XXX: do we still need aclIsProxyAuth() ?
775 bool auth_challenge
= (answer
== ACCESS_AUTH_REQUIRED
|| aclIsProxyAuth(AclMatchedName
));
776 debugs(85, 5, "Access Denied: " << http
->uri
);
777 debugs(85, 5, "AclMatchedName = " << (AclMatchedName
? AclMatchedName
: "<null>"));
780 debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg
? proxy_auth_msg
: "<null>"));
784 * NOTE: get page_id here, based on AclMatchedName because if
785 * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
786 * the clientCreateStoreEntry() call just below. Pedro Ribeiro
789 page_id
= aclGetDenyInfoPage(&Config
.denyInfoList
, AclMatchedName
, answer
!= ACCESS_AUTH_REQUIRED
);
791 http
->logType
= LOG_TCP_DENIED
;
793 if (auth_challenge
) {
795 if (http
->request
->flags
.sslBumped
) {
796 /*SSL Bumped request, authentication is not possible*/
797 status
= Http::scForbidden
;
798 } else if (!http
->flags
.accel
) {
799 /* Proxy authorisation needed */
800 status
= Http::scProxyAuthenticationRequired
;
802 /* WWW authorisation needed */
803 status
= Http::scUnauthorized
;
806 // need auth, but not possible to do.
807 status
= Http::scForbidden
;
809 if (page_id
== ERR_NONE
)
810 page_id
= ERR_CACHE_ACCESS_DENIED
;
812 status
= Http::scForbidden
;
814 if (page_id
== ERR_NONE
)
815 page_id
= ERR_ACCESS_DENIED
;
818 Ip::Address tmpnoaddr
;
819 tmpnoaddr
.setNoAddr();
820 error
= clientBuildError(page_id
, status
,
822 http
->getConn() != NULL
? http
->getConn()->clientConnection
->remote
: tmpnoaddr
,
827 error
->auth_user_request
=
828 http
->getConn() != NULL
&& http
->getConn()->getAuth() != NULL
?
829 http
->getConn()->getAuth() : http
->request
->auth_user_request
;
832 readNextRequest
= true;
835 /* ACCESS_ALLOWED continues here ... */
836 safe_free(http
->uri
);
837 const SBuf
tmp(http
->request
->effectiveRequestUri());
838 http
->uri
= xstrndup(tmp
.rawContent(), tmp
.length()+1);
844 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g
)
846 debugs(93,3,HERE
<< this << " adaptationAclCheckDone called");
849 Adaptation::Icap::History::Pointer ih
= request
->icapHistory();
851 if (getConn() != NULL
&& getConn()->clientConnection
!= NULL
) {
852 ih
->rfc931
= getConn()->clientConnection
->rfc931
;
854 if (getConn()->clientConnection
->isOpen()) {
855 ih
->ssluser
= sslGetUserEmail(fd_table
[getConn()->clientConnection
->fd
].ssl
);
859 ih
->log_uri
= log_uri
;
865 debugs(85,3, HERE
<< "no adaptation needed");
876 clientRedirectAccessCheckDone(allow_t answer
, void *data
)
878 ClientRequestContext
*context
= (ClientRequestContext
*)data
;
879 ClientHttpRequest
*http
= context
->http
;
880 context
->acl_checklist
= NULL
;
882 if (answer
== ACCESS_ALLOWED
)
883 redirectStart(http
, clientRedirectDoneWrapper
, context
);
885 Helper::Reply nilReply
;
886 nilReply
.result
= Helper::Error
;
887 context
->clientRedirectDone(nilReply
);
892 ClientRequestContext::clientRedirectStart()
894 debugs(33, 5, HERE
<< "'" << http
->uri
<< "'");
895 (void)SyncNotes(*http
->al
, *http
->request
);
896 if (Config
.accessList
.redirector
) {
897 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.redirector
, http
);
898 acl_checklist
->nonBlockingCheck(clientRedirectAccessCheckDone
, this);
900 redirectStart(http
, clientRedirectDoneWrapper
, this);
904 * This methods handles Access checks result of StoreId access list.
905 * Will handle as "ERR" (no change) in a case Access is not allowed.
908 clientStoreIdAccessCheckDone(allow_t answer
, void *data
)
910 ClientRequestContext
*context
= static_cast<ClientRequestContext
*>(data
);
911 ClientHttpRequest
*http
= context
->http
;
912 context
->acl_checklist
= NULL
;
914 if (answer
== ACCESS_ALLOWED
)
915 storeIdStart(http
, clientStoreIdDoneWrapper
, context
);
917 debugs(85, 3, "access denied expected ERR reply handling: " << answer
);
918 Helper::Reply nilReply
;
919 nilReply
.result
= Helper::Error
;
920 context
->clientStoreIdDone(nilReply
);
925 * Start locating an alternative storeage ID string (if any) from admin
926 * configured helper program. This is an asynchronous operation terminating in
927 * ClientRequestContext::clientStoreIdDone() when completed.
930 ClientRequestContext::clientStoreIdStart()
932 debugs(33, 5,"'" << http
->uri
<< "'");
934 if (Config
.accessList
.store_id
) {
935 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.store_id
, http
);
936 acl_checklist
->nonBlockingCheck(clientStoreIdAccessCheckDone
, this);
938 storeIdStart(http
, clientStoreIdDoneWrapper
, this);
942 clientHierarchical(ClientHttpRequest
* http
)
944 HttpRequest
*request
= http
->request
;
945 HttpRequestMethod method
= request
->method
;
947 // intercepted requests MUST NOT (yet) be sent to peers unless verified
948 if (!request
->flags
.hostVerified
&& (request
->flags
.intercepted
|| request
->flags
.interceptTproxy
))
952 * IMS needs a private key, so we can use the hierarchy for IMS only if our
953 * neighbors support private keys
956 if (request
->flags
.ims
&& !neighbors_do_private_keys
)
960 * This is incorrect: authenticating requests can be sent via a hierarchy
961 * (they can even be cached if the correct headers are set on the reply)
963 if (request
->flags
.auth
)
966 if (method
== Http::METHOD_TRACE
)
969 if (method
!= Http::METHOD_GET
)
972 if (request
->flags
.loopDetected
)
975 if (request
->url
.getScheme() == AnyP::PROTO_HTTP
)
976 return method
.respMaybeCacheable();
978 if (request
->url
.getScheme() == AnyP::PROTO_GOPHER
)
979 return gopherCachable(request
);
981 if (request
->url
.getScheme() == AnyP::PROTO_CACHE_OBJECT
)
988 clientCheckPinning(ClientHttpRequest
* http
)
990 HttpRequest
*request
= http
->request
;
991 HttpHeader
*req_hdr
= &request
->header
;
992 ConnStateData
*http_conn
= http
->getConn();
994 /* Internal requests such as those from ESI includes may be without
995 * a client connection
1000 request
->flags
.connectionAuthDisabled
= http_conn
->port
->connection_auth_disabled
;
1001 if (!request
->flags
.connectionAuthDisabled
) {
1002 if (Comm::IsConnOpen(http_conn
->pinning
.serverConnection
)) {
1003 if (http_conn
->pinning
.auth
) {
1004 request
->flags
.connectionAuth
= true;
1005 request
->flags
.auth
= true;
1007 request
->flags
.connectionProxyAuth
= true;
1009 // These should already be linked correctly.
1010 assert(request
->clientConnectionManager
== http_conn
);
1014 /* check if connection auth is used, and flag as candidate for pinning
1016 * Note: we may need to set flags.connectionAuth even if the connection
1017 * is already pinned if it was pinned earlier due to proxy auth
1019 if (!request
->flags
.connectionAuth
) {
1020 if (req_hdr
->has(HDR_AUTHORIZATION
) || req_hdr
->has(HDR_PROXY_AUTHORIZATION
)) {
1021 HttpHeaderPos pos
= HttpHeaderInitPos
;
1024 while ((e
= req_hdr
->getEntry(&pos
))) {
1025 if (e
->id
== HDR_AUTHORIZATION
|| e
->id
== HDR_PROXY_AUTHORIZATION
) {
1026 const char *value
= e
->value
.rawBuf();
1027 if (strncasecmp(value
, "NTLM ", 5) == 0
1029 strncasecmp(value
, "Negotiate ", 10) == 0
1031 strncasecmp(value
, "Kerberos ", 9) == 0) {
1032 if (e
->id
== HDR_AUTHORIZATION
) {
1033 request
->flags
.connectionAuth
= true;
1036 request
->flags
.connectionProxyAuth
= true;
1042 if (may_pin
&& !request
->pinnedConnection()) {
1043 // These should already be linked correctly. Just need the ServerConnection to pinn.
1044 assert(request
->clientConnectionManager
== http_conn
);
1051 clientInterpretRequestHeaders(ClientHttpRequest
* http
)
1053 HttpRequest
*request
= http
->request
;
1054 HttpHeader
*req_hdr
= &request
->header
;
1055 bool no_cache
= false;
1058 request
->imslen
= -1;
1059 request
->ims
= req_hdr
->getTime(HDR_IF_MODIFIED_SINCE
);
1061 if (request
->ims
> 0)
1062 request
->flags
.ims
= true;
1064 if (!request
->flags
.ignoreCc
) {
1065 if (request
->cache_control
) {
1066 if (request
->cache_control
->hasNoCache())
1069 // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1070 } else if (req_hdr
->has(HDR_PRAGMA
))
1071 no_cache
= req_hdr
->hasListMember(HDR_PRAGMA
,"no-cache",',');
1074 * Work around for supporting the Reload button in IE browsers when Squid
1075 * is used as an accelerator or transparent proxy, by turning accelerated
1076 * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1077 * actually only fixed in SP1, but we can't tell whether we are talking to
1078 * SP1 or not so all 5.5 versions are treated 'normally').
1080 if (Config
.onoff
.ie_refresh
) {
1081 if (http
->flags
.accel
&& request
->flags
.ims
) {
1082 if ((str
= req_hdr
->getStr(HDR_USER_AGENT
))) {
1083 if (strstr(str
, "MSIE 5.01") != NULL
)
1085 else if (strstr(str
, "MSIE 5.0") != NULL
)
1087 else if (strstr(str
, "MSIE 4.") != NULL
)
1089 else if (strstr(str
, "MSIE 3.") != NULL
)
1096 if (request
->method
== Http::METHOD_OTHER
) {
1101 #if USE_HTTP_VIOLATIONS
1103 if (Config
.onoff
.reload_into_ims
)
1104 request
->flags
.nocacheHack
= true;
1105 else if (refresh_nocache_hack
)
1106 request
->flags
.nocacheHack
= true;
1110 request
->flags
.noCache
= true;
1113 /* ignore range header in non-GETs or non-HEADs */
1114 if (request
->method
== Http::METHOD_GET
|| request
->method
== Http::METHOD_HEAD
) {
1115 // XXX: initialize if we got here without HttpRequest::parseHeader()
1116 if (!request
->range
)
1117 request
->range
= req_hdr
->getRange();
1119 if (request
->range
) {
1120 request
->flags
.isRanged
= true;
1121 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
1122 /* XXX: This is suboptimal. We should give the stream the range set,
1123 * and thereby let the top of the stream set the offset when the
1124 * size becomes known. As it is, we will end up requesting from 0
1125 * for evey -X range specification.
1126 * RBC - this may be somewhat wrong. We should probably set the range
1127 * iter up at this point.
1129 node
->readBuffer
.offset
= request
->range
->lowestOffset(0);
1130 http
->range_iter
.pos
= request
->range
->begin();
1131 http
->range_iter
.end
= request
->range
->end();
1132 http
->range_iter
.valid
= true;
1136 /* Only HEAD and GET requests permit a Range or Request-Range header.
1137 * If these headers appear on any other type of request, delete them now.
1140 req_hdr
->delById(HDR_RANGE
);
1141 req_hdr
->delById(HDR_REQUEST_RANGE
);
1142 request
->ignoreRange("neither HEAD nor GET");
1145 if (req_hdr
->has(HDR_AUTHORIZATION
))
1146 request
->flags
.auth
= true;
1148 clientCheckPinning(http
);
1150 if (!request
->url
.userInfo().isEmpty())
1151 request
->flags
.auth
= true;
1153 if (req_hdr
->has(HDR_VIA
)) {
1154 String s
= req_hdr
->getList(HDR_VIA
);
1156 * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1157 * Note ThisCache2 has a space prepended to the hostname so we don't
1158 * accidentally match super-domains.
1161 if (strListIsSubstr(&s
, ThisCache2
, ',')) {
1162 debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1163 request
, (ObjPackMethod
) & httpRequestPack
);
1164 request
->flags
.loopDetected
= true;
1168 fvdbCountVia(s
.termedBuf());
1177 if (req_hdr
->has(HDR_X_FORWARDED_FOR
)) {
1178 String s
= req_hdr
->getList(HDR_X_FORWARDED_FOR
);
1179 fvdbCountForw(s
.termedBuf());
1185 request
->flags
.cachable
= http
->request
->maybeCacheable();
1187 if (clientHierarchical(http
))
1188 request
->flags
.hierarchical
= true;
1190 debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1191 (request
->flags
.noCache
? "SET" : "NOT SET"));
1192 debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1193 (request
->flags
.cachable
? "SET" : "NOT SET"));
1194 debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1195 (request
->flags
.hierarchical
? "SET" : "NOT SET"));
1200 clientRedirectDoneWrapper(void *data
, const Helper::Reply
&result
)
1202 ClientRequestContext
*calloutContext
= (ClientRequestContext
*)data
;
1204 if (!calloutContext
->httpStateIsValid())
1207 calloutContext
->clientRedirectDone(result
);
1211 clientStoreIdDoneWrapper(void *data
, const Helper::Reply
&result
)
1213 ClientRequestContext
*calloutContext
= (ClientRequestContext
*)data
;
1215 if (!calloutContext
->httpStateIsValid())
1218 calloutContext
->clientStoreIdDone(result
);
1222 ClientRequestContext::clientRedirectDone(const Helper::Reply
&reply
)
1224 HttpRequest
*old_request
= http
->request
;
1225 debugs(85, 5, HERE
<< "'" << http
->uri
<< "' result=" << reply
);
1226 assert(redirect_state
== REDIRECT_PENDING
);
1227 redirect_state
= REDIRECT_DONE
;
1229 // Put helper response Notes into the transaction state record (ALE) eventually
1230 // do it early to ensure that no matter what the outcome the notes are present.
1231 if (http
->al
!= NULL
)
1232 (void)SyncNotes(*http
->al
, *old_request
);
1234 UpdateRequestNotes(http
->getConn(), *old_request
, reply
.notes
);
1236 switch (reply
.result
) {
1237 case Helper::TimedOut
:
1238 if (Config
.onUrlRewriteTimeout
.action
!= toutActBypass
) {
1239 http
->calloutsError(ERR_GATEWAY_FAILURE
, ERR_DETAIL_REDIRECTOR_TIMEDOUT
);
1240 debugs(85, DBG_IMPORTANT
, "ERROR: URL rewrite helper: Timedout");
1244 case Helper::Unknown
:
1246 // Handler in redirect.cc should have already mapped Unknown
1247 // IF it contained valid entry for the old URL-rewrite helper protocol
1248 debugs(85, DBG_IMPORTANT
, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply
);
1251 case Helper::BrokenHelper
:
1252 debugs(85, DBG_IMPORTANT
, "ERROR: URL rewrite helper: " << reply
);
1256 // no change to be done.
1259 case Helper::Okay
: {
1260 // #1: redirect with a specific status code OK status=NNN url="..."
1261 // #2: redirect with a default status code OK url="..."
1262 // #3: re-write the URL OK rewrite-url="..."
1264 const char *statusNote
= reply
.notes
.findFirst("status");
1265 const char *urlNote
= reply
.notes
.findFirst("url");
1267 if (urlNote
!= NULL
) {
1268 // HTTP protocol redirect to be done.
1270 // TODO: change default redirect status for appropriate requests
1271 // Squid defaults to 302 status for now for better compatibility with old clients.
1272 // HTTP/1.0 client should get 302 (Http::scFound)
1273 // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1274 // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1275 Http::StatusCode status
= Http::scFound
;
1276 if (statusNote
!= NULL
) {
1277 const char * result
= statusNote
;
1278 status
= static_cast<Http::StatusCode
>(atoi(result
));
1281 if (status
== Http::scMovedPermanently
1282 || status
== Http::scFound
1283 || status
== Http::scSeeOther
1284 || status
== Http::scPermanentRedirect
1285 || status
== Http::scTemporaryRedirect
) {
1286 http
->redirect
.status
= status
;
1287 http
->redirect
.location
= xstrdup(urlNote
);
1288 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1290 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid " << status
<< " redirect Location: " << urlNote
);
1293 // URL-rewrite wanted. Ew.
1294 urlNote
= reply
.notes
.findFirst("rewrite-url");
1296 // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1297 if (urlNote
!= NULL
&& strcmp(urlNote
, http
->uri
)) {
1298 // XXX: validate the URL properly *without* generating a whole new request object right here.
1299 // XXX: the clone() should be done only AFTER we know the new URL is valid.
1300 HttpRequest
*new_request
= old_request
->clone();
1301 if (urlParse(old_request
->method
, const_cast<char*>(urlNote
), new_request
)) {
1302 debugs(61, 2, "URL-rewriter diverts URL from " << old_request
->effectiveRequestUri() << " to " << new_request
->effectiveRequestUri());
1304 // update the new request to flag the re-writing was done on it
1305 new_request
->flags
.redirected
= true;
1307 // unlink bodypipe from the old request. Not needed there any longer.
1308 if (old_request
->body_pipe
!= NULL
) {
1309 old_request
->body_pipe
= NULL
;
1310 debugs(61,2, HERE
<< "URL-rewriter diverts body_pipe " << new_request
->body_pipe
<<
1311 " from request " << old_request
<< " to " << new_request
);
1314 // update the current working ClientHttpRequest fields
1315 safe_free(http
->uri
);
1316 const SBuf
tmp(new_request
->effectiveRequestUri());
1317 http
->uri
= xstrndup(tmp
.rawContent(), tmp
.length()+1);
1318 HTTPMSGUNLOCK(old_request
);
1319 http
->request
= new_request
;
1320 HTTPMSGLOCK(http
->request
);
1322 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid request: " <<
1323 old_request
->method
<< " " << urlNote
<< " " << old_request
->http_ver
);
1332 /* FIXME PIPELINE: This is innacurate during pipelining */
1334 if (http
->getConn() != NULL
&& Comm::IsConnOpen(http
->getConn()->clientConnection
))
1335 fd_note(http
->getConn()->clientConnection
->fd
, http
->uri
);
1343 * This method handles the different replies from StoreID helper.
1346 ClientRequestContext::clientStoreIdDone(const Helper::Reply
&reply
)
1348 HttpRequest
*old_request
= http
->request
;
1349 debugs(85, 5, "'" << http
->uri
<< "' result=" << reply
);
1350 assert(store_id_state
== REDIRECT_PENDING
);
1351 store_id_state
= REDIRECT_DONE
;
1353 // Put helper response Notes into the transaction state record (ALE) eventually
1354 // do it early to ensure that no matter what the outcome the notes are present.
1355 if (http
->al
!= NULL
)
1356 (void)SyncNotes(*http
->al
, *old_request
);
1358 UpdateRequestNotes(http
->getConn(), *old_request
, reply
.notes
);
1360 switch (reply
.result
) {
1361 case Helper::Unknown
:
1363 // Handler in redirect.cc should have already mapped Unknown
1364 // IF it contained valid entry for the old helper protocol
1365 debugs(85, DBG_IMPORTANT
, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply
);
1368 case Helper::TimedOut
:
1369 // Timeouts for storeID are not implemented
1370 case Helper::BrokenHelper
:
1371 debugs(85, DBG_IMPORTANT
, "ERROR: storeID helper: " << reply
);
1375 // no change to be done.
1378 case Helper::Okay
: {
1379 const char *urlNote
= reply
.notes
.findFirst("store-id");
1381 // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1382 if (urlNote
!= NULL
&& strcmp(urlNote
, http
->uri
) ) {
1383 // Debug section required for some very specific cases.
1384 debugs(85, 9, "Setting storeID with: " << urlNote
);
1385 http
->request
->store_id
= urlNote
;
1386 http
->store_id
= urlNote
;
1395 /** Test cache allow/deny configuration
1396 * Sets flags.cachable=1 if caching is not denied.
1399 ClientRequestContext::checkNoCache()
1401 if (Config
.accessList
.noCache
) {
1402 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.noCache
, http
);
1403 acl_checklist
->nonBlockingCheck(checkNoCacheDoneWrapper
, this);
1405 /* unless otherwise specified, we try to cache. */
1406 checkNoCacheDone(ACCESS_ALLOWED
);
1411 checkNoCacheDoneWrapper(allow_t answer
, void *data
)
1413 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
1415 if (!calloutContext
->httpStateIsValid())
1418 calloutContext
->checkNoCacheDone(answer
);
1422 ClientRequestContext::checkNoCacheDone(const allow_t
&answer
)
1424 acl_checklist
= NULL
;
1425 http
->request
->flags
.cachable
= (answer
== ACCESS_ALLOWED
);
1431 ClientRequestContext::sslBumpAccessCheck()
1433 // If SSL connection tunneling or bumping decision has been made, obey it.
1434 const Ssl::BumpMode bumpMode
= http
->getConn()->sslBumpMode
;
1435 if (bumpMode
!= Ssl::bumpEnd
) {
1436 debugs(85, 5, HERE
<< "SslBump already decided (" << bumpMode
<<
1437 "), " << "ignoring ssl_bump for " << http
->getConn());
1438 if (!http
->getConn()->serverBump())
1439 http
->sslBumpNeed(bumpMode
); // for processRequest() to bump if needed and not already bumped
1440 http
->al
->ssl
.bumpMode
= bumpMode
; // inherited from bumped connection
1444 // If we have not decided yet, decide whether to bump now.
1446 // Bumping here can only start with a CONNECT request on a bumping port
1447 // (bumping of intercepted SSL conns is decided before we get 1st request).
1448 // We also do not bump redirected CONNECT requests.
1449 if (http
->request
->method
!= Http::METHOD_CONNECT
|| http
->redirect
.status
||
1450 !Config
.accessList
.ssl_bump
||
1451 !http
->getConn()->port
->flags
.tunnelSslBumping
) {
1452 http
->al
->ssl
.bumpMode
= Ssl::bumpEnd
; // SslBump does not apply; log -
1453 debugs(85, 5, HERE
<< "cannot SslBump this request");
1457 // Do not bump during authentication: clients would not proxy-authenticate
1458 // if we delay a 407 response and respond with 200 OK to CONNECT.
1459 if (error
&& error
->httpStatus
== Http::scProxyAuthenticationRequired
) {
1460 http
->al
->ssl
.bumpMode
= Ssl::bumpEnd
; // SslBump does not apply; log -
1461 debugs(85, 5, HERE
<< "no SslBump during proxy authentication");
1465 debugs(85, 5, HERE
<< "SslBump possible, checking ACL");
1467 ACLFilledChecklist
*aclChecklist
= clientAclChecklistCreate(Config
.accessList
.ssl_bump
, http
);
1468 aclChecklist
->nonBlockingCheck(sslBumpAccessCheckDoneWrapper
, this);
1473 * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1474 * as ACLFilledChecklist callback
1477 sslBumpAccessCheckDoneWrapper(allow_t answer
, void *data
)
1479 ClientRequestContext
*calloutContext
= static_cast<ClientRequestContext
*>(data
);
1481 if (!calloutContext
->httpStateIsValid())
1483 calloutContext
->sslBumpAccessCheckDone(answer
);
1487 ClientRequestContext::sslBumpAccessCheckDone(const allow_t
&answer
)
1489 if (!httpStateIsValid())
1492 const Ssl::BumpMode bumpMode
= answer
== ACCESS_ALLOWED
?
1493 static_cast<Ssl::BumpMode
>(answer
.kind
) : Ssl::bumpNone
;
1494 http
->sslBumpNeed(bumpMode
); // for processRequest() to bump if needed
1495 http
->al
->ssl
.bumpMode
= bumpMode
; // for logging
1502 * Identify requests that do not go through the store and client side stream
1503 * and forward them to the appropriate location. All other requests, request
1507 ClientHttpRequest::processRequest()
1509 debugs(85, 4, request
->method
<< ' ' << uri
);
1511 if (request
->method
== Http::METHOD_CONNECT
&& !redirect
.status
) {
1513 if (sslBumpNeeded()) {
1518 getConn()->stopReading(); // tunnels read for themselves
1519 tunnelStart(this, &out
.size
, &al
->http
.code
, al
);
1527 ClientHttpRequest::httpStart()
1529 PROF_start(httpStart
);
1530 logType
= LOG_TAG_NONE
;
1531 debugs(85, 4, logType
.c_str() << " for '" << uri
<< "'");
1533 /* no one should have touched this */
1534 assert(out
.offset
== 0);
1535 /* Use the Stream Luke */
1536 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->data
;
1537 clientStreamRead(node
, this, node
->readBuffer
);
1538 PROF_stop(httpStart
);
1544 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode
)
1546 debugs(83, 3, HERE
<< "sslBump required: "<< Ssl::bumpMode(mode
));
1547 sslBumpNeed_
= mode
;
1550 // called when comm_write has completed
1552 SslBumpEstablish(const Comm::ConnectionPointer
&, char *, size_t, Comm::Flag errflag
, int, void *data
)
1554 ClientHttpRequest
*r
= static_cast<ClientHttpRequest
*>(data
);
1555 debugs(85, 5, HERE
<< "responded to CONNECT: " << r
<< " ? " << errflag
);
1557 assert(r
&& cbdataReferenceValid(r
));
1558 r
->sslBumpEstablish(errflag
);
1562 ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag
)
1564 // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1565 if (errflag
== Comm::ERR_CLOSING
)
1569 debugs(85, 3, HERE
<< "CONNECT response failure in SslBump: " << errflag
);
1570 getConn()->clientConnection
->close();
1574 // We lack HttpReply which logRequest() uses to log the status code.
1575 // TODO: Use HttpReply instead of the "200 Connection established" string.
1576 al
->http
.code
= 200;
1579 // Preserve authentication info for the ssl-bumped request
1580 if (request
->auth_user_request
!= NULL
)
1581 getConn()->setAuth(request
->auth_user_request
, "SSL-bumped CONNECT");
1584 assert(sslBumpNeeded());
1585 getConn()->switchToHttps(request
, sslBumpNeed_
);
1589 ClientHttpRequest::sslBumpStart()
1591 debugs(85, 5, HERE
<< "Confirming " << Ssl::bumpMode(sslBumpNeed_
) <<
1592 "-bumped CONNECT tunnel on FD " << getConn()->clientConnection
);
1593 getConn()->sslBumpMode
= sslBumpNeed_
;
1595 AsyncCall::Pointer bumpCall
= commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1596 CommIoCbPtrFun(&SslBumpEstablish
, this));
1598 if (request
->flags
.interceptTproxy
|| request
->flags
.intercepted
) {
1599 CommIoCbParams
¶ms
= GetCommParams
<CommIoCbParams
>(bumpCall
);
1600 params
.flag
= Comm::OK
;
1601 params
.conn
= getConn()->clientConnection
;
1602 ScheduleCallHere(bumpCall
);
1606 // send an HTTP 200 response to kick client SSL negotiation
1607 // TODO: Unify with tunnel.cc and add a Server(?) header
1608 static const char *const conn_established
= "HTTP/1.1 200 Connection established\r\n\r\n";
1609 Comm::Write(getConn()->clientConnection
, conn_established
, strlen(conn_established
), bumpCall
, NULL
);
1615 ClientHttpRequest::gotEnough() const
1617 /** TODO: should be querying the stream. */
1618 int64_t contentLength
=
1619 memObject()->getReply()->bodySize(request
->method
);
1620 assert(contentLength
>= 0);
1622 if (out
.offset
< contentLength
)
1629 ClientHttpRequest::storeEntry(StoreEntry
*newEntry
)
1635 ClientHttpRequest::loggingEntry(StoreEntry
*newEntry
)
1638 loggingEntry_
->unlock("ClientHttpRequest::loggingEntry");
1640 loggingEntry_
= newEntry
;
1643 loggingEntry_
->lock("ClientHttpRequest::loggingEntry");
1647 * doCallouts() - This function controls the order of "callout"
1648 * executions, including non-blocking access control checks, the
1649 * redirector, and ICAP. Previously, these callouts were chained
1650 * together such that "clientAccessCheckDone()" would call
1651 * "clientRedirectStart()" and so on.
1653 * The ClientRequestContext (aka calloutContext) class holds certain
1654 * state data for the callout/callback operations. Previously
1655 * ClientHttpRequest would sort of hand off control to ClientRequestContext
1656 * for a short time. ClientRequestContext would then delete itself
1657 * and pass control back to ClientHttpRequest when all callouts
1660 * This caused some problems for ICAP because we want to make the
1661 * ICAP callout after checking ACLs, but before checking the no_cache
1662 * list. We can't stuff the ICAP state into the ClientRequestContext
1663 * class because we still need the ICAP state after ClientRequestContext
1666 * Note that ClientRequestContext is created before the first call
1669 * If one of the callouts notices that ClientHttpRequest is no
1670 * longer valid, it should call cbdataReferenceDone() so that
1671 * ClientHttpRequest's reference count goes to zero and it will get
1672 * deleted. ClientHttpRequest will then delete ClientRequestContext.
1674 * Note that we set the _done flags here before actually starting
1675 * the callout. This is strictly for convenience.
1678 tos_t
aclMapTOS (acl_tos
* head
, ACLChecklist
* ch
);
1679 nfmark_t
aclMapNfmark (acl_nfmark
* head
, ACLChecklist
* ch
);
1682 ClientHttpRequest::doCallouts()
1684 assert(calloutContext
);
1686 /*Save the original request for logging purposes*/
1687 if (!calloutContext
->http
->al
->request
) {
1688 calloutContext
->http
->al
->request
= request
;
1689 HTTPMSGLOCK(calloutContext
->http
->al
->request
);
1691 NotePairs
¬es
= SyncNotes(*calloutContext
->http
->al
, *calloutContext
->http
->request
);
1692 // Make the previously set client connection ID available as annotation.
1693 if (ConnStateData
*csd
= calloutContext
->http
->getConn()) {
1694 if (!csd
->connectionTag().isEmpty())
1695 notes
.add("clt_conn_tag", SBuf(csd
->connectionTag()).c_str());
1699 if (!calloutContext
->error
) {
1700 // CVE-2009-0801: verify the Host: header is consistent with other known details.
1701 if (!calloutContext
->host_header_verify_done
) {
1702 debugs(83, 3, HERE
<< "Doing calloutContext->hostHeaderVerify()");
1703 calloutContext
->host_header_verify_done
= true;
1704 calloutContext
->hostHeaderVerify();
1708 if (!calloutContext
->http_access_done
) {
1709 debugs(83, 3, HERE
<< "Doing calloutContext->clientAccessCheck()");
1710 calloutContext
->http_access_done
= true;
1711 calloutContext
->clientAccessCheck();
1716 if (!calloutContext
->adaptation_acl_check_done
) {
1717 calloutContext
->adaptation_acl_check_done
= true;
1718 if (Adaptation::AccessCheck::Start(
1719 Adaptation::methodReqmod
, Adaptation::pointPreCache
,
1720 request
, NULL
, calloutContext
->http
->al
, this))
1721 return; // will call callback
1725 if (!calloutContext
->redirect_done
) {
1726 calloutContext
->redirect_done
= true;
1727 assert(calloutContext
->redirect_state
== REDIRECT_NONE
);
1729 if (Config
.Program
.redirect
) {
1730 debugs(83, 3, HERE
<< "Doing calloutContext->clientRedirectStart()");
1731 calloutContext
->redirect_state
= REDIRECT_PENDING
;
1732 calloutContext
->clientRedirectStart();
1737 if (!calloutContext
->adapted_http_access_done
) {
1738 debugs(83, 3, HERE
<< "Doing calloutContext->clientAccessCheck2()");
1739 calloutContext
->adapted_http_access_done
= true;
1740 calloutContext
->clientAccessCheck2();
1744 if (!calloutContext
->store_id_done
) {
1745 calloutContext
->store_id_done
= true;
1746 assert(calloutContext
->store_id_state
== REDIRECT_NONE
);
1748 if (Config
.Program
.store_id
) {
1749 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1750 calloutContext
->store_id_state
= REDIRECT_PENDING
;
1751 calloutContext
->clientStoreIdStart();
1756 if (!calloutContext
->interpreted_req_hdrs
) {
1757 debugs(83, 3, HERE
<< "Doing clientInterpretRequestHeaders()");
1758 calloutContext
->interpreted_req_hdrs
= 1;
1759 clientInterpretRequestHeaders(this);
1762 if (!calloutContext
->no_cache_done
) {
1763 calloutContext
->no_cache_done
= true;
1765 if (Config
.accessList
.noCache
&& request
->flags
.cachable
) {
1766 debugs(83, 3, HERE
<< "Doing calloutContext->checkNoCache()");
1767 calloutContext
->checkNoCache();
1771 } // if !calloutContext->error
1773 if (!calloutContext
->tosToClientDone
) {
1774 calloutContext
->tosToClientDone
= true;
1775 if (getConn() != NULL
&& Comm::IsConnOpen(getConn()->clientConnection
)) {
1776 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1777 ch
.src_addr
= request
->client_addr
;
1778 ch
.my_addr
= request
->my_addr
;
1779 tos_t tos
= aclMapTOS(Ip::Qos::TheConfig
.tosToClient
, &ch
);
1781 Ip::Qos::setSockTos(getConn()->clientConnection
, tos
);
1785 if (!calloutContext
->nfmarkToClientDone
) {
1786 calloutContext
->nfmarkToClientDone
= true;
1787 if (getConn() != NULL
&& Comm::IsConnOpen(getConn()->clientConnection
)) {
1788 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1789 ch
.src_addr
= request
->client_addr
;
1790 ch
.my_addr
= request
->my_addr
;
1791 nfmark_t mark
= aclMapNfmark(Ip::Qos::TheConfig
.nfmarkToClient
, &ch
);
1793 Ip::Qos::setSockNfmark(getConn()->clientConnection
, mark
);
1798 // We need to check for SslBump even if the calloutContext->error is set
1799 // because bumping may require delaying the error until after CONNECT.
1800 if (!calloutContext
->sslBumpCheckDone
) {
1801 calloutContext
->sslBumpCheckDone
= true;
1802 if (calloutContext
->sslBumpAccessCheck())
1804 /* else no ssl bump required*/
1808 if (calloutContext
->error
) {
1809 // XXX: prformance regression. c_str() reallocates
1810 SBuf
storeUri(request
->storeId());
1811 StoreEntry
*e
= storeCreateEntry(storeUri
.c_str(), storeUri
.c_str(), request
->flags
, request
->method
);
1813 if (sslBumpNeeded()) {
1814 // We have to serve an error, so bump the client first.
1815 sslBumpNeed(Ssl::bumpClientFirst
);
1816 // set final error but delay sending until we bump
1817 Ssl::ServerBump
*srvBump
= new Ssl::ServerBump(request
, e
);
1818 errorAppendEntry(e
, calloutContext
->error
);
1819 calloutContext
->error
= NULL
;
1820 getConn()->setServerBump(srvBump
);
1821 e
->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1825 // send the error to the client now
1826 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
1827 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
1828 assert (repContext
);
1829 repContext
->setReplyToStoreEntry(e
, "immediate SslBump error");
1830 errorAppendEntry(e
, calloutContext
->error
);
1831 calloutContext
->error
= NULL
;
1832 if (calloutContext
->readNextRequest
)
1833 getConn()->flags
.readMore
= true; // resume any pipeline reads.
1834 node
= (clientStreamNode
*)client_stream
.tail
->data
;
1835 clientStreamRead(node
, this, node
->readBuffer
);
1836 e
->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1841 cbdataReferenceDone(calloutContext
->http
);
1842 delete calloutContext
;
1843 calloutContext
= NULL
;
1846 headersLog(0, 1, request
->method
, request
);
1849 debugs(83, 3, HERE
<< "calling processRequest()");
1853 Adaptation::Icap::History::Pointer ih
= request
->icapHistory();
1855 ih
->logType
= logType
;
1860 #include "client_side_request.cci"
1864 /// Initiate an asynchronous adaptation transaction which will call us back.
1866 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer
&g
)
1868 debugs(85, 3, HERE
<< "adaptation needed for " << this);
1869 assert(!virginHeadSource
);
1870 assert(!adaptedBodySource
);
1871 virginHeadSource
= initiateAdaptation(
1872 new Adaptation::Iterator(request
, NULL
, al
, g
));
1874 // we could try to guess whether we can bypass this adaptation
1875 // initiation failure, but it should not really happen
1876 Must(initiated(virginHeadSource
));
1880 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer
&answer
)
1882 assert(cbdataReferenceValid(this)); // indicates bug
1883 clearAdaptation(virginHeadSource
);
1884 assert(!adaptedBodySource
);
1886 switch (answer
.kind
) {
1887 case Adaptation::Answer::akForward
:
1888 handleAdaptedHeader(const_cast<HttpMsg
*>(answer
.message
.getRaw()));
1891 case Adaptation::Answer::akBlock
:
1892 handleAdaptationBlock(answer
);
1895 case Adaptation::Answer::akError
:
1896 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT
, !answer
.final
);
1902 ClientHttpRequest::handleAdaptedHeader(HttpMsg
*msg
)
1906 if (HttpRequest
*new_req
= dynamic_cast<HttpRequest
*>(msg
)) {
1908 * Replace the old request with the new request.
1910 HTTPMSGUNLOCK(request
);
1912 HTTPMSGLOCK(request
);
1914 // update the new message to flag whether URL re-writing was done on it
1915 if (request
->effectiveRequestUri().cmp(uri
) != 0)
1916 request
->flags
.redirected
= 1;
1919 * Store the new URI for logging
1922 const SBuf
tmp(request
->effectiveRequestUri());
1923 uri
= xstrndup(tmp
.rawContent(), tmp
.length());
1924 setLogUri(this, urlCanonicalClean(request
));
1925 assert(request
->method
.id());
1926 } else if (HttpReply
*new_rep
= dynamic_cast<HttpReply
*>(msg
)) {
1927 debugs(85,3,HERE
<< "REQMOD reply is HTTP reply");
1929 // subscribe to receive reply body
1930 if (new_rep
->body_pipe
!= NULL
) {
1931 adaptedBodySource
= new_rep
->body_pipe
;
1932 int consumer_ok
= adaptedBodySource
->setConsumerIfNotLate(this);
1933 assert(consumer_ok
);
1936 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
1937 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
1939 repContext
->createStoreEntry(request
->method
, request
->flags
);
1941 EBIT_CLR(storeEntry()->flags
, ENTRY_FWD_HDR_WAIT
);
1942 request_satisfaction_mode
= true;
1943 request_satisfaction_offset
= 0;
1944 storeEntry()->replaceHttpReply(new_rep
);
1945 storeEntry()->timestampsSet();
1947 if (!adaptedBodySource
) // no body
1948 storeEntry()->complete();
1949 clientGetMoreData(node
, this);
1952 // we are done with getting headers (but may be receiving body)
1953 clearAdaptation(virginHeadSource
);
1955 if (!request_satisfaction_mode
)
1960 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer
&answer
)
1962 request
->detailError(ERR_ACCESS_DENIED
, ERR_DETAIL_REQMOD_BLOCK
);
1963 AclMatchedName
= answer
.ruleId
.termedBuf();
1964 assert(calloutContext
);
1965 calloutContext
->clientAccessCheckDone(ACCESS_DENIED
);
1966 AclMatchedName
= NULL
;
1970 ClientHttpRequest::resumeBodyStorage()
1972 if (!adaptedBodySource
)
1975 noteMoreBodyDataAvailable(adaptedBodySource
);
1979 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer
)
1981 assert(request_satisfaction_mode
);
1982 assert(adaptedBodySource
!= NULL
);
1984 if (size_t contentSize
= adaptedBodySource
->buf().contentSize()) {
1985 const size_t spaceAvailable
= storeEntry()->bytesWanted(Range
<size_t>(0,contentSize
));
1987 if (spaceAvailable
< contentSize
) {
1988 // No or partial body data consuming
1989 typedef NullaryMemFunT
<ClientHttpRequest
> Dialer
;
1990 AsyncCall::Pointer call
= asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1991 Dialer(this, &ClientHttpRequest::resumeBodyStorage
));
1992 storeEntry()->deferProducer(call
);
1995 if (!spaceAvailable
)
1998 if (spaceAvailable
< contentSize
)
1999 contentSize
= spaceAvailable
;
2001 BodyPipeCheckout
bpc(*adaptedBodySource
);
2002 const StoreIOBuffer
ioBuf(&bpc
.buf
, request_satisfaction_offset
, contentSize
);
2003 storeEntry()->write(ioBuf
);
2004 // assume StoreEntry::write() writes the entire ioBuf
2005 request_satisfaction_offset
+= ioBuf
.length
;
2006 bpc
.buf
.consume(contentSize
);
2010 if (adaptedBodySource
->exhausted())
2011 endRequestSatisfaction();
2012 // else wait for more body data
2016 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer
)
2018 assert(!virginHeadSource
);
2019 // should we end request satisfaction now?
2020 if (adaptedBodySource
!= NULL
&& adaptedBodySource
->exhausted())
2021 endRequestSatisfaction();
2025 ClientHttpRequest::endRequestSatisfaction()
2027 debugs(85,4, HERE
<< this << " ends request satisfaction");
2028 assert(request_satisfaction_mode
);
2029 stopConsumingFrom(adaptedBodySource
);
2031 // TODO: anything else needed to end store entry formation correctly?
2032 storeEntry()->complete();
2036 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer
)
2038 assert(!virginHeadSource
);
2039 stopConsumingFrom(adaptedBodySource
);
2041 debugs(85,3, HERE
<< "REQMOD body production failed");
2042 if (request_satisfaction_mode
) { // too late to recover or serve an error
2043 request
->detailError(ERR_ICAP_FAILURE
, ERR_DETAIL_CLT_REQMOD_RESP_BODY
);
2044 const Comm::ConnectionPointer c
= getConn()->clientConnection
;
2045 Must(Comm::IsConnOpen(c
));
2046 c
->close(); // drastic, but we may be writing a response already
2048 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY
);
2053 ClientHttpRequest::handleAdaptationFailure(int errDetail
, bool bypassable
)
2055 debugs(85,3, HERE
<< "handleAdaptationFailure(" << bypassable
<< ")");
2057 const bool usedStore
= storeEntry() && !storeEntry()->isEmpty();
2058 const bool usedPipe
= request
->body_pipe
!= NULL
&&
2059 request
->body_pipe
->consumedSize() > 0;
2061 if (bypassable
&& !usedStore
&& !usedPipe
) {
2062 debugs(85,3, HERE
<< "ICAP REQMOD callout failed, bypassing: " << calloutContext
);
2068 debugs(85,3, HERE
<< "ICAP REQMOD callout failed, responding with error");
2070 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
2071 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2074 calloutsError(ERR_ICAP_FAILURE
, errDetail
);
2082 // XXX: modify and use with ClientRequestContext::clientAccessCheckDone too.
2084 ClientHttpRequest::calloutsError(const err_type error
, const int errDetail
)
2086 // The original author of the code also wanted to pass an errno to
2087 // setReplyToError, but it seems unlikely that the errno reflects the
2088 // true cause of the error at this point, so I did not pass it.
2089 if (calloutContext
) {
2092 ConnStateData
* c
= getConn();
2093 calloutContext
->error
= clientBuildError(error
, Http::scInternalServerError
,
2095 c
!= NULL
? c
->clientConnection
->remote
: noAddr
,
2099 calloutContext
->error
->auth_user_request
=
2100 c
!= NULL
&& c
->getAuth() != NULL
? c
->getAuth() : request
->auth_user_request
;
2102 calloutContext
->error
->detailError(errDetail
);
2103 calloutContext
->readNextRequest
= true;
2105 c
->expectNoForwarding();
2107 //else if(calloutContext == NULL) is it possible?