3 * DEBUG: section 85 Client-side Request Routines
4 * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
6 * SQUID Web Proxy Cache http://www.squid-cache.org/
7 * ----------------------------------------------------------
9 * Squid is the result of efforts by numerous individuals from
10 * the Internet community; see the CONTRIBUTORS file for full
11 * details. Many organizations have provided support for Squid's
12 * development; see the SPONSORS file for full details. Squid is
13 * Copyrighted (C) 2001 by the Regents of the University of
14 * California; see the COPYRIGHT file for full details. Squid
15 * incorporates software developed and/or copyrighted by other
16 * sources; see the CREDITS file for full details.
18 * This program is free software; you can redistribute it and/or modify
19 * it under the terms of the GNU General Public License as published by
20 * the Free Software Foundation; either version 2 of the License, or
21 * (at your option) any later version.
23 * This program is distributed in the hope that it will be useful,
24 * but WITHOUT ANY WARRANTY; without even the implied warranty of
25 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 * GNU General Public License for more details.
28 * You should have received a copy of the GNU General Public License
29 * along with this program; if not, write to the Free Software
30 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
35 * General logic of request processing:
37 * We run a series of tests to determine if access will be permitted, and to do
38 * any redirection. Then we call into the result clientStream to retrieve data.
39 * From that point on it's up to reply management.
43 #include "acl/FilledChecklist.h"
44 #include "acl/Gadgets.h"
45 #include "anyp/PortCfg.h"
46 #include "ClientRequestContext.h"
47 #include "client_side.h"
48 #include "client_side_reply.h"
49 #include "client_side_request.h"
50 #include "clientStream.h"
51 #include "comm/Connection.h"
52 #include "comm/Write.h"
53 #include "compat/inet_pton.h"
54 #include "err_detail_type.h"
55 #include "errorpage.h"
58 #include "format/Token.h"
62 #include "HttpHdrCc.h"
63 #include "HttpReply.h"
64 #include "HttpRequest.h"
66 #include "ip/QosConfig.h"
67 #include "log/access_log.h"
68 #include "MemObject.h"
70 #include "profiler/Profiler.h"
72 #include "SquidConfig.h"
73 #include "SquidTime.h"
80 #include "auth/UserRequest.h"
83 #include "adaptation/AccessCheck.h"
84 #include "adaptation/Answer.h"
85 #include "adaptation/Iterator.h"
86 #include "adaptation/Service.h"
88 #include "adaptation/icap/History.h"
92 #include "ssl/support.h"
93 #include "ssl/ServerBump.h"
97 #define comm_close comm_lingering_close
100 static const char *const crlf
= "\r\n";
102 #if FOLLOW_X_FORWARDED_FOR
103 static void clientFollowXForwardedForCheck(allow_t answer
, void *data
);
104 #endif /* FOLLOW_X_FORWARDED_FOR */
106 ErrorState
*clientBuildError(err_type
, Http::StatusCode
, char const *url
, Ip::Address
&, HttpRequest
*);
108 CBDATA_CLASS_INIT(ClientRequestContext
);
111 ClientRequestContext::operator new (size_t size
)
113 assert (size
== sizeof(ClientRequestContext
));
114 CBDATA_INIT_TYPE(ClientRequestContext
);
115 ClientRequestContext
*result
= cbdataAlloc(ClientRequestContext
);
120 ClientRequestContext::operator delete (void *address
)
122 ClientRequestContext
*t
= static_cast<ClientRequestContext
*>(address
);
126 /* Local functions */
128 static void clientAccessCheckDoneWrapper(allow_t
, void *);
130 static void sslBumpAccessCheckDoneWrapper(allow_t
, void *);
132 static int clientHierarchical(ClientHttpRequest
* http
);
133 static void clientInterpretRequestHeaders(ClientHttpRequest
* http
);
134 static HLPCB clientRedirectDoneWrapper
;
135 static HLPCB clientStoreIdDoneWrapper
;
136 static void checkNoCacheDoneWrapper(allow_t
, void *);
137 SQUIDCEXTERN CSR clientGetMoreData
;
138 SQUIDCEXTERN CSS clientReplyStatus
;
139 SQUIDCEXTERN CSD clientReplyDetach
;
140 static void checkFailureRatio(err_type
, hier_code
);
142 ClientRequestContext::~ClientRequestContext()
145 * Release our "lock" on our parent, ClientHttpRequest, if we
150 cbdataReferenceDone(http
);
153 debugs(85,3, HERE
<< this << " ClientRequestContext destructed");
156 ClientRequestContext::ClientRequestContext(ClientHttpRequest
*anHttp
) : http(cbdataReference(anHttp
)), acl_checklist (NULL
), redirect_state (REDIRECT_NONE
), store_id_state(REDIRECT_NONE
),error(NULL
), readNextRequest(false)
158 http_access_done
= false;
159 redirect_done
= false;
160 redirect_fail_count
= 0;
161 store_id_done
= false;
162 store_id_fail_count
= 0;
163 no_cache_done
= false;
164 interpreted_req_hdrs
= false;
166 sslBumpCheckDone
= false;
168 debugs(85,3, HERE
<< this << " ClientRequestContext constructed");
171 CBDATA_CLASS_INIT(ClientHttpRequest
);
174 ClientHttpRequest::operator new (size_t size
)
176 assert (size
== sizeof (ClientHttpRequest
));
177 CBDATA_INIT_TYPE(ClientHttpRequest
);
178 ClientHttpRequest
*result
= cbdataAlloc(ClientHttpRequest
);
183 ClientHttpRequest::operator delete (void *address
)
185 ClientHttpRequest
*t
= static_cast<ClientHttpRequest
*>(address
);
189 ClientHttpRequest::ClientHttpRequest(ConnStateData
* aConn
) :
191 AsyncJob("ClientHttpRequest"),
195 start_time
= current_time
;
197 al
= new AccessLogEntry
;
198 al
->tcpClient
= clientConnection
= aConn
->clientConnection
;
200 if (aConn
->clientConnection
!= NULL
&& aConn
->clientConnection
->isOpen()) {
201 if (SSL
*ssl
= fd_table
[aConn
->clientConnection
->fd
].ssl
)
202 al
->cache
.sslClientCert
.reset(SSL_get_peer_certificate(ssl
));
205 dlinkAdd(this, &active
, &ClientActiveRequests
);
207 request_satisfaction_mode
= false;
210 sslBumpNeed_
= Ssl::bumpEnd
;
215 * returns true if client specified that the object must come from the cache
216 * without contacting origin server
219 ClientHttpRequest::onlyIfCached()const
222 return request
->cache_control
&&
223 request
->cache_control
->onlyIfCached();
227 * This function is designed to serve a fairly specific purpose.
228 * Occasionally our vBNS-connected caches can talk to each other, but not
229 * the rest of the world. Here we try to detect frequent failures which
230 * make the cache unusable (e.g. DNS lookup and connect() failures). If
231 * the failure:success ratio goes above 1.0 then we go into "hit only"
232 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors
233 * will only fetch HITs from us if they are using the ICP protocol. We
234 * stay in this mode for 5 minutes.
236 * Duane W., Sept 16, 1996
239 checkFailureRatio(err_type etype
, hier_code hcode
)
241 // Can be set at compile time with -D compiler flag
242 #ifndef FAILURE_MODE_TIME
243 #define FAILURE_MODE_TIME 300
246 if (hcode
== HIER_NONE
)
249 // don't bother when ICP is disabled.
250 if (Config
.Port
.icp
<= 0)
253 static double magic_factor
= 100.0;
257 n_good
= magic_factor
/ (1.0 + request_failure_ratio
);
259 n_bad
= magic_factor
- n_good
;
265 case ERR_CONNECT_FAIL
:
266 case ERR_SECURE_CONNECT_FAIL
:
276 request_failure_ratio
= n_bad
/ n_good
;
278 if (hit_only_mode_until
> squid_curtime
)
281 if (request_failure_ratio
< 1.0)
284 debugs(33, DBG_CRITICAL
, "WARNING: Failure Ratio at "<< std::setw(4)<<
285 std::setprecision(3) << request_failure_ratio
);
287 debugs(33, DBG_CRITICAL
, "WARNING: ICP going into HIT-only mode for " <<
288 FAILURE_MODE_TIME
/ 60 << " minutes...");
290 hit_only_mode_until
= squid_curtime
+ FAILURE_MODE_TIME
;
292 request_failure_ratio
= 0.8; /* reset to something less than 1.0 */
295 ClientHttpRequest::~ClientHttpRequest()
297 debugs(33, 3, "httpRequestFree: " << uri
);
298 PROF_start(httpRequestFree
);
300 // Even though freeResources() below may destroy the request,
301 // we no longer set request->body_pipe to NULL here
302 // because we did not initiate that pipe (ConnStateData did)
304 /* the ICP check here was erroneous
305 * - StoreEntry::releaseRequest was always called if entry was valid
307 assert(logType
< LOG_TYPE_MAX
);
314 checkFailureRatio(request
->errType
, al
->hier
.code
);
319 announceInitiatorAbort(virginHeadSource
);
321 if (adaptedBodySource
!= NULL
)
322 stopConsumingFrom(adaptedBodySource
);
326 delete calloutContext
;
328 clientConnection
= NULL
;
331 cbdataReferenceDone(conn_
);
333 /* moving to the next connection is handled by the context free */
334 dlinkDelete(&active
, &ClientActiveRequests
);
336 PROF_stop(httpRequestFree
);
340 * Create a request and kick it off
345 * TODO: Pass in the buffers to be used in the inital Read request, as they are
346 * determined by the user
349 clientBeginRequest(const HttpRequestMethod
& method
, char const *url
, CSCB
* streamcallback
,
350 CSD
* streamdetach
, ClientStreamData streamdata
, HttpHeader
const *header
,
351 char *tailbuf
, size_t taillen
)
354 ClientHttpRequest
*http
= new ClientHttpRequest(NULL
);
355 HttpRequest
*request
;
356 StoreIOBuffer tempBuffer
;
357 http
->start_time
= current_time
;
358 /* this is only used to adjust the connection offset in client_side.c */
360 tempBuffer
.length
= taillen
;
361 tempBuffer
.data
= tailbuf
;
362 /* client stream setup */
363 clientStreamInit(&http
->client_stream
, clientGetMoreData
, clientReplyDetach
,
364 clientReplyStatus
, new clientReplyContext(http
), streamcallback
,
365 streamdetach
, streamdata
, tempBuffer
);
366 /* make it visible in the 'current acctive requests list' */
368 /* internal requests only makes sense in an
369 * accelerator today. TODO: accept flags ? */
370 http
->flags
.accel
= true;
371 /* allow size for url rewriting */
372 url_sz
= strlen(url
) + Config
.appendDomainLen
+ 5;
373 http
->uri
= (char *)xcalloc(url_sz
, 1);
374 strcpy(http
->uri
, url
);
376 if ((request
= HttpRequest::CreateFromUrlAndMethod(http
->uri
, method
)) == NULL
) {
377 debugs(85, 5, "Invalid URL: " << http
->uri
);
382 * now update the headers in request with our supplied headers. urlParse
383 * should return a blank header set, but we use Update to be sure of
387 request
->header
.update(header
, NULL
);
389 http
->log_uri
= xstrdup(urlCanonicalClean(request
));
391 /* http struct now ready */
394 * build new header list *? TODO
396 request
->flags
.accelerated
= http
->flags
.accel
;
398 request
->flags
.internalClient
= true;
400 /* this is an internally created
401 * request, not subject to acceleration
402 * target overrides */
404 * FIXME? Do we want to detect and handle internal requests of internal
408 /* Internally created requests cannot have bodies today */
409 request
->content_length
= 0;
411 request
->client_addr
.SetNoAddr();
413 #if FOLLOW_X_FORWARDED_FOR
414 request
->indirect_client_addr
.SetNoAddr();
415 #endif /* FOLLOW_X_FORWARDED_FOR */
417 request
->my_addr
.SetNoAddr(); /* undefined for internal requests */
419 request
->my_addr
.SetPort(0);
421 /* Our version is HTTP/1.1 */
422 request
->http_ver
= Http::ProtocolVersion(1,1);
424 http
->request
= request
;
425 HTTPMSGLOCK(http
->request
);
427 /* optional - skip the access check ? */
428 http
->calloutContext
= new ClientRequestContext(http
);
430 http
->calloutContext
->http_access_done
= false;
432 http
->calloutContext
->redirect_done
= true;
434 http
->calloutContext
->no_cache_done
= true;
442 ClientRequestContext::httpStateIsValid()
444 ClientHttpRequest
*http_
= http
;
446 if (cbdataReferenceValid(http_
))
451 cbdataReferenceDone(http_
);
456 #if FOLLOW_X_FORWARDED_FOR
458 * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
459 * against the followXFF ACL, or cleans up and passes control to
460 * clientAccessCheck().
462 * The trust model here is a little ambiguous. So to clarify the logic:
463 * - we may always use the direct client address as the client IP.
464 * - these trust tests merey tell whether we trust given IP enough to believe the
465 * IP string which it appended to the X-Forwarded-For: header.
466 * - if at any point we don't trust what an IP adds we stop looking.
467 * - at that point the current contents of indirect_client_addr are the value set
468 * by the last previously trusted IP.
469 * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
472 clientFollowXForwardedForCheck(allow_t answer
, void *data
)
474 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
476 if (!calloutContext
->httpStateIsValid())
479 ClientHttpRequest
*http
= calloutContext
->http
;
480 HttpRequest
*request
= http
->request
;
483 * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
484 * called as a result of ACL checks, or -1 if we are called when
485 * there's nothing left to do.
487 if (answer
== ACCESS_ALLOWED
&&
488 request
->x_forwarded_for_iterator
.size () != 0) {
491 * Remove the last comma-delimited element from the
492 * x_forwarded_for_iterator and use it to repeat the cycle.
495 const char *asciiaddr
;
498 p
= request
->x_forwarded_for_iterator
.termedBuf();
499 l
= request
->x_forwarded_for_iterator
.size();
502 * XXX x_forwarded_for_iterator should really be a list of
503 * IP addresses, but it's a String instead. We have to
504 * walk backwards through the String, biting off the last
505 * comma-delimited part each time. As long as the data is in
506 * a String, we should probably implement and use a variant of
507 * strListGetItem() that walks backwards instead of forwards
508 * through a comma-separated list. But we don't even do that;
509 * we just do the work in-line here.
511 /* skip trailing space and commas */
512 while (l
> 0 && (p
[l
-1] == ',' || xisspace(p
[l
-1])))
514 request
->x_forwarded_for_iterator
.cut(l
);
515 /* look for start of last item in list */
516 while (l
> 0 && ! (p
[l
-1] == ',' || xisspace(p
[l
-1])))
519 if ((addr
= asciiaddr
)) {
520 request
->indirect_client_addr
= addr
;
521 request
->x_forwarded_for_iterator
.cut(l
);
522 calloutContext
->acl_checklist
= clientAclChecklistCreate(Config
.accessList
.followXFF
, http
);
523 if (!Config
.onoff
.acl_uses_indirect_client
) {
524 /* override the default src_addr tested if we have to go deeper than one level into XFF */
525 Filled(calloutContext
->acl_checklist
)->src_addr
= request
->indirect_client_addr
;
527 calloutContext
->acl_checklist
->nonBlockingCheck(clientFollowXForwardedForCheck
, data
);
530 } /*if (answer == ACCESS_ALLOWED &&
531 request->x_forwarded_for_iterator.size () != 0)*/
533 /* clean up, and pass control to clientAccessCheck */
534 if (Config
.onoff
.log_uses_indirect_client
) {
536 * Ensure that the access log shows the indirect client
537 * instead of the direct client.
539 ConnStateData
*conn
= http
->getConn();
540 conn
->log_addr
= request
->indirect_client_addr
;
542 request
->x_forwarded_for_iterator
.clean();
543 request
->flags
.done_follow_x_forwarded_for
= true;
545 if (answer
!= ACCESS_ALLOWED
&& answer
!= ACCESS_DENIED
) {
546 debugs(28, DBG_CRITICAL
, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request
->indirect_client_addr
);
549 /* process actual access ACL as normal. */
550 calloutContext
->clientAccessCheck();
552 #endif /* FOLLOW_X_FORWARDED_FOR */
555 hostHeaderIpVerifyWrapper(const ipcache_addrs
* ia
, const DnsLookupDetails
&dns
, void *data
)
557 ClientRequestContext
*c
= static_cast<ClientRequestContext
*>(data
);
558 c
->hostHeaderIpVerify(ia
, dns
);
562 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs
* ia
, const DnsLookupDetails
&dns
)
564 Comm::ConnectionPointer clientConn
= http
->getConn()->clientConnection
;
566 // note the DNS details for the transaction stats.
567 http
->request
->recordLookup(dns
);
569 if (ia
!= NULL
&& ia
->count
> 0) {
570 // Is the NAT destination IP in DNS?
571 for (int i
= 0; i
< ia
->count
; ++i
) {
572 if (clientConn
->local
.matchIPAddr(ia
->in_addrs
[i
]) == 0) {
573 debugs(85, 3, HERE
<< "validate IP " << clientConn
->local
<< " possible from Host:");
574 http
->request
->flags
.hostVerified
= true;
578 debugs(85, 3, HERE
<< "validate IP " << clientConn
->local
<< " non-match from Host: IP " << ia
->in_addrs
[i
]);
581 debugs(85, 3, HERE
<< "FAIL: validate IP " << clientConn
->local
<< " possible from Host:");
582 hostHeaderVerifyFailed("local IP", "any domain IP");
586 ClientRequestContext::hostHeaderVerifyFailed(const char *A
, const char *B
)
588 // IP address validation for Host: failed. Admin wants to ignore them.
589 // NP: we do not yet handle CONNECT tunnels well, so ignore for them
590 if (!Config
.onoff
.hostStrictVerify
&& http
->request
->method
!= Http::METHOD_CONNECT
) {
591 debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http
->getConn()->clientConnection
<<
592 " (" << A
<< " does not match " << B
<< ") on URL: " << urlCanonical(http
->request
));
594 // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
595 // The problems here are about WRITE for new cache content, which means flags.cachable
596 http
->request
->flags
.cachable
= false; // MUST NOT cache (for now)
597 // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
598 http
->request
->flags
.hierarchical
= false; // MUST NOT pass to peers (for now)
599 // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
604 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: Host header forgery detected on " <<
605 http
->getConn()->clientConnection
<< " (" << A
<< " does not match " << B
<< ")");
606 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: By user agent: " << http
->request
->header
.getStr(HDR_USER_AGENT
));
607 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: on URL: " << urlCanonical(http
->request
));
609 // IP address validation for Host: failed. reject the connection.
610 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->prev
->data
;
611 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
613 repContext
->setReplyToError(ERR_CONFLICT_HOST
, Http::scConflict
,
614 http
->request
->method
, NULL
,
615 http
->getConn()->clientConnection
->remote
,
619 http
->getConn() != NULL
&& http
->getConn()->auth_user_request
!= NULL
?
620 http
->getConn()->auth_user_request
: http
->request
->auth_user_request
);
624 node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
625 clientStreamRead(node
, http
, node
->readBuffer
);
629 ClientRequestContext::hostHeaderVerify()
631 // Require a Host: header.
632 const char *host
= http
->request
->header
.getStr(HDR_HOST
);
635 // TODO: dump out the HTTP/1.1 error about missing host header.
636 // otherwise this is fine, can't forge a header value when its not even set.
637 debugs(85, 3, HERE
<< "validate skipped with no Host: header present.");
642 if (http
->request
->flags
.internal
) {
643 // TODO: kill this when URL handling allows partial URLs out of accel mode
644 // and we no longer screw with the URL just to add our internal host there
645 debugs(85, 6, HERE
<< "validate skipped due to internal composite URL.");
650 // Locate if there is a port attached, strip ready for IP lookup
651 char *portStr
= NULL
;
652 char *hostB
= xstrdup(host
);
654 if (host
[0] == '[') {
656 portStr
= strchr(hostB
, ']');
657 if (portStr
&& *(++portStr
) != ':') {
661 // Domain or IPv4 literal with port
662 portStr
= strrchr(hostB
, ':');
667 *portStr
= '\0'; // strip the ':'
668 if (*(++portStr
) != '\0')
669 port
= xatoi(portStr
);
672 debugs(85, 3, HERE
<< "validate host=" << host
<< ", port=" << port
<< ", portStr=" << (portStr
?portStr
:"NULL"));
673 if (http
->request
->flags
.intercepted
|| http
->request
->flags
.spoofClientIp
) {
674 // verify the Host: port (if any) matches the apparent destination
675 if (portStr
&& port
!= http
->getConn()->clientConnection
->local
.GetPort()) {
676 debugs(85, 3, HERE
<< "FAIL on validate port " << http
->getConn()->clientConnection
->local
.GetPort() <<
677 " matches Host: port " << port
<< " (" << portStr
<< ")");
678 hostHeaderVerifyFailed("intercepted port", portStr
);
680 // XXX: match the scheme default port against the apparent destination
682 // verify the destination DNS is one of the Host: headers IPs
683 ipcache_nbgethostbyname(host
, hostHeaderIpVerifyWrapper
, this);
685 } else if (!Config
.onoff
.hostStrictVerify
) {
686 debugs(85, 3, HERE
<< "validate skipped.");
688 } else if (strlen(host
) != strlen(http
->request
->GetHost())) {
689 // Verify forward-proxy requested URL domain matches the Host: header
690 debugs(85, 3, HERE
<< "FAIL on validate URL domain length " << http
->request
->GetHost() << " matches Host: " << host
);
691 hostHeaderVerifyFailed(host
, http
->request
->GetHost());
692 } else if (matchDomainName(host
, http
->request
->GetHost()) != 0) {
693 // Verify forward-proxy requested URL domain matches the Host: header
694 debugs(85, 3, HERE
<< "FAIL on validate URL domain " << http
->request
->GetHost() << " matches Host: " << host
);
695 hostHeaderVerifyFailed(host
, http
->request
->GetHost());
696 } else if (portStr
&& port
!= http
->request
->port
) {
697 // Verify forward-proxy requested URL domain matches the Host: header
698 debugs(85, 3, HERE
<< "FAIL on validate URL port " << http
->request
->port
<< " matches Host: port " << portStr
);
699 hostHeaderVerifyFailed("URL port", portStr
);
700 } else if (!portStr
&& http
->request
->method
!= Http::METHOD_CONNECT
&& http
->request
->port
!= urlDefaultPort(http
->request
->protocol
)) {
701 // Verify forward-proxy requested URL domain matches the Host: header
702 // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
703 debugs(85, 3, HERE
<< "FAIL on validate URL port " << http
->request
->port
<< " matches Host: default port " << urlDefaultPort(http
->request
->protocol
));
704 hostHeaderVerifyFailed("URL port", "default port");
707 debugs(85, 3, HERE
<< "validate passed.");
708 http
->request
->flags
.hostVerified
= true;
714 /* This is the entry point for external users of the client_side routines */
716 ClientRequestContext::clientAccessCheck()
718 #if FOLLOW_X_FORWARDED_FOR
719 if (!http
->request
->flags
.doneFollowXff() &&
720 Config
.accessList
.followXFF
&&
721 http
->request
->header
.has(HDR_X_FORWARDED_FOR
)) {
723 /* we always trust the direct client address for actual use */
724 http
->request
->indirect_client_addr
= http
->request
->client_addr
;
725 http
->request
->indirect_client_addr
.SetPort(0);
727 /* setup the XFF iterator for processing */
728 http
->request
->x_forwarded_for_iterator
= http
->request
->header
.getList(HDR_X_FORWARDED_FOR
);
730 /* begin by checking to see if we trust direct client enough to walk XFF */
731 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.followXFF
, http
);
732 acl_checklist
->nonBlockingCheck(clientFollowXForwardedForCheck
, this);
737 if (Config
.accessList
.http
) {
738 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.http
, http
);
739 acl_checklist
->nonBlockingCheck(clientAccessCheckDoneWrapper
, this);
741 debugs(0, DBG_CRITICAL
, "No http_access configuration found. This will block ALL traffic");
742 clientAccessCheckDone(ACCESS_DENIED
);
747 * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
748 * The default here is to allow all. Since the earlier http_access should do a default deny all.
749 * This check is just for a last-minute denial based on adapted request headers.
752 ClientRequestContext::clientAccessCheck2()
754 if (Config
.accessList
.adapted_http
) {
755 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.adapted_http
, http
);
756 acl_checklist
->nonBlockingCheck(clientAccessCheckDoneWrapper
, this);
758 debugs(85, 2, HERE
<< "No adapted_http_access configuration. default: ALLOW");
759 clientAccessCheckDone(ACCESS_ALLOWED
);
764 clientAccessCheckDoneWrapper(allow_t answer
, void *data
)
766 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
768 if (!calloutContext
->httpStateIsValid())
771 calloutContext
->clientAccessCheckDone(answer
);
775 ClientRequestContext::clientAccessCheckDone(const allow_t
&answer
)
777 acl_checklist
= NULL
;
779 Http::StatusCode status
;
780 debugs(85, 2, "The request " <<
781 RequestMethodStr(http
->request
->method
) << " " <<
782 http
->uri
<< " is " << answer
<<
783 ", because it matched '" <<
784 (AclMatchedName
? AclMatchedName
: "NO ACL's") << "'" );
787 char const *proxy_auth_msg
= "<null>";
788 if (http
->getConn() != NULL
&& http
->getConn()->auth_user_request
!= NULL
)
789 proxy_auth_msg
= http
->getConn()->auth_user_request
->denyMessage("<null>");
790 else if (http
->request
->auth_user_request
!= NULL
)
791 proxy_auth_msg
= http
->request
->auth_user_request
->denyMessage("<null>");
794 if (answer
!= ACCESS_ALLOWED
) {
795 // auth has a grace period where credentials can be expired but okay not to challenge.
797 /* Send an auth challenge or error */
798 // XXX: do we still need aclIsProxyAuth() ?
799 bool auth_challenge
= (answer
== ACCESS_AUTH_REQUIRED
|| aclIsProxyAuth(AclMatchedName
));
800 debugs(85, 5, "Access Denied: " << http
->uri
);
801 debugs(85, 5, "AclMatchedName = " << (AclMatchedName
? AclMatchedName
: "<null>"));
804 debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg
? proxy_auth_msg
: "<null>"));
808 * NOTE: get page_id here, based on AclMatchedName because if
809 * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
810 * the clientCreateStoreEntry() call just below. Pedro Ribeiro
813 page_id
= aclGetDenyInfoPage(&Config
.denyInfoList
, AclMatchedName
, answer
!= ACCESS_AUTH_REQUIRED
);
815 http
->logType
= LOG_TCP_DENIED
;
817 if (auth_challenge
) {
819 if (http
->request
->flags
.sslBumped
) {
820 /*SSL Bumped request, authentication is not possible*/
821 status
= Http::scForbidden
;
822 } else if (!http
->flags
.accel
) {
823 /* Proxy authorisation needed */
824 status
= Http::scProxyAuthenticationRequired
;
826 /* WWW authorisation needed */
827 status
= Http::scUnauthorized
;
830 // need auth, but not possible to do.
831 status
= Http::scForbidden
;
833 if (page_id
== ERR_NONE
)
834 page_id
= ERR_CACHE_ACCESS_DENIED
;
836 status
= Http::scForbidden
;
838 if (page_id
== ERR_NONE
)
839 page_id
= ERR_ACCESS_DENIED
;
842 Ip::Address tmpnoaddr
;
843 tmpnoaddr
.SetNoAddr();
844 error
= clientBuildError(page_id
, status
,
846 http
->getConn() != NULL
? http
->getConn()->clientConnection
->remote
: tmpnoaddr
,
851 error
->auth_user_request
=
852 http
->getConn() != NULL
&& http
->getConn()->auth_user_request
!= NULL
?
853 http
->getConn()->auth_user_request
: http
->request
->auth_user_request
;
856 readNextRequest
= true;
859 /* ACCESS_ALLOWED continues here ... */
860 safe_free(http
->uri
);
862 http
->uri
= xstrdup(urlCanonical(http
->request
));
869 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g
)
871 debugs(93,3,HERE
<< this << " adaptationAclCheckDone called");
874 Adaptation::Icap::History::Pointer ih
= request
->icapHistory();
876 if (getConn() != NULL
) {
877 ih
->rfc931
= getConn()->clientConnection
->rfc931
;
879 assert(getConn()->clientConnection
!= NULL
);
880 ih
->ssluser
= sslGetUserEmail(fd_table
[getConn()->clientConnection
->fd
].ssl
);
883 ih
->log_uri
= log_uri
;
889 debugs(85,3, HERE
<< "no adaptation needed");
900 clientRedirectAccessCheckDone(allow_t answer
, void *data
)
902 ClientRequestContext
*context
= (ClientRequestContext
*)data
;
903 ClientHttpRequest
*http
= context
->http
;
904 context
->acl_checklist
= NULL
;
906 if (answer
== ACCESS_ALLOWED
)
907 redirectStart(http
, clientRedirectDoneWrapper
, context
);
909 HelperReply nilReply
;
910 context
->clientRedirectDone(nilReply
);
915 ClientRequestContext::clientRedirectStart()
917 debugs(33, 5, HERE
<< "'" << http
->uri
<< "'");
919 if (Config
.accessList
.redirector
) {
920 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.redirector
, http
);
921 acl_checklist
->nonBlockingCheck(clientRedirectAccessCheckDone
, this);
923 redirectStart(http
, clientRedirectDoneWrapper
, this);
927 * This methods handles Access checks result of StoreId access list.
928 * Will handle as "ERR" (no change) in a case Access is not allowed.
931 clientStoreIdAccessCheckDone(allow_t answer
, void *data
)
933 ClientRequestContext
*context
= static_cast<ClientRequestContext
*>(data
);
934 ClientHttpRequest
*http
= context
->http
;
935 context
->acl_checklist
= NULL
;
937 if (answer
== ACCESS_ALLOWED
)
938 storeIdStart(http
, clientStoreIdDoneWrapper
, context
);
940 debugs(85, 3, "access denied expected ERR reply handling: " << answer
);
941 HelperReply nilReply
;
942 nilReply
.result
= HelperReply::Error
;
943 context
->clientStoreIdDone(nilReply
);
948 * Start locating an alternative storeage ID string (if any) from admin
949 * configured helper program. This is an asynchronous operation terminating in
950 * ClientRequestContext::clientStoreIdDone() when completed.
953 ClientRequestContext::clientStoreIdStart()
955 debugs(33, 5,"'" << http
->uri
<< "'");
957 if (Config
.accessList
.store_id
) {
958 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.store_id
, http
);
959 acl_checklist
->nonBlockingCheck(clientStoreIdAccessCheckDone
, this);
961 storeIdStart(http
, clientStoreIdDoneWrapper
, this);
965 clientHierarchical(ClientHttpRequest
* http
)
967 const char *url
= http
->uri
;
968 HttpRequest
*request
= http
->request
;
969 HttpRequestMethod method
= request
->method
;
970 const wordlist
*p
= NULL
;
972 // intercepted requests MUST NOT (yet) be sent to peers unless verified
973 if (!request
->flags
.hostVerified
&& (request
->flags
.intercepted
|| request
->flags
.spoofClientIp
))
977 * IMS needs a private key, so we can use the hierarchy for IMS only if our
978 * neighbors support private keys
981 if (request
->flags
.ims
&& !neighbors_do_private_keys
)
985 * This is incorrect: authenticating requests can be sent via a hierarchy
986 * (they can even be cached if the correct headers are set on the reply)
988 if (request
->flags
.auth
)
991 if (method
== Http::METHOD_TRACE
)
994 if (method
!= Http::METHOD_GET
)
997 /* scan hierarchy_stoplist */
998 for (p
= Config
.hierarchy_stoplist
; p
; p
= p
->next
)
999 if (strstr(url
, p
->key
))
1002 if (request
->flags
.loopDetected
)
1005 if (request
->protocol
== AnyP::PROTO_HTTP
)
1006 return method
.respMaybeCacheable();
1008 if (request
->protocol
== AnyP::PROTO_GOPHER
)
1009 return gopherCachable(request
);
1011 if (request
->protocol
== AnyP::PROTO_CACHE_OBJECT
)
1018 clientCheckPinning(ClientHttpRequest
* http
)
1020 HttpRequest
*request
= http
->request
;
1021 HttpHeader
*req_hdr
= &request
->header
;
1022 ConnStateData
*http_conn
= http
->getConn();
1024 /* Internal requests such as those from ESI includes may be without
1025 * a client connection
1030 request
->flags
.connectionAuthDisabled
= http_conn
->port
->connection_auth_disabled
;
1031 if (!request
->flags
.connectionAuthDisabled
) {
1032 if (Comm::IsConnOpen(http_conn
->pinning
.serverConnection
)) {
1033 if (http_conn
->pinning
.auth
) {
1034 request
->flags
.connectionAuth
= true;
1035 request
->flags
.auth
= true;
1037 request
->flags
.connectionProxyAuth
= true;
1039 // These should already be linked correctly.
1040 assert(request
->clientConnectionManager
== http_conn
);
1044 /* check if connection auth is used, and flag as candidate for pinning
1046 * Note: we may need to set flags.connectionAuth even if the connection
1047 * is already pinned if it was pinned earlier due to proxy auth
1049 if (!request
->flags
.connectionAuth
) {
1050 if (req_hdr
->has(HDR_AUTHORIZATION
) || req_hdr
->has(HDR_PROXY_AUTHORIZATION
)) {
1051 HttpHeaderPos pos
= HttpHeaderInitPos
;
1054 while ((e
= req_hdr
->getEntry(&pos
))) {
1055 if (e
->id
== HDR_AUTHORIZATION
|| e
->id
== HDR_PROXY_AUTHORIZATION
) {
1056 const char *value
= e
->value
.rawBuf();
1057 if (strncasecmp(value
, "NTLM ", 5) == 0
1059 strncasecmp(value
, "Negotiate ", 10) == 0
1061 strncasecmp(value
, "Kerberos ", 9) == 0) {
1062 if (e
->id
== HDR_AUTHORIZATION
) {
1063 request
->flags
.connectionAuth
= true;
1066 request
->flags
.connectionProxyAuth
= true;
1072 if (may_pin
&& !request
->pinnedConnection()) {
1073 // These should already be linked correctly. Just need the ServerConnection to pinn.
1074 assert(request
->clientConnectionManager
== http_conn
);
1081 clientInterpretRequestHeaders(ClientHttpRequest
* http
)
1083 HttpRequest
*request
= http
->request
;
1084 HttpHeader
*req_hdr
= &request
->header
;
1085 bool no_cache
= false;
1088 request
->imslen
= -1;
1089 request
->ims
= req_hdr
->getTime(HDR_IF_MODIFIED_SINCE
);
1091 if (request
->ims
> 0)
1092 request
->flags
.ims
= true;
1094 if (!request
->flags
.ignoreCc
) {
1095 if (request
->cache_control
) {
1096 if (request
->cache_control
->noCache())
1099 // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1100 } else if (req_hdr
->has(HDR_PRAGMA
))
1101 no_cache
= req_hdr
->hasListMember(HDR_PRAGMA
,"no-cache",',');
1104 * Work around for supporting the Reload button in IE browsers when Squid
1105 * is used as an accelerator or transparent proxy, by turning accelerated
1106 * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1107 * actually only fixed in SP1, but we can't tell whether we are talking to
1108 * SP1 or not so all 5.5 versions are treated 'normally').
1110 if (Config
.onoff
.ie_refresh
) {
1111 if (http
->flags
.accel
&& request
->flags
.ims
) {
1112 if ((str
= req_hdr
->getStr(HDR_USER_AGENT
))) {
1113 if (strstr(str
, "MSIE 5.01") != NULL
)
1115 else if (strstr(str
, "MSIE 5.0") != NULL
)
1117 else if (strstr(str
, "MSIE 4.") != NULL
)
1119 else if (strstr(str
, "MSIE 3.") != NULL
)
1126 if (request
->method
== Http::METHOD_OTHER
) {
1131 #if USE_HTTP_VIOLATIONS
1133 if (Config
.onoff
.reload_into_ims
)
1134 request
->flags
.nocacheHack
= true;
1135 else if (refresh_nocache_hack
)
1136 request
->flags
.nocacheHack
= true;
1140 request
->flags
.noCache
= true;
1143 /* ignore range header in non-GETs or non-HEADs */
1144 if (request
->method
== Http::METHOD_GET
|| request
->method
== Http::METHOD_HEAD
) {
1145 // XXX: initialize if we got here without HttpRequest::parseHeader()
1146 if (!request
->range
)
1147 request
->range
= req_hdr
->getRange();
1149 if (request
->range
) {
1150 request
->flags
.isRanged
= true;
1151 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
1152 /* XXX: This is suboptimal. We should give the stream the range set,
1153 * and thereby let the top of the stream set the offset when the
1154 * size becomes known. As it is, we will end up requesting from 0
1155 * for evey -X range specification.
1156 * RBC - this may be somewhat wrong. We should probably set the range
1157 * iter up at this point.
1159 node
->readBuffer
.offset
= request
->range
->lowestOffset(0);
1160 http
->range_iter
.pos
= request
->range
->begin();
1161 http
->range_iter
.valid
= true;
1165 /* Only HEAD and GET requests permit a Range or Request-Range header.
1166 * If these headers appear on any other type of request, delete them now.
1169 req_hdr
->delById(HDR_RANGE
);
1170 req_hdr
->delById(HDR_REQUEST_RANGE
);
1171 delete request
->range
;
1172 request
->range
= NULL
;
1175 if (req_hdr
->has(HDR_AUTHORIZATION
))
1176 request
->flags
.auth
= true;
1178 clientCheckPinning(http
);
1180 if (request
->login
[0] != '\0')
1181 request
->flags
.auth
= true;
1183 if (req_hdr
->has(HDR_VIA
)) {
1184 String s
= req_hdr
->getList(HDR_VIA
);
1186 * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1187 * Note ThisCache2 has a space prepended to the hostname so we don't
1188 * accidentally match super-domains.
1191 if (strListIsSubstr(&s
, ThisCache2
, ',')) {
1192 debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1193 request
, (ObjPackMethod
) & httpRequestPack
);
1194 request
->flags
.loopDetected
= true;
1198 fvdbCountVia(s
.termedBuf());
1207 if (req_hdr
->has(HDR_X_FORWARDED_FOR
)) {
1208 String s
= req_hdr
->getList(HDR_X_FORWARDED_FOR
);
1209 fvdbCountForw(s
.termedBuf());
1215 request
->flags
.cachable
= http
->request
->maybeCacheable();
1217 if (clientHierarchical(http
))
1218 request
->flags
.hierarchical
= true;
1220 debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1221 (request
->flags
.noCache
? "SET" : "NOT SET"));
1222 debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1223 (request
->flags
.cachable
? "SET" : "NOT SET"));
1224 debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1225 (request
->flags
.hierarchical
? "SET" : "NOT SET"));
1230 clientRedirectDoneWrapper(void *data
, const HelperReply
&result
)
1232 ClientRequestContext
*calloutContext
= (ClientRequestContext
*)data
;
1234 if (!calloutContext
->httpStateIsValid())
1237 calloutContext
->clientRedirectDone(result
);
1241 clientStoreIdDoneWrapper(void *data
, const HelperReply
&result
)
1243 ClientRequestContext
*calloutContext
= (ClientRequestContext
*)data
;
1245 if (!calloutContext
->httpStateIsValid())
1248 calloutContext
->clientStoreIdDone(result
);
1252 ClientRequestContext::clientRedirectDone(const HelperReply
&reply
)
1254 HttpRequest
*old_request
= http
->request
;
1255 debugs(85, 5, HERE
<< "'" << http
->uri
<< "' result=" << reply
);
1256 assert(redirect_state
== REDIRECT_PENDING
);
1257 redirect_state
= REDIRECT_DONE
;
1259 // copy the URL rewriter response Notes to the HTTP request for logging
1260 // do it early to ensure that no matter what the outcome the notes are present.
1261 // TODO put them straight into the transaction state record (ALE?) eventually
1262 if (!old_request
->helperNotes
)
1263 old_request
->helperNotes
= new Notes
;
1264 old_request
->helperNotes
->add(reply
.notes
);
1266 switch (reply
.result
) {
1267 case HelperReply::Unknown
:
1268 case HelperReply::TT
:
1269 // Handler in redirect.cc should have already mapped Unknown
1270 // IF it contained valid entry for the old URL-rewrite helper protocol
1271 debugs(85, DBG_IMPORTANT
, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply
);
1274 case HelperReply::BrokenHelper
:
1275 debugs(85, DBG_IMPORTANT
, "ERROR: URL rewrite helper: " << reply
<< ", attempt #" << (redirect_fail_count
+1) << " of 2");
1276 if (redirect_fail_count
< 2) { // XXX: make this configurable ?
1277 ++redirect_fail_count
;
1278 // reset state flag to try redirector again from scratch.
1279 redirect_done
= false;
1283 case HelperReply::Error
:
1284 // no change to be done.
1287 case HelperReply::Okay
: {
1288 // #1: redirect with a specific status code OK status=NNN url="..."
1289 // #2: redirect with a default status code OK url="..."
1290 // #3: re-write the URL OK rewrite-url="..."
1292 Note::Pointer statusNote
= reply
.notes
.find("status");
1293 Note::Pointer urlNote
= reply
.notes
.find("url");
1295 if (urlNote
!= NULL
) {
1296 // HTTP protocol redirect to be done.
1298 // TODO: change default redirect status for appropriate requests
1299 // Squid defaults to 302 status for now for better compatibility with old clients.
1300 // HTTP/1.0 client should get 302 (Http::scMovedTemporarily)
1301 // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1302 // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1303 Http::StatusCode status
= Http::scMovedTemporarily
;
1304 if (statusNote
!= NULL
) {
1305 const char * result
= statusNote
->firstValue();
1306 status
= static_cast<Http::StatusCode
>(atoi(result
));
1309 if (status
== Http::scMovedPermanently
1310 || status
== Http::scMovedTemporarily
1311 || status
== Http::scSeeOther
1312 || status
== Http::scPermanentRedirect
1313 || status
== Http::scTemporaryRedirect
) {
1314 http
->redirect
.status
= status
;
1315 http
->redirect
.location
= xstrdup(urlNote
->firstValue());
1316 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1318 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid " << status
<< " redirect Location: " << urlNote
->firstValue());
1321 // URL-rewrite wanted. Ew.
1322 urlNote
= reply
.notes
.find("rewrite-url");
1324 // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1325 if (urlNote
!= NULL
&& strcmp(urlNote
->firstValue(), http
->uri
)) {
1326 // XXX: validate the URL properly *without* generating a whole new request object right here.
1327 // XXX: the clone() should be done only AFTER we know the new URL is valid.
1328 HttpRequest
*new_request
= old_request
->clone();
1329 if (urlParse(old_request
->method
, const_cast<char*>(urlNote
->firstValue()), new_request
)) {
1330 debugs(61,2, HERE
<< "URL-rewriter diverts URL from " << urlCanonical(old_request
) << " to " << urlCanonical(new_request
));
1332 // update the new request to flag the re-writing was done on it
1333 new_request
->flags
.redirected
= true;
1335 // unlink bodypipe from the old request. Not needed there any longer.
1336 if (old_request
->body_pipe
!= NULL
) {
1337 old_request
->body_pipe
= NULL
;
1338 debugs(61,2, HERE
<< "URL-rewriter diverts body_pipe " << new_request
->body_pipe
<<
1339 " from request " << old_request
<< " to " << new_request
);
1342 // update the current working ClientHttpRequest fields
1343 safe_free(http
->uri
);
1344 http
->uri
= xstrdup(urlCanonical(new_request
));
1345 HTTPMSGUNLOCK(old_request
);
1346 http
->request
= new_request
;
1347 HTTPMSGLOCK(http
->request
);
1349 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid request: " <<
1350 old_request
->method
<< " " << urlNote
->firstValue() << " " << old_request
->http_ver
);
1359 /* FIXME PIPELINE: This is innacurate during pipelining */
1361 if (http
->getConn() != NULL
&& Comm::IsConnOpen(http
->getConn()->clientConnection
))
1362 fd_note(http
->getConn()->clientConnection
->fd
, http
->uri
);
1370 * This method handles the different replies from StoreID helper.
1373 ClientRequestContext::clientStoreIdDone(const HelperReply
&reply
)
1375 HttpRequest
*old_request
= http
->request
;
1376 debugs(85, 5, "'" << http
->uri
<< "' result=" << reply
);
1377 assert(store_id_state
== REDIRECT_PENDING
);
1378 store_id_state
= REDIRECT_DONE
;
1380 // copy the helper response Notes to the HTTP request for logging
1381 // do it early to ensure that no matter what the outcome the notes are present.
1382 // TODO put them straight into the transaction state record (ALE?) eventually
1383 if (!old_request
->helperNotes
)
1384 old_request
->helperNotes
= new Notes
;
1385 old_request
->helperNotes
->add(reply
.notes
);
1387 switch (reply
.result
) {
1388 case HelperReply::Unknown
:
1389 case HelperReply::TT
:
1390 // Handler in redirect.cc should have already mapped Unknown
1391 // IF it contained valid entry for the old helper protocol
1392 debugs(85, DBG_IMPORTANT
, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply
);
1395 case HelperReply::BrokenHelper
:
1396 debugs(85, DBG_IMPORTANT
, "ERROR: storeID helper: " << reply
<< ", attempt #" << (store_id_fail_count
+1) << " of 2");
1397 if (store_id_fail_count
< 2) { // XXX: make this configurable ?
1398 ++store_id_fail_count
;
1399 // reset state flag to try StoreID again from scratch.
1400 store_id_done
= false;
1404 case HelperReply::Error
:
1405 // no change to be done.
1408 case HelperReply::Okay
: {
1409 Note::Pointer urlNote
= reply
.notes
.find("store-id");
1411 // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1412 if (urlNote
!= NULL
&& strcmp(urlNote
->firstValue(), http
->uri
) ) {
1413 // Debug section required for some very specific cases.
1414 debugs(85, 9, "Setting storeID with: " << urlNote
->firstValue() );
1415 http
->request
->store_id
= urlNote
->firstValue();
1416 http
->store_id
= urlNote
->firstValue();
1425 /** Test cache allow/deny configuration
1426 * Sets flags.cachable=1 if caching is not denied.
1429 ClientRequestContext::checkNoCache()
1431 if (Config
.accessList
.noCache
) {
1432 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.noCache
, http
);
1433 acl_checklist
->nonBlockingCheck(checkNoCacheDoneWrapper
, this);
1435 /* unless otherwise specified, we try to cache. */
1436 checkNoCacheDone(ACCESS_ALLOWED
);
1441 checkNoCacheDoneWrapper(allow_t answer
, void *data
)
1443 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
1445 if (!calloutContext
->httpStateIsValid())
1448 calloutContext
->checkNoCacheDone(answer
);
1452 ClientRequestContext::checkNoCacheDone(const allow_t
&answer
)
1454 acl_checklist
= NULL
;
1455 http
->request
->flags
.cachable
= (answer
== ACCESS_ALLOWED
);
1461 ClientRequestContext::sslBumpAccessCheck()
1463 // If SSL connection tunneling or bumping decision has been made, obey it.
1464 const Ssl::BumpMode bumpMode
= http
->getConn()->sslBumpMode
;
1465 if (bumpMode
!= Ssl::bumpEnd
) {
1466 debugs(85, 5, HERE
<< "SslBump already decided (" << bumpMode
<<
1467 "), " << "ignoring ssl_bump for " << http
->getConn());
1468 http
->al
->ssl
.bumpMode
= bumpMode
; // inherited from bumped connection
1472 // If we have not decided yet, decide whether to bump now.
1474 // Bumping here can only start with a CONNECT request on a bumping port
1475 // (bumping of intercepted SSL conns is decided before we get 1st request).
1476 // We also do not bump redirected CONNECT requests.
1477 if (http
->request
->method
!= Http::METHOD_CONNECT
|| http
->redirect
.status
||
1478 !Config
.accessList
.ssl_bump
||
1479 !http
->getConn()->port
->flags
.tunnelSslBumping
) {
1480 http
->al
->ssl
.bumpMode
= Ssl::bumpEnd
; // SslBump does not apply; log -
1481 debugs(85, 5, HERE
<< "cannot SslBump this request");
1485 // Do not bump during authentication: clients would not proxy-authenticate
1486 // if we delay a 407 response and respond with 200 OK to CONNECT.
1487 if (error
&& error
->httpStatus
== Http::scProxyAuthenticationRequired
) {
1488 http
->al
->ssl
.bumpMode
= Ssl::bumpEnd
; // SslBump does not apply; log -
1489 debugs(85, 5, HERE
<< "no SslBump during proxy authentication");
1493 debugs(85, 5, HERE
<< "SslBump possible, checking ACL");
1495 ACLFilledChecklist
*aclChecklist
= clientAclChecklistCreate(Config
.accessList
.ssl_bump
, http
);
1496 aclChecklist
->nonBlockingCheck(sslBumpAccessCheckDoneWrapper
, this);
1501 * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1502 * as ACLFilledChecklist callback
1505 sslBumpAccessCheckDoneWrapper(allow_t answer
, void *data
)
1507 ClientRequestContext
*calloutContext
= static_cast<ClientRequestContext
*>(data
);
1509 if (!calloutContext
->httpStateIsValid())
1511 calloutContext
->sslBumpAccessCheckDone(answer
);
1515 ClientRequestContext::sslBumpAccessCheckDone(const allow_t
&answer
)
1517 if (!httpStateIsValid())
1520 const Ssl::BumpMode bumpMode
= answer
== ACCESS_ALLOWED
?
1521 static_cast<Ssl::BumpMode
>(answer
.kind
) : Ssl::bumpNone
;
1522 http
->sslBumpNeed(bumpMode
); // for processRequest() to bump if needed
1523 http
->al
->ssl
.bumpMode
= bumpMode
; // for logging
1530 * Identify requests that do not go through the store and client side stream
1531 * and forward them to the appropriate location. All other requests, request
1535 ClientHttpRequest::processRequest()
1537 debugs(85, 4, "clientProcessRequest: " << RequestMethodStr(request
->method
) << " '" << uri
<< "'");
1539 if (request
->method
== Http::METHOD_CONNECT
&& !redirect
.status
) {
1541 if (sslBumpNeeded()) {
1546 logType
= LOG_TCP_MISS
;
1547 getConn()->stopReading(); // tunnels read for themselves
1548 tunnelStart(this, &out
.size
, &al
->http
.code
);
1556 ClientHttpRequest::httpStart()
1558 PROF_start(httpStart
);
1559 logType
= LOG_TAG_NONE
;
1560 debugs(85, 4, LogTags_str
[logType
] << " for '" << uri
<< "'");
1562 /* no one should have touched this */
1563 assert(out
.offset
== 0);
1564 /* Use the Stream Luke */
1565 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->data
;
1566 clientStreamRead(node
, this, node
->readBuffer
);
1567 PROF_stop(httpStart
);
1573 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode
)
1575 debugs(83, 3, HERE
<< "sslBump required: "<< Ssl::bumpMode(mode
));
1576 sslBumpNeed_
= mode
;
1579 // called when comm_write has completed
1581 SslBumpEstablish(const Comm::ConnectionPointer
&, char *, size_t, comm_err_t errflag
, int, void *data
)
1583 ClientHttpRequest
*r
= static_cast<ClientHttpRequest
*>(data
);
1584 debugs(85, 5, HERE
<< "responded to CONNECT: " << r
<< " ? " << errflag
);
1586 assert(r
&& cbdataReferenceValid(r
));
1587 r
->sslBumpEstablish(errflag
);
1591 ClientHttpRequest::sslBumpEstablish(comm_err_t errflag
)
1593 // Bail out quickly on COMM_ERR_CLOSING - close handlers will tidy up
1594 if (errflag
== COMM_ERR_CLOSING
)
1598 debugs(85, 3, HERE
<< "CONNECT response failure in SslBump: " << errflag
);
1599 getConn()->clientConnection
->close();
1603 // We lack HttpReply which logRequest() uses to log the status code.
1604 // TODO: Use HttpReply instead of the "200 Connection established" string.
1605 al
->http
.code
= 200;
1608 // Preserve authentication info for the ssl-bumped request
1609 if (request
->auth_user_request
!= NULL
)
1610 getConn()->auth_user_request
= request
->auth_user_request
;
1613 assert(sslBumpNeeded());
1614 getConn()->switchToHttps(request
, sslBumpNeed_
);
1618 ClientHttpRequest::sslBumpStart()
1620 debugs(85, 5, HERE
<< "Confirming " << Ssl::bumpMode(sslBumpNeed_
) <<
1621 "-bumped CONNECT tunnel on FD " << getConn()->clientConnection
);
1622 getConn()->sslBumpMode
= sslBumpNeed_
;
1624 // send an HTTP 200 response to kick client SSL negotiation
1625 // TODO: Unify with tunnel.cc and add a Server(?) header
1626 static const char *const conn_established
= "HTTP/1.1 200 Connection established\r\n\r\n";
1627 AsyncCall::Pointer call
= commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1628 CommIoCbPtrFun(&SslBumpEstablish
, this));
1629 Comm::Write(getConn()->clientConnection
, conn_established
, strlen(conn_established
), call
, NULL
);
1635 ClientHttpRequest::gotEnough() const
1637 /** TODO: should be querying the stream. */
1638 int64_t contentLength
=
1639 memObject()->getReply()->bodySize(request
->method
);
1640 assert(contentLength
>= 0);
1642 if (out
.offset
< contentLength
)
1649 ClientHttpRequest::storeEntry(StoreEntry
*newEntry
)
1655 ClientHttpRequest::loggingEntry(StoreEntry
*newEntry
)
1658 loggingEntry_
->unlock();
1660 loggingEntry_
= newEntry
;
1663 loggingEntry_
->lock();
1667 * doCallouts() - This function controls the order of "callout"
1668 * executions, including non-blocking access control checks, the
1669 * redirector, and ICAP. Previously, these callouts were chained
1670 * together such that "clientAccessCheckDone()" would call
1671 * "clientRedirectStart()" and so on.
1673 * The ClientRequestContext (aka calloutContext) class holds certain
1674 * state data for the callout/callback operations. Previously
1675 * ClientHttpRequest would sort of hand off control to ClientRequestContext
1676 * for a short time. ClientRequestContext would then delete itself
1677 * and pass control back to ClientHttpRequest when all callouts
1680 * This caused some problems for ICAP because we want to make the
1681 * ICAP callout after checking ACLs, but before checking the no_cache
1682 * list. We can't stuff the ICAP state into the ClientRequestContext
1683 * class because we still need the ICAP state after ClientRequestContext
1686 * Note that ClientRequestContext is created before the first call
1689 * If one of the callouts notices that ClientHttpRequest is no
1690 * longer valid, it should call cbdataReferenceDone() so that
1691 * ClientHttpRequest's reference count goes to zero and it will get
1692 * deleted. ClientHttpRequest will then delete ClientRequestContext.
1694 * Note that we set the _done flags here before actually starting
1695 * the callout. This is strictly for convenience.
1698 tos_t
aclMapTOS (acl_tos
* head
, ACLChecklist
* ch
);
1699 nfmark_t
aclMapNfmark (acl_nfmark
* head
, ACLChecklist
* ch
);
1702 ClientHttpRequest::doCallouts()
1704 assert(calloutContext
);
1706 /*Save the original request for logging purposes*/
1707 if (!calloutContext
->http
->al
->request
) {
1708 calloutContext
->http
->al
->request
= request
;
1709 HTTPMSGLOCK(calloutContext
->http
->al
->request
);
1712 if (!calloutContext
->error
) {
1713 // CVE-2009-0801: verify the Host: header is consistent with other known details.
1714 if (!calloutContext
->host_header_verify_done
) {
1715 debugs(83, 3, HERE
<< "Doing calloutContext->hostHeaderVerify()");
1716 calloutContext
->host_header_verify_done
= true;
1717 calloutContext
->hostHeaderVerify();
1721 if (!calloutContext
->http_access_done
) {
1722 debugs(83, 3, HERE
<< "Doing calloutContext->clientAccessCheck()");
1723 calloutContext
->http_access_done
= true;
1724 calloutContext
->clientAccessCheck();
1729 if (!calloutContext
->adaptation_acl_check_done
) {
1730 calloutContext
->adaptation_acl_check_done
= true;
1731 if (Adaptation::AccessCheck::Start(
1732 Adaptation::methodReqmod
, Adaptation::pointPreCache
,
1733 request
, NULL
, this))
1734 return; // will call callback
1738 if (!calloutContext
->redirect_done
) {
1739 calloutContext
->redirect_done
= true;
1740 assert(calloutContext
->redirect_state
== REDIRECT_NONE
);
1742 if (Config
.Program
.redirect
) {
1743 debugs(83, 3, HERE
<< "Doing calloutContext->clientRedirectStart()");
1744 calloutContext
->redirect_state
= REDIRECT_PENDING
;
1745 calloutContext
->clientRedirectStart();
1750 if (!calloutContext
->adapted_http_access_done
) {
1751 debugs(83, 3, HERE
<< "Doing calloutContext->clientAccessCheck2()");
1752 calloutContext
->adapted_http_access_done
= true;
1753 calloutContext
->clientAccessCheck2();
1757 if (!calloutContext
->store_id_done
) {
1758 calloutContext
->store_id_done
= true;
1759 assert(calloutContext
->store_id_state
== REDIRECT_NONE
);
1761 if (Config
.Program
.store_id
) {
1762 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1763 calloutContext
->store_id_state
= REDIRECT_PENDING
;
1764 calloutContext
->clientStoreIdStart();
1769 if (!calloutContext
->interpreted_req_hdrs
) {
1770 debugs(83, 3, HERE
<< "Doing clientInterpretRequestHeaders()");
1771 calloutContext
->interpreted_req_hdrs
= 1;
1772 clientInterpretRequestHeaders(this);
1775 if (!calloutContext
->no_cache_done
) {
1776 calloutContext
->no_cache_done
= true;
1778 if (Config
.accessList
.noCache
&& request
->flags
.cachable
) {
1779 debugs(83, 3, HERE
<< "Doing calloutContext->checkNoCache()");
1780 calloutContext
->checkNoCache();
1784 } // if !calloutContext->error
1786 if (!calloutContext
->tosToClientDone
) {
1787 calloutContext
->tosToClientDone
= true;
1788 if (getConn() != NULL
&& Comm::IsConnOpen(getConn()->clientConnection
)) {
1789 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1790 ch
.src_addr
= request
->client_addr
;
1791 ch
.my_addr
= request
->my_addr
;
1792 tos_t tos
= aclMapTOS(Ip::Qos::TheConfig
.tosToClient
, &ch
);
1794 Ip::Qos::setSockTos(getConn()->clientConnection
, tos
);
1798 if (!calloutContext
->nfmarkToClientDone
) {
1799 calloutContext
->nfmarkToClientDone
= true;
1800 if (getConn() != NULL
&& Comm::IsConnOpen(getConn()->clientConnection
)) {
1801 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1802 ch
.src_addr
= request
->client_addr
;
1803 ch
.my_addr
= request
->my_addr
;
1804 nfmark_t mark
= aclMapNfmark(Ip::Qos::TheConfig
.nfmarkToClient
, &ch
);
1806 Ip::Qos::setSockNfmark(getConn()->clientConnection
, mark
);
1811 // We need to check for SslBump even if the calloutContext->error is set
1812 // because bumping may require delaying the error until after CONNECT.
1813 if (!calloutContext
->sslBumpCheckDone
) {
1814 calloutContext
->sslBumpCheckDone
= true;
1815 if (calloutContext
->sslBumpAccessCheck())
1817 /* else no ssl bump required*/
1821 if (calloutContext
->error
) {
1822 const char *storeUri
= request
->storeId();
1823 StoreEntry
*e
= storeCreateEntry(storeUri
, storeUri
, request
->flags
, request
->method
);
1825 if (sslBumpNeeded()) {
1826 // set final error but delay sending until we bump
1827 Ssl::ServerBump
*srvBump
= new Ssl::ServerBump(request
, e
);
1828 errorAppendEntry(e
, calloutContext
->error
);
1829 calloutContext
->error
= NULL
;
1830 getConn()->setServerBump(srvBump
);
1835 // send the error to the client now
1836 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
1837 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
1838 assert (repContext
);
1839 repContext
->setReplyToStoreEntry(e
);
1840 errorAppendEntry(e
, calloutContext
->error
);
1841 calloutContext
->error
= NULL
;
1842 if (calloutContext
->readNextRequest
)
1843 getConn()->flags
.readMore
= true; // resume any pipeline reads.
1844 node
= (clientStreamNode
*)client_stream
.tail
->data
;
1845 clientStreamRead(node
, this, node
->readBuffer
);
1851 cbdataReferenceDone(calloutContext
->http
);
1852 delete calloutContext
;
1853 calloutContext
= NULL
;
1856 headersLog(0, 1, request
->method
, request
);
1859 debugs(83, 3, HERE
<< "calling processRequest()");
1863 Adaptation::Icap::History::Pointer ih
= request
->icapHistory();
1865 ih
->logType
= logType
;
1870 #include "client_side_request.cci"
1874 /// Initiate an asynchronous adaptation transaction which will call us back.
1876 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer
&g
)
1878 debugs(85, 3, HERE
<< "adaptation needed for " << this);
1879 assert(!virginHeadSource
);
1880 assert(!adaptedBodySource
);
1881 virginHeadSource
= initiateAdaptation(
1882 new Adaptation::Iterator(request
, NULL
, g
));
1884 // we could try to guess whether we can bypass this adaptation
1885 // initiation failure, but it should not really happen
1886 Must(initiated(virginHeadSource
));
1890 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer
&answer
)
1892 assert(cbdataReferenceValid(this)); // indicates bug
1893 clearAdaptation(virginHeadSource
);
1894 assert(!adaptedBodySource
);
1896 switch (answer
.kind
) {
1897 case Adaptation::Answer::akForward
:
1898 handleAdaptedHeader(const_cast<HttpMsg
*>(answer
.message
.getRaw()));
1901 case Adaptation::Answer::akBlock
:
1902 handleAdaptationBlock(answer
);
1905 case Adaptation::Answer::akError
:
1906 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT
, !answer
.final
);
1912 ClientHttpRequest::handleAdaptedHeader(HttpMsg
*msg
)
1916 if (HttpRequest
*new_req
= dynamic_cast<HttpRequest
*>(msg
)) {
1918 * Replace the old request with the new request.
1920 HTTPMSGUNLOCK(request
);
1922 HTTPMSGLOCK(request
);
1924 * Store the new URI for logging
1927 uri
= xstrdup(urlCanonical(request
));
1928 setLogUri(this, urlCanonicalClean(request
));
1929 assert(request
->method
.id());
1930 } else if (HttpReply
*new_rep
= dynamic_cast<HttpReply
*>(msg
)) {
1931 debugs(85,3,HERE
<< "REQMOD reply is HTTP reply");
1933 // subscribe to receive reply body
1934 if (new_rep
->body_pipe
!= NULL
) {
1935 adaptedBodySource
= new_rep
->body_pipe
;
1936 int consumer_ok
= adaptedBodySource
->setConsumerIfNotLate(this);
1937 assert(consumer_ok
);
1940 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
1941 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
1943 repContext
->createStoreEntry(request
->method
, request
->flags
);
1945 EBIT_CLR(storeEntry()->flags
, ENTRY_FWD_HDR_WAIT
);
1946 request_satisfaction_mode
= true;
1947 request_satisfaction_offset
= 0;
1948 storeEntry()->replaceHttpReply(new_rep
);
1949 storeEntry()->timestampsSet();
1951 if (!adaptedBodySource
) // no body
1952 storeEntry()->complete();
1953 clientGetMoreData(node
, this);
1956 // we are done with getting headers (but may be receiving body)
1957 clearAdaptation(virginHeadSource
);
1959 if (!request_satisfaction_mode
)
1964 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer
&answer
)
1966 request
->detailError(ERR_ACCESS_DENIED
, ERR_DETAIL_REQMOD_BLOCK
);
1967 AclMatchedName
= answer
.ruleId
.termedBuf();
1968 assert(calloutContext
);
1969 calloutContext
->clientAccessCheckDone(ACCESS_DENIED
);
1970 AclMatchedName
= NULL
;
1974 ClientHttpRequest::resumeBodyStorage()
1976 if (!adaptedBodySource
)
1979 noteMoreBodyDataAvailable(adaptedBodySource
);
1983 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer
)
1985 assert(request_satisfaction_mode
);
1986 assert(adaptedBodySource
!= NULL
);
1988 if (size_t contentSize
= adaptedBodySource
->buf().contentSize()) {
1989 const size_t spaceAvailable
= storeEntry()->bytesWanted(Range
<size_t>(0,contentSize
));
1991 if (spaceAvailable
< contentSize
) {
1992 // No or partial body data consuming
1993 typedef NullaryMemFunT
<ClientHttpRequest
> Dialer
;
1994 AsyncCall::Pointer call
= asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1995 Dialer(this, &ClientHttpRequest::resumeBodyStorage
));
1996 storeEntry()->deferProducer(call
);
1999 if (!spaceAvailable
)
2002 if (spaceAvailable
< contentSize
)
2003 contentSize
= spaceAvailable
;
2005 BodyPipeCheckout
bpc(*adaptedBodySource
);
2006 const StoreIOBuffer
ioBuf(&bpc
.buf
, request_satisfaction_offset
, contentSize
);
2007 storeEntry()->write(ioBuf
);
2008 // assume StoreEntry::write() writes the entire ioBuf
2009 request_satisfaction_offset
+= ioBuf
.length
;
2010 bpc
.buf
.consume(contentSize
);
2014 if (adaptedBodySource
->exhausted())
2015 endRequestSatisfaction();
2016 // else wait for more body data
2020 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer
)
2022 assert(!virginHeadSource
);
2023 // should we end request satisfaction now?
2024 if (adaptedBodySource
!= NULL
&& adaptedBodySource
->exhausted())
2025 endRequestSatisfaction();
2029 ClientHttpRequest::endRequestSatisfaction()
2031 debugs(85,4, HERE
<< this << " ends request satisfaction");
2032 assert(request_satisfaction_mode
);
2033 stopConsumingFrom(adaptedBodySource
);
2035 // TODO: anything else needed to end store entry formation correctly?
2036 storeEntry()->complete();
2040 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer
)
2042 assert(!virginHeadSource
);
2043 stopConsumingFrom(adaptedBodySource
);
2045 debugs(85,3, HERE
<< "REQMOD body production failed");
2046 if (request_satisfaction_mode
) { // too late to recover or serve an error
2047 request
->detailError(ERR_ICAP_FAILURE
, ERR_DETAIL_CLT_REQMOD_RESP_BODY
);
2048 const Comm::ConnectionPointer c
= getConn()->clientConnection
;
2049 Must(Comm::IsConnOpen(c
));
2050 c
->close(); // drastic, but we may be writing a response already
2052 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY
);
2057 ClientHttpRequest::handleAdaptationFailure(int errDetail
, bool bypassable
)
2059 debugs(85,3, HERE
<< "handleAdaptationFailure(" << bypassable
<< ")");
2061 const bool usedStore
= storeEntry() && !storeEntry()->isEmpty();
2062 const bool usedPipe
= request
->body_pipe
!= NULL
&&
2063 request
->body_pipe
->consumedSize() > 0;
2065 if (bypassable
&& !usedStore
&& !usedPipe
) {
2066 debugs(85,3, HERE
<< "ICAP REQMOD callout failed, bypassing: " << calloutContext
);
2072 debugs(85,3, HERE
<< "ICAP REQMOD callout failed, responding with error");
2074 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
2075 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2078 // The original author of the code also wanted to pass an errno to
2079 // setReplyToError, but it seems unlikely that the errno reflects the
2080 // true cause of the error at this point, so I did not pass it.
2081 if (calloutContext
) {
2084 ConnStateData
* c
= getConn();
2085 calloutContext
->error
= clientBuildError(ERR_ICAP_FAILURE
, Http::scInternalServerError
,
2087 c
!= NULL
? c
->clientConnection
->remote
: noAddr
,
2091 calloutContext
->error
->auth_user_request
=
2092 c
!= NULL
&& c
->auth_user_request
!= NULL
? c
->auth_user_request
: request
->auth_user_request
;
2094 calloutContext
->error
->detailError(errDetail
);
2095 calloutContext
->readNextRequest
= true;
2097 c
->expectNoForwarding();
2100 //else if(calloutContext == NULL) is it possible?