3 * DEBUG: section 85 Client-side Request Routines
4 * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
6 * SQUID Web Proxy Cache http://www.squid-cache.org/
7 * ----------------------------------------------------------
9 * Squid is the result of efforts by numerous individuals from
10 * the Internet community; see the CONTRIBUTORS file for full
11 * details. Many organizations have provided support for Squid's
12 * development; see the SPONSORS file for full details. Squid is
13 * Copyrighted (C) 2001 by the Regents of the University of
14 * California; see the COPYRIGHT file for full details. Squid
15 * incorporates software developed and/or copyrighted by other
16 * sources; see the CREDITS file for full details.
18 * This program is free software; you can redistribute it and/or modify
19 * it under the terms of the GNU General Public License as published by
20 * the Free Software Foundation; either version 2 of the License, or
21 * (at your option) any later version.
23 * This program is distributed in the hope that it will be useful,
24 * but WITHOUT ANY WARRANTY; without even the implied warranty of
25 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 * GNU General Public License for more details.
28 * You should have received a copy of the GNU General Public License
29 * along with this program; if not, write to the Free Software
30 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
35 * General logic of request processing:
37 * We run a series of tests to determine if access will be permitted, and to do
38 * any redirection. Then we call into the result clientStream to retrieve data.
39 * From that point on it's up to reply management.
43 #include "acl/FilledChecklist.h"
44 #include "acl/Gadgets.h"
45 #include "anyp/PortCfg.h"
46 #include "ClientRequestContext.h"
47 #include "client_side.h"
48 #include "client_side_reply.h"
49 #include "client_side_request.h"
50 #include "clientStream.h"
51 #include "comm/Connection.h"
52 #include "comm/Write.h"
53 #include "compat/inet_pton.h"
54 #include "err_detail_type.h"
55 #include "errorpage.h"
58 #include "format/Token.h"
61 #include "HttpHdrCc.h"
62 #include "HttpReply.h"
63 #include "HttpRequest.h"
65 #include "ip/QosConfig.h"
66 #include "log/access_log.h"
67 #include "MemObject.h"
69 #include "profiler/Profiler.h"
70 #include "SquidConfig.h"
71 #include "SquidTime.h"
78 #include "auth/UserRequest.h"
81 #include "adaptation/AccessCheck.h"
82 #include "adaptation/Answer.h"
83 #include "adaptation/Iterator.h"
84 #include "adaptation/Service.h"
86 #include "adaptation/icap/History.h"
90 #include "ssl/support.h"
91 #include "ssl/ServerBump.h"
95 #define comm_close comm_lingering_close
98 static const char *const crlf
= "\r\n";
100 #if FOLLOW_X_FORWARDED_FOR
101 static void clientFollowXForwardedForCheck(allow_t answer
, void *data
);
102 #endif /* FOLLOW_X_FORWARDED_FOR */
104 ErrorState
*clientBuildError(err_type
, http_status
, char const *url
, Ip::Address
&, HttpRequest
*);
106 CBDATA_CLASS_INIT(ClientRequestContext
);
109 ClientRequestContext::operator new (size_t size
)
111 assert (size
== sizeof(ClientRequestContext
));
112 CBDATA_INIT_TYPE(ClientRequestContext
);
113 ClientRequestContext
*result
= cbdataAlloc(ClientRequestContext
);
118 ClientRequestContext::operator delete (void *address
)
120 ClientRequestContext
*t
= static_cast<ClientRequestContext
*>(address
);
124 /* Local functions */
126 static void clientAccessCheckDoneWrapper(allow_t
, void *);
128 static void sslBumpAccessCheckDoneWrapper(allow_t
, void *);
130 static int clientHierarchical(ClientHttpRequest
* http
);
131 static void clientInterpretRequestHeaders(ClientHttpRequest
* http
);
132 static RH clientRedirectDoneWrapper
;
133 static void checkNoCacheDoneWrapper(allow_t
, void *);
134 SQUIDCEXTERN CSR clientGetMoreData
;
135 SQUIDCEXTERN CSS clientReplyStatus
;
136 SQUIDCEXTERN CSD clientReplyDetach
;
137 static void checkFailureRatio(err_type
, hier_code
);
139 ClientRequestContext::~ClientRequestContext()
142 * Release our "lock" on our parent, ClientHttpRequest, if we
147 cbdataReferenceDone(http
);
150 debugs(85,3, HERE
<< this << " ClientRequestContext destructed");
153 ClientRequestContext::ClientRequestContext(ClientHttpRequest
*anHttp
) : http(cbdataReference(anHttp
)), acl_checklist (NULL
), redirect_state (REDIRECT_NONE
), error(NULL
), readNextRequest(false)
155 http_access_done
= false;
156 redirect_done
= false;
157 no_cache_done
= false;
158 interpreted_req_hdrs
= false;
160 sslBumpCheckDone
= false;
162 debugs(85,3, HERE
<< this << " ClientRequestContext constructed");
165 CBDATA_CLASS_INIT(ClientHttpRequest
);
168 ClientHttpRequest::operator new (size_t size
)
170 assert (size
== sizeof (ClientHttpRequest
));
171 CBDATA_INIT_TYPE(ClientHttpRequest
);
172 ClientHttpRequest
*result
= cbdataAlloc(ClientHttpRequest
);
177 ClientHttpRequest::operator delete (void *address
)
179 ClientHttpRequest
*t
= static_cast<ClientHttpRequest
*>(address
);
183 ClientHttpRequest::ClientHttpRequest(ConnStateData
* aConn
) :
185 AsyncJob("ClientHttpRequest"),
189 start_time
= current_time
;
191 al
= new AccessLogEntry
;
192 al
->tcpClient
= clientConnection
= aConn
->clientConnection
;
194 if (aConn
->clientConnection
!= NULL
&& aConn
->clientConnection
->isOpen()) {
195 if (SSL
*ssl
= fd_table
[aConn
->clientConnection
->fd
].ssl
)
196 al
->cache
.sslClientCert
.reset(SSL_get_peer_certificate(ssl
));
199 dlinkAdd(this, &active
, &ClientActiveRequests
);
201 request_satisfaction_mode
= false;
204 sslBumpNeed_
= Ssl::bumpEnd
;
209 * returns true if client specified that the object must come from the cache
210 * without contacting origin server
213 ClientHttpRequest::onlyIfCached()const
216 return request
->cache_control
&&
217 request
->cache_control
->onlyIfCached();
221 * This function is designed to serve a fairly specific purpose.
222 * Occasionally our vBNS-connected caches can talk to each other, but not
223 * the rest of the world. Here we try to detect frequent failures which
224 * make the cache unusable (e.g. DNS lookup and connect() failures). If
225 * the failure:success ratio goes above 1.0 then we go into "hit only"
226 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors
227 * will only fetch HITs from us if they are using the ICP protocol. We
228 * stay in this mode for 5 minutes.
230 * Duane W., Sept 16, 1996
233 checkFailureRatio(err_type etype
, hier_code hcode
)
235 // Can be set at compile time with -D compiler flag
236 #ifndef FAILURE_MODE_TIME
237 #define FAILURE_MODE_TIME 300
240 if (hcode
== HIER_NONE
)
243 // don't bother when ICP is disabled.
244 if (Config
.Port
.icp
<= 0)
247 static double magic_factor
= 100.0;
251 n_good
= magic_factor
/ (1.0 + request_failure_ratio
);
253 n_bad
= magic_factor
- n_good
;
259 case ERR_CONNECT_FAIL
:
260 case ERR_SECURE_CONNECT_FAIL
:
270 request_failure_ratio
= n_bad
/ n_good
;
272 if (hit_only_mode_until
> squid_curtime
)
275 if (request_failure_ratio
< 1.0)
278 debugs(33, DBG_CRITICAL
, "WARNING: Failure Ratio at "<< std::setw(4)<<
279 std::setprecision(3) << request_failure_ratio
);
281 debugs(33, DBG_CRITICAL
, "WARNING: ICP going into HIT-only mode for " <<
282 FAILURE_MODE_TIME
/ 60 << " minutes...");
284 hit_only_mode_until
= squid_curtime
+ FAILURE_MODE_TIME
;
286 request_failure_ratio
= 0.8; /* reset to something less than 1.0 */
289 ClientHttpRequest::~ClientHttpRequest()
291 debugs(33, 3, "httpRequestFree: " << uri
);
292 PROF_start(httpRequestFree
);
294 // Even though freeResources() below may destroy the request,
295 // we no longer set request->body_pipe to NULL here
296 // because we did not initiate that pipe (ConnStateData did)
298 /* the ICP check here was erroneous
299 * - StoreEntry::releaseRequest was always called if entry was valid
301 assert(logType
< LOG_TYPE_MAX
);
308 checkFailureRatio(request
->errType
, al
->hier
.code
);
313 announceInitiatorAbort(virginHeadSource
);
315 if (adaptedBodySource
!= NULL
)
316 stopConsumingFrom(adaptedBodySource
);
320 delete calloutContext
;
322 clientConnection
= NULL
;
325 cbdataReferenceDone(conn_
);
327 /* moving to the next connection is handled by the context free */
328 dlinkDelete(&active
, &ClientActiveRequests
);
330 PROF_stop(httpRequestFree
);
334 * Create a request and kick it off
339 * TODO: Pass in the buffers to be used in the inital Read request, as they are
340 * determined by the user
343 clientBeginRequest(const HttpRequestMethod
& method
, char const *url
, CSCB
* streamcallback
,
344 CSD
* streamdetach
, ClientStreamData streamdata
, HttpHeader
const *header
,
345 char *tailbuf
, size_t taillen
)
348 ClientHttpRequest
*http
= new ClientHttpRequest(NULL
);
349 HttpRequest
*request
;
350 StoreIOBuffer tempBuffer
;
351 http
->start_time
= current_time
;
352 /* this is only used to adjust the connection offset in client_side.c */
354 tempBuffer
.length
= taillen
;
355 tempBuffer
.data
= tailbuf
;
356 /* client stream setup */
357 clientStreamInit(&http
->client_stream
, clientGetMoreData
, clientReplyDetach
,
358 clientReplyStatus
, new clientReplyContext(http
), streamcallback
,
359 streamdetach
, streamdata
, tempBuffer
);
360 /* make it visible in the 'current acctive requests list' */
362 /* internal requests only makes sense in an
363 * accelerator today. TODO: accept flags ? */
364 http
->flags
.accel
= 1;
365 /* allow size for url rewriting */
366 url_sz
= strlen(url
) + Config
.appendDomainLen
+ 5;
367 http
->uri
= (char *)xcalloc(url_sz
, 1);
368 strcpy(http
->uri
, url
);
370 if ((request
= HttpRequest::CreateFromUrlAndMethod(http
->uri
, method
)) == NULL
) {
371 debugs(85, 5, "Invalid URL: " << http
->uri
);
376 * now update the headers in request with our supplied headers. urlParse
377 * should return a blank header set, but we use Update to be sure of
381 request
->header
.update(header
, NULL
);
383 http
->log_uri
= xstrdup(urlCanonicalClean(request
));
385 /* http struct now ready */
388 * build new header list *? TODO
390 request
->flags
.accelerated
= http
->flags
.accel
;
392 request
->flags
.internalClient
= 1;
394 /* this is an internally created
395 * request, not subject to acceleration
396 * target overrides */
398 * FIXME? Do we want to detect and handle internal requests of internal
402 /* Internally created requests cannot have bodies today */
403 request
->content_length
= 0;
405 request
->client_addr
.SetNoAddr();
407 #if FOLLOW_X_FORWARDED_FOR
408 request
->indirect_client_addr
.SetNoAddr();
409 #endif /* FOLLOW_X_FORWARDED_FOR */
411 request
->my_addr
.SetNoAddr(); /* undefined for internal requests */
413 request
->my_addr
.SetPort(0);
415 /* Our version is HTTP/1.1 */
416 HttpVersion
http_ver(1,1);
417 request
->http_ver
= http_ver
;
419 http
->request
= HTTPMSGLOCK(request
);
421 /* optional - skip the access check ? */
422 http
->calloutContext
= new ClientRequestContext(http
);
424 http
->calloutContext
->http_access_done
= false;
426 http
->calloutContext
->redirect_done
= true;
428 http
->calloutContext
->no_cache_done
= true;
436 ClientRequestContext::httpStateIsValid()
438 ClientHttpRequest
*http_
= http
;
440 if (cbdataReferenceValid(http_
))
445 cbdataReferenceDone(http_
);
450 #if FOLLOW_X_FORWARDED_FOR
452 * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
453 * against the followXFF ACL, or cleans up and passes control to
454 * clientAccessCheck().
456 * The trust model here is a little ambiguous. So to clarify the logic:
457 * - we may always use the direct client address as the client IP.
458 * - these trust tests merey tell whether we trust given IP enough to believe the
459 * IP string which it appended to the X-Forwarded-For: header.
460 * - if at any point we don't trust what an IP adds we stop looking.
461 * - at that point the current contents of indirect_client_addr are the value set
462 * by the last previously trusted IP.
463 * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
466 clientFollowXForwardedForCheck(allow_t answer
, void *data
)
468 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
470 if (!calloutContext
->httpStateIsValid())
473 ClientHttpRequest
*http
= calloutContext
->http
;
474 HttpRequest
*request
= http
->request
;
477 * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
478 * called as a result of ACL checks, or -1 if we are called when
479 * there's nothing left to do.
481 if (answer
== ACCESS_ALLOWED
&&
482 request
->x_forwarded_for_iterator
.size () != 0) {
485 * Remove the last comma-delimited element from the
486 * x_forwarded_for_iterator and use it to repeat the cycle.
489 const char *asciiaddr
;
492 p
= request
->x_forwarded_for_iterator
.termedBuf();
493 l
= request
->x_forwarded_for_iterator
.size();
496 * XXX x_forwarded_for_iterator should really be a list of
497 * IP addresses, but it's a String instead. We have to
498 * walk backwards through the String, biting off the last
499 * comma-delimited part each time. As long as the data is in
500 * a String, we should probably implement and use a variant of
501 * strListGetItem() that walks backwards instead of forwards
502 * through a comma-separated list. But we don't even do that;
503 * we just do the work in-line here.
505 /* skip trailing space and commas */
506 while (l
> 0 && (p
[l
-1] == ',' || xisspace(p
[l
-1])))
508 request
->x_forwarded_for_iterator
.cut(l
);
509 /* look for start of last item in list */
510 while (l
> 0 && ! (p
[l
-1] == ',' || xisspace(p
[l
-1])))
513 if ((addr
= asciiaddr
)) {
514 request
->indirect_client_addr
= addr
;
515 request
->x_forwarded_for_iterator
.cut(l
);
516 calloutContext
->acl_checklist
= clientAclChecklistCreate(Config
.accessList
.followXFF
, http
);
517 if (!Config
.onoff
.acl_uses_indirect_client
) {
518 /* override the default src_addr tested if we have to go deeper than one level into XFF */
519 Filled(calloutContext
->acl_checklist
)->src_addr
= request
->indirect_client_addr
;
521 calloutContext
->acl_checklist
->nonBlockingCheck(clientFollowXForwardedForCheck
, data
);
524 } /*if (answer == ACCESS_ALLOWED &&
525 request->x_forwarded_for_iterator.size () != 0)*/
527 /* clean up, and pass control to clientAccessCheck */
528 if (Config
.onoff
.log_uses_indirect_client
) {
530 * Ensure that the access log shows the indirect client
531 * instead of the direct client.
533 ConnStateData
*conn
= http
->getConn();
534 conn
->log_addr
= request
->indirect_client_addr
;
536 request
->x_forwarded_for_iterator
.clean();
537 request
->flags
.done_follow_x_forwarded_for
=true;
539 if (answer
!= ACCESS_ALLOWED
&& answer
!= ACCESS_DENIED
) {
540 debugs(28, DBG_CRITICAL
, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request
->indirect_client_addr
);
543 /* process actual access ACL as normal. */
544 calloutContext
->clientAccessCheck();
546 #endif /* FOLLOW_X_FORWARDED_FOR */
549 hostHeaderIpVerifyWrapper(const ipcache_addrs
* ia
, const DnsLookupDetails
&dns
, void *data
)
551 ClientRequestContext
*c
= static_cast<ClientRequestContext
*>(data
);
552 c
->hostHeaderIpVerify(ia
, dns
);
556 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs
* ia
, const DnsLookupDetails
&dns
)
558 Comm::ConnectionPointer clientConn
= http
->getConn()->clientConnection
;
560 // note the DNS details for the transaction stats.
561 http
->request
->recordLookup(dns
);
563 if (ia
!= NULL
&& ia
->count
> 0) {
564 // Is the NAT destination IP in DNS?
565 for (int i
= 0; i
< ia
->count
; ++i
) {
566 if (clientConn
->local
.matchIPAddr(ia
->in_addrs
[i
]) == 0) {
567 debugs(85, 3, HERE
<< "validate IP " << clientConn
->local
<< " possible from Host:");
568 http
->request
->flags
.hostVerified
= 1;
572 debugs(85, 3, HERE
<< "validate IP " << clientConn
->local
<< " non-match from Host: IP " << ia
->in_addrs
[i
]);
575 debugs(85, 3, HERE
<< "FAIL: validate IP " << clientConn
->local
<< " possible from Host:");
576 hostHeaderVerifyFailed("local IP", "any domain IP");
580 ClientRequestContext::hostHeaderVerifyFailed(const char *A
, const char *B
)
582 // IP address validation for Host: failed. Admin wants to ignore them.
583 // NP: we do not yet handle CONNECT tunnels well, so ignore for them
584 if (!Config
.onoff
.hostStrictVerify
&& http
->request
->method
!= Http::METHOD_CONNECT
) {
585 debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http
->getConn()->clientConnection
<<
586 " (" << A
<< " does not match " << B
<< ") on URL: " << urlCanonical(http
->request
));
588 // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
589 // The problems here are about WRITE for new cache content, which means flags.cachable
590 http
->request
->flags
.cachable
= 0; // MUST NOT cache (for now)
591 // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
592 http
->request
->flags
.hierarchical
= 0; // MUST NOT pass to peers (for now)
593 // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
598 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: Host header forgery detected on " <<
599 http
->getConn()->clientConnection
<< " (" << A
<< " does not match " << B
<< ")");
600 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: By user agent: " << http
->request
->header
.getStr(HDR_USER_AGENT
));
601 debugs(85, DBG_IMPORTANT
, "SECURITY ALERT: on URL: " << urlCanonical(http
->request
));
603 // IP address validation for Host: failed. reject the connection.
604 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->prev
->data
;
605 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
607 repContext
->setReplyToError(ERR_CONFLICT_HOST
, HTTP_CONFLICT
,
608 http
->request
->method
, NULL
,
609 http
->getConn()->clientConnection
->remote
,
613 http
->getConn() != NULL
&& http
->getConn()->auth_user_request
!= NULL
?
614 http
->getConn()->auth_user_request
: http
->request
->auth_user_request
);
618 node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
619 clientStreamRead(node
, http
, node
->readBuffer
);
623 ClientRequestContext::hostHeaderVerify()
625 // Require a Host: header.
626 const char *host
= http
->request
->header
.getStr(HDR_HOST
);
629 // TODO: dump out the HTTP/1.1 error about missing host header.
630 // otherwise this is fine, can't forge a header value when its not even set.
631 debugs(85, 3, HERE
<< "validate skipped with no Host: header present.");
636 if (http
->request
->flags
.internal
) {
637 // TODO: kill this when URL handling allows partial URLs out of accel mode
638 // and we no longer screw with the URL just to add our internal host there
639 debugs(85, 6, HERE
<< "validate skipped due to internal composite URL.");
644 // Locate if there is a port attached, strip ready for IP lookup
645 char *portStr
= NULL
;
646 char *hostB
= xstrdup(host
);
648 if (host
[0] == '[') {
650 portStr
= strchr(hostB
, ']');
651 if (portStr
&& *(++portStr
) != ':') {
655 // Domain or IPv4 literal with port
656 portStr
= strrchr(hostB
, ':');
661 *portStr
= '\0'; // strip the ':'
662 if (*(++portStr
) != '\0')
663 port
= xatoi(portStr
);
666 debugs(85, 3, HERE
<< "validate host=" << host
<< ", port=" << port
<< ", portStr=" << (portStr
?portStr
:"NULL"));
667 if (http
->request
->flags
.intercepted
|| http
->request
->flags
.spoofClientIp
) {
668 // verify the Host: port (if any) matches the apparent destination
669 if (portStr
&& port
!= http
->getConn()->clientConnection
->local
.GetPort()) {
670 debugs(85, 3, HERE
<< "FAIL on validate port " << http
->getConn()->clientConnection
->local
.GetPort() <<
671 " matches Host: port " << port
<< " (" << portStr
<< ")");
672 hostHeaderVerifyFailed("intercepted port", portStr
);
674 // XXX: match the scheme default port against the apparent destination
676 // verify the destination DNS is one of the Host: headers IPs
677 ipcache_nbgethostbyname(host
, hostHeaderIpVerifyWrapper
, this);
679 } else if (!Config
.onoff
.hostStrictVerify
) {
680 debugs(85, 3, HERE
<< "validate skipped.");
682 } else if (strlen(host
) != strlen(http
->request
->GetHost())) {
683 // Verify forward-proxy requested URL domain matches the Host: header
684 debugs(85, 3, HERE
<< "FAIL on validate URL domain length " << http
->request
->GetHost() << " matches Host: " << host
);
685 hostHeaderVerifyFailed(host
, http
->request
->GetHost());
686 } else if (matchDomainName(host
, http
->request
->GetHost()) != 0) {
687 // Verify forward-proxy requested URL domain matches the Host: header
688 debugs(85, 3, HERE
<< "FAIL on validate URL domain " << http
->request
->GetHost() << " matches Host: " << host
);
689 hostHeaderVerifyFailed(host
, http
->request
->GetHost());
690 } else if (portStr
&& port
!= http
->request
->port
) {
691 // Verify forward-proxy requested URL domain matches the Host: header
692 debugs(85, 3, HERE
<< "FAIL on validate URL port " << http
->request
->port
<< " matches Host: port " << portStr
);
693 hostHeaderVerifyFailed("URL port", portStr
);
694 } else if (!portStr
&& http
->request
->method
!= Http::METHOD_CONNECT
&& http
->request
->port
!= urlDefaultPort(http
->request
->protocol
)) {
695 // Verify forward-proxy requested URL domain matches the Host: header
696 // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
697 debugs(85, 3, HERE
<< "FAIL on validate URL port " << http
->request
->port
<< " matches Host: default port " << urlDefaultPort(http
->request
->protocol
));
698 hostHeaderVerifyFailed("URL port", "default port");
701 debugs(85, 3, HERE
<< "validate passed.");
702 http
->request
->flags
.hostVerified
= 1;
708 /* This is the entry point for external users of the client_side routines */
710 ClientRequestContext::clientAccessCheck()
712 #if FOLLOW_X_FORWARDED_FOR
713 if (!http
->request
->flags
.doneFollowXff() &&
714 Config
.accessList
.followXFF
&&
715 http
->request
->header
.has(HDR_X_FORWARDED_FOR
)) {
717 /* we always trust the direct client address for actual use */
718 http
->request
->indirect_client_addr
= http
->request
->client_addr
;
719 http
->request
->indirect_client_addr
.SetPort(0);
721 /* setup the XFF iterator for processing */
722 http
->request
->x_forwarded_for_iterator
= http
->request
->header
.getList(HDR_X_FORWARDED_FOR
);
724 /* begin by checking to see if we trust direct client enough to walk XFF */
725 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.followXFF
, http
);
726 acl_checklist
->nonBlockingCheck(clientFollowXForwardedForCheck
, this);
731 if (Config
.accessList
.http
) {
732 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.http
, http
);
733 acl_checklist
->nonBlockingCheck(clientAccessCheckDoneWrapper
, this);
735 debugs(0, DBG_CRITICAL
, "No http_access configuration found. This will block ALL traffic");
736 clientAccessCheckDone(ACCESS_DENIED
);
741 * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
742 * The default here is to allow all. Since the earlier http_access should do a default deny all.
743 * This check is just for a last-minute denial based on adapted request headers.
746 ClientRequestContext::clientAccessCheck2()
748 if (Config
.accessList
.adapted_http
) {
749 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.adapted_http
, http
);
750 acl_checklist
->nonBlockingCheck(clientAccessCheckDoneWrapper
, this);
752 debugs(85, 2, HERE
<< "No adapted_http_access configuration. default: ALLOW");
753 clientAccessCheckDone(ACCESS_ALLOWED
);
758 clientAccessCheckDoneWrapper(allow_t answer
, void *data
)
760 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
762 if (!calloutContext
->httpStateIsValid())
765 calloutContext
->clientAccessCheckDone(answer
);
769 ClientRequestContext::clientAccessCheckDone(const allow_t
&answer
)
771 acl_checklist
= NULL
;
774 debugs(85, 2, "The request " <<
775 RequestMethodStr(http
->request
->method
) << " " <<
776 http
->uri
<< " is " << answer
<<
777 ", because it matched '" <<
778 (AclMatchedName
? AclMatchedName
: "NO ACL's") << "'" );
781 char const *proxy_auth_msg
= "<null>";
782 if (http
->getConn() != NULL
&& http
->getConn()->auth_user_request
!= NULL
)
783 proxy_auth_msg
= http
->getConn()->auth_user_request
->denyMessage("<null>");
784 else if (http
->request
->auth_user_request
!= NULL
)
785 proxy_auth_msg
= http
->request
->auth_user_request
->denyMessage("<null>");
788 if (answer
!= ACCESS_ALLOWED
) {
789 // auth has a grace period where credentials can be expired but okay not to challenge.
791 /* Send an auth challenge or error */
792 // XXX: do we still need aclIsProxyAuth() ?
793 bool auth_challenge
= (answer
== ACCESS_AUTH_REQUIRED
|| aclIsProxyAuth(AclMatchedName
));
794 debugs(85, 5, "Access Denied: " << http
->uri
);
795 debugs(85, 5, "AclMatchedName = " << (AclMatchedName
? AclMatchedName
: "<null>"));
798 debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg
? proxy_auth_msg
: "<null>"));
802 * NOTE: get page_id here, based on AclMatchedName because if
803 * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
804 * the clientCreateStoreEntry() call just below. Pedro Ribeiro
807 page_id
= aclGetDenyInfoPage(&Config
.denyInfoList
, AclMatchedName
, answer
!= ACCESS_AUTH_REQUIRED
);
809 http
->logType
= LOG_TCP_DENIED
;
811 if (auth_challenge
) {
813 if (http
->request
->flags
.sslBumped
) {
814 /*SSL Bumped request, authentication is not possible*/
815 status
= HTTP_FORBIDDEN
;
816 } else if (!http
->flags
.accel
) {
817 /* Proxy authorisation needed */
818 status
= HTTP_PROXY_AUTHENTICATION_REQUIRED
;
820 /* WWW authorisation needed */
821 status
= HTTP_UNAUTHORIZED
;
824 // need auth, but not possible to do.
825 status
= HTTP_FORBIDDEN
;
827 if (page_id
== ERR_NONE
)
828 page_id
= ERR_CACHE_ACCESS_DENIED
;
830 status
= HTTP_FORBIDDEN
;
832 if (page_id
== ERR_NONE
)
833 page_id
= ERR_ACCESS_DENIED
;
836 Ip::Address tmpnoaddr
;
837 tmpnoaddr
.SetNoAddr();
838 error
= clientBuildError(page_id
, status
,
840 http
->getConn() != NULL
? http
->getConn()->clientConnection
->remote
: tmpnoaddr
,
845 error
->auth_user_request
=
846 http
->getConn() != NULL
&& http
->getConn()->auth_user_request
!= NULL
?
847 http
->getConn()->auth_user_request
: http
->request
->auth_user_request
;
850 readNextRequest
= true;
853 /* ACCESS_ALLOWED continues here ... */
854 safe_free(http
->uri
);
856 http
->uri
= xstrdup(urlCanonical(http
->request
));
863 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g
)
865 debugs(93,3,HERE
<< this << " adaptationAclCheckDone called");
868 Adaptation::Icap::History::Pointer ih
= request
->icapHistory();
870 if (getConn() != NULL
) {
871 ih
->rfc931
= getConn()->clientConnection
->rfc931
;
873 assert(getConn()->clientConnection
!= NULL
);
874 ih
->ssluser
= sslGetUserEmail(fd_table
[getConn()->clientConnection
->fd
].ssl
);
877 ih
->log_uri
= log_uri
;
883 debugs(85,3, HERE
<< "no adaptation needed");
894 clientRedirectAccessCheckDone(allow_t answer
, void *data
)
896 ClientRequestContext
*context
= (ClientRequestContext
*)data
;
897 ClientHttpRequest
*http
= context
->http
;
898 context
->acl_checklist
= NULL
;
900 if (answer
== ACCESS_ALLOWED
)
901 redirectStart(http
, clientRedirectDoneWrapper
, context
);
903 context
->clientRedirectDone(NULL
);
907 ClientRequestContext::clientRedirectStart()
909 debugs(33, 5, HERE
<< "'" << http
->uri
<< "'");
911 if (Config
.accessList
.redirector
) {
912 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.redirector
, http
);
913 acl_checklist
->nonBlockingCheck(clientRedirectAccessCheckDone
, this);
915 redirectStart(http
, clientRedirectDoneWrapper
, this);
919 clientHierarchical(ClientHttpRequest
* http
)
921 const char *url
= http
->uri
;
922 HttpRequest
*request
= http
->request
;
923 HttpRequestMethod method
= request
->method
;
924 const wordlist
*p
= NULL
;
926 // intercepted requests MUST NOT (yet) be sent to peers unless verified
927 if (!request
->flags
.hostVerified
&& (request
->flags
.intercepted
|| request
->flags
.spoofClientIp
))
931 * IMS needs a private key, so we can use the hierarchy for IMS only if our
932 * neighbors support private keys
935 if (request
->flags
.ims
&& !neighbors_do_private_keys
)
939 * This is incorrect: authenticating requests can be sent via a hierarchy
940 * (they can even be cached if the correct headers are set on the reply)
942 if (request
->flags
.auth
)
945 if (method
== Http::METHOD_TRACE
)
948 if (method
!= Http::METHOD_GET
)
951 /* scan hierarchy_stoplist */
952 for (p
= Config
.hierarchy_stoplist
; p
; p
= p
->next
)
953 if (strstr(url
, p
->key
))
956 if (request
->flags
.loopDetected
)
959 if (request
->protocol
== AnyP::PROTO_HTTP
)
960 return method
.respMaybeCacheable();
962 if (request
->protocol
== AnyP::PROTO_GOPHER
)
963 return gopherCachable(request
);
965 if (request
->protocol
== AnyP::PROTO_CACHE_OBJECT
)
972 clientCheckPinning(ClientHttpRequest
* http
)
974 HttpRequest
*request
= http
->request
;
975 HttpHeader
*req_hdr
= &request
->header
;
976 ConnStateData
*http_conn
= http
->getConn();
978 /* Internal requests such as those from ESI includes may be without
979 * a client connection
984 request
->flags
.connectionAuthDisabled
= http_conn
->port
->connection_auth_disabled
;
985 if (!request
->flags
.connectionAuthDisabled
) {
986 if (Comm::IsConnOpen(http_conn
->pinning
.serverConnection
)) {
987 if (http_conn
->pinning
.auth
) {
988 request
->flags
.connectionAuth
= 1;
989 request
->flags
.auth
= 1;
991 request
->flags
.connectionProxyAuth
= 1;
993 // These should already be linked correctly.
994 assert(request
->clientConnectionManager
== http_conn
);
998 /* check if connection auth is used, and flag as candidate for pinning
1000 * Note: we may need to set flags.connectionAuth even if the connection
1001 * is already pinned if it was pinned earlier due to proxy auth
1003 if (!request
->flags
.connectionAuth
) {
1004 if (req_hdr
->has(HDR_AUTHORIZATION
) || req_hdr
->has(HDR_PROXY_AUTHORIZATION
)) {
1005 HttpHeaderPos pos
= HttpHeaderInitPos
;
1008 while ((e
= req_hdr
->getEntry(&pos
))) {
1009 if (e
->id
== HDR_AUTHORIZATION
|| e
->id
== HDR_PROXY_AUTHORIZATION
) {
1010 const char *value
= e
->value
.rawBuf();
1011 if (strncasecmp(value
, "NTLM ", 5) == 0
1013 strncasecmp(value
, "Negotiate ", 10) == 0
1015 strncasecmp(value
, "Kerberos ", 9) == 0) {
1016 if (e
->id
== HDR_AUTHORIZATION
) {
1017 request
->flags
.connectionAuth
= 1;
1020 request
->flags
.connectionProxyAuth
= 1;
1026 if (may_pin
&& !request
->pinnedConnection()) {
1027 // These should already be linked correctly. Just need the ServerConnection to pinn.
1028 assert(request
->clientConnectionManager
== http_conn
);
1035 clientInterpretRequestHeaders(ClientHttpRequest
* http
)
1037 HttpRequest
*request
= http
->request
;
1038 HttpHeader
*req_hdr
= &request
->header
;
1039 bool no_cache
= false;
1042 request
->imslen
= -1;
1043 request
->ims
= req_hdr
->getTime(HDR_IF_MODIFIED_SINCE
);
1045 if (request
->ims
> 0)
1046 request
->flags
.ims
= 1;
1048 if (!request
->flags
.ignoreCc
) {
1049 if (request
->cache_control
) {
1050 if (request
->cache_control
->noCache())
1053 // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1054 } else if (req_hdr
->has(HDR_PRAGMA
))
1055 no_cache
= req_hdr
->hasListMember(HDR_PRAGMA
,"no-cache",',');
1058 * Work around for supporting the Reload button in IE browsers when Squid
1059 * is used as an accelerator or transparent proxy, by turning accelerated
1060 * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1061 * actually only fixed in SP1, but we can't tell whether we are talking to
1062 * SP1 or not so all 5.5 versions are treated 'normally').
1064 if (Config
.onoff
.ie_refresh
) {
1065 if (http
->flags
.accel
&& request
->flags
.ims
) {
1066 if ((str
= req_hdr
->getStr(HDR_USER_AGENT
))) {
1067 if (strstr(str
, "MSIE 5.01") != NULL
)
1069 else if (strstr(str
, "MSIE 5.0") != NULL
)
1071 else if (strstr(str
, "MSIE 4.") != NULL
)
1073 else if (strstr(str
, "MSIE 3.") != NULL
)
1080 if (request
->method
== Http::METHOD_OTHER
) {
1085 #if USE_HTTP_VIOLATIONS
1087 if (Config
.onoff
.reload_into_ims
)
1088 request
->flags
.nocacheHack
= 1;
1089 else if (refresh_nocache_hack
)
1090 request
->flags
.nocacheHack
= 1;
1094 request
->flags
.noCache
= 1;
1097 /* ignore range header in non-GETs or non-HEADs */
1098 if (request
->method
== Http::METHOD_GET
|| request
->method
== Http::METHOD_HEAD
) {
1099 // XXX: initialize if we got here without HttpRequest::parseHeader()
1100 if (!request
->range
)
1101 request
->range
= req_hdr
->getRange();
1103 if (request
->range
) {
1104 request
->flags
.isRanged
=true;
1105 clientStreamNode
*node
= (clientStreamNode
*)http
->client_stream
.tail
->data
;
1106 /* XXX: This is suboptimal. We should give the stream the range set,
1107 * and thereby let the top of the stream set the offset when the
1108 * size becomes known. As it is, we will end up requesting from 0
1109 * for evey -X range specification.
1110 * RBC - this may be somewhat wrong. We should probably set the range
1111 * iter up at this point.
1113 node
->readBuffer
.offset
= request
->range
->lowestOffset(0);
1114 http
->range_iter
.pos
= request
->range
->begin();
1115 http
->range_iter
.valid
= true;
1119 /* Only HEAD and GET requests permit a Range or Request-Range header.
1120 * If these headers appear on any other type of request, delete them now.
1123 req_hdr
->delById(HDR_RANGE
);
1124 req_hdr
->delById(HDR_REQUEST_RANGE
);
1125 delete request
->range
;
1126 request
->range
= NULL
;
1129 if (req_hdr
->has(HDR_AUTHORIZATION
))
1130 request
->flags
.auth
= 1;
1132 clientCheckPinning(http
);
1134 if (request
->login
[0] != '\0')
1135 request
->flags
.auth
= 1;
1137 if (req_hdr
->has(HDR_VIA
)) {
1138 String s
= req_hdr
->getList(HDR_VIA
);
1140 * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1141 * Note ThisCache2 has a space prepended to the hostname so we don't
1142 * accidentally match super-domains.
1145 if (strListIsSubstr(&s
, ThisCache2
, ',')) {
1146 debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1147 request
, (ObjPackMethod
) & httpRequestPack
);
1148 request
->flags
.loopDetected
= 1;
1152 fvdbCountVia(s
.termedBuf());
1161 if (req_hdr
->has(HDR_X_FORWARDED_FOR
)) {
1162 String s
= req_hdr
->getList(HDR_X_FORWARDED_FOR
);
1163 fvdbCountForw(s
.termedBuf());
1169 request
->flags
.cachable
= http
->request
->maybeCacheable();
1171 if (clientHierarchical(http
))
1172 request
->flags
.hierarchical
= 1;
1174 debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1175 (request
->flags
.noCache
? "SET" : "NOT SET"));
1176 debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1177 (request
->flags
.cachable
? "SET" : "NOT SET"));
1178 debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1179 (request
->flags
.hierarchical
? "SET" : "NOT SET"));
1184 clientRedirectDoneWrapper(void *data
, char *result
)
1186 ClientRequestContext
*calloutContext
= (ClientRequestContext
*)data
;
1188 if (!calloutContext
->httpStateIsValid())
1191 calloutContext
->clientRedirectDone(result
);
1195 ClientRequestContext::clientRedirectDone(char *result
)
1197 HttpRequest
*old_request
= http
->request
;
1198 debugs(85, 5, "clientRedirectDone: '" << http
->uri
<< "' result=" << (result
? result
: "NULL"));
1199 assert(redirect_state
== REDIRECT_PENDING
);
1200 redirect_state
= REDIRECT_DONE
;
1203 http_status status
= (http_status
) atoi(result
);
1205 if (status
== HTTP_MOVED_PERMANENTLY
1206 || status
== HTTP_MOVED_TEMPORARILY
1207 || status
== HTTP_SEE_OTHER
1208 || status
== HTTP_PERMANENT_REDIRECT
1209 || status
== HTTP_TEMPORARY_REDIRECT
) {
1212 if ((t
= strchr(result
, ':')) != NULL
) {
1213 http
->redirect
.status
= status
;
1214 http
->redirect
.location
= xstrdup(t
+ 1);
1215 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1217 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid " << status
<< " redirect Location: " << result
);
1219 } else if (strcmp(result
, http
->uri
)) {
1220 // XXX: validate the URL properly *without* generating a whole new request object right here.
1221 // XXX: the clone() should be done only AFTER we know the new URL is valid.
1222 HttpRequest
*new_request
= old_request
->clone();
1223 if (urlParse(old_request
->method
, result
, new_request
)) {
1224 debugs(61,2, HERE
<< "URL-rewriter diverts URL from " << urlCanonical(old_request
) << " to " << urlCanonical(new_request
));
1226 // update the new request to flag the re-writing was done on it
1227 new_request
->flags
.redirected
= 1;
1229 // unlink bodypipe from the old request. Not needed there any longer.
1230 if (old_request
->body_pipe
!= NULL
) {
1231 old_request
->body_pipe
= NULL
;
1232 debugs(61,2, HERE
<< "URL-rewriter diverts body_pipe " << new_request
->body_pipe
<<
1233 " from request " << old_request
<< " to " << new_request
);
1236 // update the current working ClientHttpRequest fields
1237 safe_free(http
->uri
);
1238 http
->uri
= xstrdup(urlCanonical(new_request
));
1239 HTTPMSGUNLOCK(old_request
);
1240 http
->request
= HTTPMSGLOCK(new_request
);
1242 debugs(85, DBG_CRITICAL
, "ERROR: URL-rewrite produces invalid request: " <<
1243 old_request
->method
<< " " << result
<< " " << old_request
->http_ver
);
1249 /* FIXME PIPELINE: This is innacurate during pipelining */
1251 if (http
->getConn() != NULL
&& Comm::IsConnOpen(http
->getConn()->clientConnection
))
1252 fd_note(http
->getConn()->clientConnection
->fd
, http
->uri
);
1259 /** Test cache allow/deny configuration
1260 * Sets flags.cachable=1 if caching is not denied.
1263 ClientRequestContext::checkNoCache()
1265 if (Config
.accessList
.noCache
) {
1266 acl_checklist
= clientAclChecklistCreate(Config
.accessList
.noCache
, http
);
1267 acl_checklist
->nonBlockingCheck(checkNoCacheDoneWrapper
, this);
1269 /* unless otherwise specified, we try to cache. */
1270 checkNoCacheDone(ACCESS_ALLOWED
);
1275 checkNoCacheDoneWrapper(allow_t answer
, void *data
)
1277 ClientRequestContext
*calloutContext
= (ClientRequestContext
*) data
;
1279 if (!calloutContext
->httpStateIsValid())
1282 calloutContext
->checkNoCacheDone(answer
);
1286 ClientRequestContext::checkNoCacheDone(const allow_t
&answer
)
1288 acl_checklist
= NULL
;
1289 http
->request
->flags
.cachable
= (answer
== ACCESS_ALLOWED
);
1295 ClientRequestContext::sslBumpAccessCheck()
1297 // If SSL connection tunneling or bumping decision has been made, obey it.
1298 const Ssl::BumpMode bumpMode
= http
->getConn()->sslBumpMode
;
1299 if (bumpMode
!= Ssl::bumpEnd
) {
1300 debugs(85, 5, HERE
<< "SslBump already decided (" << bumpMode
<<
1301 "), " << "ignoring ssl_bump for " << http
->getConn());
1302 http
->al
->ssl
.bumpMode
= bumpMode
; // inherited from bumped connection
1306 // If we have not decided yet, decide whether to bump now.
1308 // Bumping here can only start with a CONNECT request on a bumping port
1309 // (bumping of intercepted SSL conns is decided before we get 1st request).
1310 // We also do not bump redirected CONNECT requests.
1311 if (http
->request
->method
!= Http::METHOD_CONNECT
|| http
->redirect
.status
||
1312 !Config
.accessList
.ssl_bump
|| !http
->getConn()->port
->sslBump
) {
1313 http
->al
->ssl
.bumpMode
= Ssl::bumpEnd
; // SslBump does not apply; log -
1314 debugs(85, 5, HERE
<< "cannot SslBump this request");
1318 // Do not bump during authentication: clients would not proxy-authenticate
1319 // if we delay a 407 response and respond with 200 OK to CONNECT.
1320 if (error
&& error
->httpStatus
== HTTP_PROXY_AUTHENTICATION_REQUIRED
) {
1321 http
->al
->ssl
.bumpMode
= Ssl::bumpEnd
; // SslBump does not apply; log -
1322 debugs(85, 5, HERE
<< "no SslBump during proxy authentication");
1326 debugs(85, 5, HERE
<< "SslBump possible, checking ACL");
1328 ACLFilledChecklist
*acl_checklist
= clientAclChecklistCreate(Config
.accessList
.ssl_bump
, http
);
1329 acl_checklist
->nonBlockingCheck(sslBumpAccessCheckDoneWrapper
, this);
1334 * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1335 * as ACLFilledChecklist callback
1338 sslBumpAccessCheckDoneWrapper(allow_t answer
, void *data
)
1340 ClientRequestContext
*calloutContext
= static_cast<ClientRequestContext
*>(data
);
1342 if (!calloutContext
->httpStateIsValid())
1344 calloutContext
->sslBumpAccessCheckDone(answer
);
1348 ClientRequestContext::sslBumpAccessCheckDone(const allow_t
&answer
)
1350 if (!httpStateIsValid())
1353 const Ssl::BumpMode bumpMode
= answer
== ACCESS_ALLOWED
?
1354 static_cast<Ssl::BumpMode
>(answer
.kind
) : Ssl::bumpNone
;
1355 http
->sslBumpNeed(bumpMode
); // for processRequest() to bump if needed
1356 http
->al
->ssl
.bumpMode
= bumpMode
; // for logging
1363 * Identify requests that do not go through the store and client side stream
1364 * and forward them to the appropriate location. All other requests, request
1368 ClientHttpRequest::processRequest()
1370 debugs(85, 4, "clientProcessRequest: " << RequestMethodStr(request
->method
) << " '" << uri
<< "'");
1372 if (request
->method
== Http::METHOD_CONNECT
&& !redirect
.status
) {
1374 if (sslBumpNeeded()) {
1379 logType
= LOG_TCP_MISS
;
1380 getConn()->stopReading(); // tunnels read for themselves
1381 tunnelStart(this, &out
.size
, &al
->http
.code
);
1389 ClientHttpRequest::httpStart()
1391 PROF_start(httpStart
);
1392 logType
= LOG_TAG_NONE
;
1393 debugs(85, 4, "ClientHttpRequest::httpStart: " << Format::log_tags
[logType
] << " for '" << uri
<< "'");
1395 /* no one should have touched this */
1396 assert(out
.offset
== 0);
1397 /* Use the Stream Luke */
1398 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->data
;
1399 clientStreamRead(node
, this, node
->readBuffer
);
1400 PROF_stop(httpStart
);
1406 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode
)
1408 debugs(83, 3, HERE
<< "sslBump required: "<< Ssl::bumpMode(mode
));
1409 sslBumpNeed_
= mode
;
1412 // called when comm_write has completed
1414 SslBumpEstablish(const Comm::ConnectionPointer
&, char *, size_t, comm_err_t errflag
, int, void *data
)
1416 ClientHttpRequest
*r
= static_cast<ClientHttpRequest
*>(data
);
1417 debugs(85, 5, HERE
<< "responded to CONNECT: " << r
<< " ? " << errflag
);
1419 assert(r
&& cbdataReferenceValid(r
));
1420 r
->sslBumpEstablish(errflag
);
1424 ClientHttpRequest::sslBumpEstablish(comm_err_t errflag
)
1426 // Bail out quickly on COMM_ERR_CLOSING - close handlers will tidy up
1427 if (errflag
== COMM_ERR_CLOSING
)
1431 debugs(85, 3, HERE
<< "CONNECT response failure in SslBump: " << errflag
);
1432 getConn()->clientConnection
->close();
1436 // We lack HttpReply which logRequest() uses to log the status code.
1437 // TODO: Use HttpReply instead of the "200 Connection established" string.
1438 al
->http
.code
= 200;
1441 // Preserve authentication info for the ssl-bumped request
1442 if (request
->auth_user_request
!= NULL
)
1443 getConn()->auth_user_request
= request
->auth_user_request
;
1446 assert(sslBumpNeeded());
1447 getConn()->switchToHttps(request
, sslBumpNeed_
);
1451 ClientHttpRequest::sslBumpStart()
1453 debugs(85, 5, HERE
<< "Confirming " << Ssl::bumpMode(sslBumpNeed_
) <<
1454 "-bumped CONNECT tunnel on FD " << getConn()->clientConnection
);
1455 getConn()->sslBumpMode
= sslBumpNeed_
;
1457 // send an HTTP 200 response to kick client SSL negotiation
1458 // TODO: Unify with tunnel.cc and add a Server(?) header
1459 static const char *const conn_established
= "HTTP/1.1 200 Connection established\r\n\r\n";
1460 AsyncCall::Pointer call
= commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1461 CommIoCbPtrFun(&SslBumpEstablish
, this));
1462 Comm::Write(getConn()->clientConnection
, conn_established
, strlen(conn_established
), call
, NULL
);
1468 ClientHttpRequest::gotEnough() const
1470 /** TODO: should be querying the stream. */
1471 int64_t contentLength
=
1472 memObject()->getReply()->bodySize(request
->method
);
1473 assert(contentLength
>= 0);
1475 if (out
.offset
< contentLength
)
1482 ClientHttpRequest::storeEntry(StoreEntry
*newEntry
)
1488 ClientHttpRequest::loggingEntry(StoreEntry
*newEntry
)
1491 loggingEntry_
->unlock();
1493 loggingEntry_
= newEntry
;
1496 loggingEntry_
->lock();
1500 * doCallouts() - This function controls the order of "callout"
1501 * executions, including non-blocking access control checks, the
1502 * redirector, and ICAP. Previously, these callouts were chained
1503 * together such that "clientAccessCheckDone()" would call
1504 * "clientRedirectStart()" and so on.
1506 * The ClientRequestContext (aka calloutContext) class holds certain
1507 * state data for the callout/callback operations. Previously
1508 * ClientHttpRequest would sort of hand off control to ClientRequestContext
1509 * for a short time. ClientRequestContext would then delete itself
1510 * and pass control back to ClientHttpRequest when all callouts
1513 * This caused some problems for ICAP because we want to make the
1514 * ICAP callout after checking ACLs, but before checking the no_cache
1515 * list. We can't stuff the ICAP state into the ClientRequestContext
1516 * class because we still need the ICAP state after ClientRequestContext
1519 * Note that ClientRequestContext is created before the first call
1522 * If one of the callouts notices that ClientHttpRequest is no
1523 * longer valid, it should call cbdataReferenceDone() so that
1524 * ClientHttpRequest's reference count goes to zero and it will get
1525 * deleted. ClientHttpRequest will then delete ClientRequestContext.
1527 * Note that we set the _done flags here before actually starting
1528 * the callout. This is strictly for convenience.
1531 tos_t
aclMapTOS (acl_tos
* head
, ACLChecklist
* ch
);
1532 nfmark_t
aclMapNfmark (acl_nfmark
* head
, ACLChecklist
* ch
);
1535 ClientHttpRequest::doCallouts()
1537 assert(calloutContext
);
1539 /*Save the original request for logging purposes*/
1540 if (!calloutContext
->http
->al
->request
)
1541 calloutContext
->http
->al
->request
= HTTPMSGLOCK(request
);
1543 if (!calloutContext
->error
) {
1544 // CVE-2009-0801: verify the Host: header is consistent with other known details.
1545 if (!calloutContext
->host_header_verify_done
) {
1546 debugs(83, 3, HERE
<< "Doing calloutContext->hostHeaderVerify()");
1547 calloutContext
->host_header_verify_done
= true;
1548 calloutContext
->hostHeaderVerify();
1552 if (!calloutContext
->http_access_done
) {
1553 debugs(83, 3, HERE
<< "Doing calloutContext->clientAccessCheck()");
1554 calloutContext
->http_access_done
= true;
1555 calloutContext
->clientAccessCheck();
1560 if (!calloutContext
->adaptation_acl_check_done
) {
1561 calloutContext
->adaptation_acl_check_done
= true;
1562 if (Adaptation::AccessCheck::Start(
1563 Adaptation::methodReqmod
, Adaptation::pointPreCache
,
1564 request
, NULL
, this))
1565 return; // will call callback
1569 if (!calloutContext
->redirect_done
) {
1570 calloutContext
->redirect_done
= true;
1571 assert(calloutContext
->redirect_state
== REDIRECT_NONE
);
1573 if (Config
.Program
.redirect
) {
1574 debugs(83, 3, HERE
<< "Doing calloutContext->clientRedirectStart()");
1575 calloutContext
->redirect_state
= REDIRECT_PENDING
;
1576 calloutContext
->clientRedirectStart();
1581 if (!calloutContext
->adapted_http_access_done
) {
1582 debugs(83, 3, HERE
<< "Doing calloutContext->clientAccessCheck2()");
1583 calloutContext
->adapted_http_access_done
= true;
1584 calloutContext
->clientAccessCheck2();
1588 if (!calloutContext
->interpreted_req_hdrs
) {
1589 debugs(83, 3, HERE
<< "Doing clientInterpretRequestHeaders()");
1590 calloutContext
->interpreted_req_hdrs
= 1;
1591 clientInterpretRequestHeaders(this);
1594 if (!calloutContext
->no_cache_done
) {
1595 calloutContext
->no_cache_done
= true;
1597 if (Config
.accessList
.noCache
&& request
->flags
.cachable
) {
1598 debugs(83, 3, HERE
<< "Doing calloutContext->checkNoCache()");
1599 calloutContext
->checkNoCache();
1603 } // if !calloutContext->error
1605 if (!calloutContext
->tosToClientDone
) {
1606 calloutContext
->tosToClientDone
= true;
1607 if (getConn() != NULL
&& Comm::IsConnOpen(getConn()->clientConnection
)) {
1608 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1609 ch
.src_addr
= request
->client_addr
;
1610 ch
.my_addr
= request
->my_addr
;
1611 tos_t tos
= aclMapTOS(Ip::Qos::TheConfig
.tosToClient
, &ch
);
1613 Ip::Qos::setSockTos(getConn()->clientConnection
, tos
);
1617 if (!calloutContext
->nfmarkToClientDone
) {
1618 calloutContext
->nfmarkToClientDone
= true;
1619 if (getConn() != NULL
&& Comm::IsConnOpen(getConn()->clientConnection
)) {
1620 ACLFilledChecklist
ch(NULL
, request
, NULL
);
1621 ch
.src_addr
= request
->client_addr
;
1622 ch
.my_addr
= request
->my_addr
;
1623 nfmark_t mark
= aclMapNfmark(Ip::Qos::TheConfig
.nfmarkToClient
, &ch
);
1625 Ip::Qos::setSockNfmark(getConn()->clientConnection
, mark
);
1630 // We need to check for SslBump even if the calloutContext->error is set
1631 // because bumping may require delaying the error until after CONNECT.
1632 if (!calloutContext
->sslBumpCheckDone
) {
1633 calloutContext
->sslBumpCheckDone
= true;
1634 if (calloutContext
->sslBumpAccessCheck())
1636 /* else no ssl bump required*/
1640 if (calloutContext
->error
) {
1641 const char *uri
= urlCanonical(request
);
1642 StoreEntry
*e
= storeCreateEntry(uri
, uri
, request
->flags
, request
->method
);
1644 if (sslBumpNeeded()) {
1645 // set final error but delay sending until we bump
1646 Ssl::ServerBump
*srvBump
= new Ssl::ServerBump(request
, e
);
1647 errorAppendEntry(e
, calloutContext
->error
);
1648 calloutContext
->error
= NULL
;
1649 getConn()->setServerBump(srvBump
);
1654 // send the error to the client now
1655 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
1656 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
1657 assert (repContext
);
1658 repContext
->setReplyToStoreEntry(e
);
1659 errorAppendEntry(e
, calloutContext
->error
);
1660 calloutContext
->error
= NULL
;
1661 if (calloutContext
->readNextRequest
)
1662 getConn()->flags
.readMore
= true; // resume any pipeline reads.
1663 node
= (clientStreamNode
*)client_stream
.tail
->data
;
1664 clientStreamRead(node
, this, node
->readBuffer
);
1670 cbdataReferenceDone(calloutContext
->http
);
1671 delete calloutContext
;
1672 calloutContext
= NULL
;
1675 headersLog(0, 1, request
->method
, request
);
1678 debugs(83, 3, HERE
<< "calling processRequest()");
1682 Adaptation::Icap::History::Pointer ih
= request
->icapHistory();
1684 ih
->logType
= logType
;
1689 #include "client_side_request.cci"
1693 /// Initiate an asynchronous adaptation transaction which will call us back.
1695 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer
&g
)
1697 debugs(85, 3, HERE
<< "adaptation needed for " << this);
1698 assert(!virginHeadSource
);
1699 assert(!adaptedBodySource
);
1700 virginHeadSource
= initiateAdaptation(
1701 new Adaptation::Iterator(request
, NULL
, g
));
1703 // we could try to guess whether we can bypass this adaptation
1704 // initiation failure, but it should not really happen
1705 Must(initiated(virginHeadSource
));
1709 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer
&answer
)
1711 assert(cbdataReferenceValid(this)); // indicates bug
1712 clearAdaptation(virginHeadSource
);
1713 assert(!adaptedBodySource
);
1715 switch (answer
.kind
) {
1716 case Adaptation::Answer::akForward
:
1717 handleAdaptedHeader(answer
.message
);
1720 case Adaptation::Answer::akBlock
:
1721 handleAdaptationBlock(answer
);
1724 case Adaptation::Answer::akError
:
1725 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT
, !answer
.final
);
1731 ClientHttpRequest::handleAdaptedHeader(HttpMsg
*msg
)
1735 if (HttpRequest
*new_req
= dynamic_cast<HttpRequest
*>(msg
)) {
1737 * Replace the old request with the new request.
1739 HTTPMSGUNLOCK(request
);
1740 request
= HTTPMSGLOCK(new_req
);
1742 * Store the new URI for logging
1745 uri
= xstrdup(urlCanonical(request
));
1746 setLogUri(this, urlCanonicalClean(request
));
1747 assert(request
->method
.id());
1748 } else if (HttpReply
*new_rep
= dynamic_cast<HttpReply
*>(msg
)) {
1749 debugs(85,3,HERE
<< "REQMOD reply is HTTP reply");
1751 // subscribe to receive reply body
1752 if (new_rep
->body_pipe
!= NULL
) {
1753 adaptedBodySource
= new_rep
->body_pipe
;
1754 int consumer_ok
= adaptedBodySource
->setConsumerIfNotLate(this);
1755 assert(consumer_ok
);
1758 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
1759 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
1760 repContext
->createStoreEntry(request
->method
, request
->flags
);
1762 EBIT_CLR(storeEntry()->flags
, ENTRY_FWD_HDR_WAIT
);
1763 request_satisfaction_mode
= true;
1764 request_satisfaction_offset
= 0;
1765 storeEntry()->replaceHttpReply(new_rep
);
1766 storeEntry()->timestampsSet();
1768 if (!adaptedBodySource
) // no body
1769 storeEntry()->complete();
1770 clientGetMoreData(node
, this);
1773 // we are done with getting headers (but may be receiving body)
1774 clearAdaptation(virginHeadSource
);
1776 if (!request_satisfaction_mode
)
1781 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer
&answer
)
1783 request
->detailError(ERR_ACCESS_DENIED
, ERR_DETAIL_REQMOD_BLOCK
);
1784 AclMatchedName
= answer
.ruleId
.termedBuf();
1785 assert(calloutContext
);
1786 calloutContext
->clientAccessCheckDone(ACCESS_DENIED
);
1787 AclMatchedName
= NULL
;
1791 ClientHttpRequest::resumeBodyStorage()
1793 if (!adaptedBodySource
)
1796 noteMoreBodyDataAvailable(adaptedBodySource
);
1800 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer
)
1802 assert(request_satisfaction_mode
);
1803 assert(adaptedBodySource
!= NULL
);
1805 if (size_t contentSize
= adaptedBodySource
->buf().contentSize()) {
1806 const size_t spaceAvailable
= storeEntry()->bytesWanted(Range
<size_t>(0,contentSize
));
1808 if (spaceAvailable
< contentSize
) {
1809 // No or partial body data consuming
1810 typedef NullaryMemFunT
<ClientHttpRequest
> Dialer
;
1811 AsyncCall::Pointer call
= asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1812 Dialer(this, &ClientHttpRequest::resumeBodyStorage
));
1813 storeEntry()->deferProducer(call
);
1816 if (!spaceAvailable
)
1819 if (spaceAvailable
< contentSize
)
1820 contentSize
= spaceAvailable
;
1822 BodyPipeCheckout
bpc(*adaptedBodySource
);
1823 const StoreIOBuffer
ioBuf(&bpc
.buf
, request_satisfaction_offset
, contentSize
);
1824 storeEntry()->write(ioBuf
);
1825 // assume StoreEntry::write() writes the entire ioBuf
1826 request_satisfaction_offset
+= ioBuf
.length
;
1827 bpc
.buf
.consume(contentSize
);
1831 if (adaptedBodySource
->exhausted())
1832 endRequestSatisfaction();
1833 // else wait for more body data
1837 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer
)
1839 assert(!virginHeadSource
);
1840 // should we end request satisfaction now?
1841 if (adaptedBodySource
!= NULL
&& adaptedBodySource
->exhausted())
1842 endRequestSatisfaction();
1846 ClientHttpRequest::endRequestSatisfaction()
1848 debugs(85,4, HERE
<< this << " ends request satisfaction");
1849 assert(request_satisfaction_mode
);
1850 stopConsumingFrom(adaptedBodySource
);
1852 // TODO: anything else needed to end store entry formation correctly?
1853 storeEntry()->complete();
1857 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer
)
1859 assert(!virginHeadSource
);
1860 stopConsumingFrom(adaptedBodySource
);
1862 debugs(85,3, HERE
<< "REQMOD body production failed");
1863 if (request_satisfaction_mode
) { // too late to recover or serve an error
1864 request
->detailError(ERR_ICAP_FAILURE
, ERR_DETAIL_CLT_REQMOD_RESP_BODY
);
1865 const Comm::ConnectionPointer c
= getConn()->clientConnection
;
1866 Must(Comm::IsConnOpen(c
));
1867 c
->close(); // drastic, but we may be writing a response already
1869 handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY
);
1874 ClientHttpRequest::handleAdaptationFailure(int errDetail
, bool bypassable
)
1876 debugs(85,3, HERE
<< "handleAdaptationFailure(" << bypassable
<< ")");
1878 const bool usedStore
= storeEntry() && !storeEntry()->isEmpty();
1879 const bool usedPipe
= request
->body_pipe
!= NULL
&&
1880 request
->body_pipe
->consumedSize() > 0;
1882 if (bypassable
&& !usedStore
&& !usedPipe
) {
1883 debugs(85,3, HERE
<< "ICAP REQMOD callout failed, bypassing: " << calloutContext
);
1889 debugs(85,3, HERE
<< "ICAP REQMOD callout failed, responding with error");
1891 clientStreamNode
*node
= (clientStreamNode
*)client_stream
.tail
->prev
->data
;
1892 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
1895 // The original author of the code also wanted to pass an errno to
1896 // setReplyToError, but it seems unlikely that the errno reflects the
1897 // true cause of the error at this point, so I did not pass it.
1898 if (calloutContext
) {
1901 ConnStateData
* c
= getConn();
1902 calloutContext
->error
= clientBuildError(ERR_ICAP_FAILURE
, HTTP_INTERNAL_SERVER_ERROR
,
1904 c
!= NULL
? c
->clientConnection
->remote
: noAddr
,
1908 calloutContext
->error
->auth_user_request
=
1909 c
!= NULL
&& c
->auth_user_request
!= NULL
? c
->auth_user_request
: request
->auth_user_request
;
1911 calloutContext
->error
->detailError(errDetail
);
1912 calloutContext
->readNextRequest
= true;
1913 c
->expectNoForwarding();
1916 //else if(calloutContext == NULL) is it possible?