3 * $Id: client_side.cc,v 1.518 2000/12/17 07:18:56 hno Exp $
5 * DEBUG: section 33 Client-side Routines
6 * AUTHOR: Duane Wessels
8 * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
9 * ----------------------------------------------------------
11 * Squid is the result of efforts by numerous individuals from the
12 * Internet community. Development is led by Duane Wessels of the
13 * National Laboratory for Applied Network Research and funded by the
14 * National Science Foundation. Squid is Copyrighted (C) 1998 by
15 * the Regents of the University of California. Please see the
16 * COPYRIGHT file for full details. Squid incorporates software
17 * developed and/or copyrighted by other sources. Please see the
18 * CREDITS file for full details.
20 * This program is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
30 * You should have received a copy of the GNU General Public License
31 * along with this program; if not, write to the Free Software
32 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
40 #include <sys/ioctl.h>
42 #include <netinet/tcp.h>
44 #if HAVE_IP_FIL_COMPAT_H
45 #include <ip_fil_compat.h>
46 #elif HAVE_NETINET_IP_FIL_COMPAT_H
47 #include <netinet/ip_fil_compat.h>
48 #elif HAVE_IP_COMPAT_H
49 #include <ip_compat.h>
50 #elif HAVE_NETINET_IP_COMPAT_H
51 #include <netinet/ip_compat.h>
55 #elif HAVE_NETINET_IP_FIL_H
56 #include <netinet/ip_fil.h>
60 #elif HAVE_NETINET_IP_NAT_H
61 #include <netinet/ip_nat.h>
66 #include <linux/netfilter_ipv4.h>
71 #define comm_close comm_lingering_close
74 static const char *const crlf
= "\r\n";
76 #define FAILURE_MODE_TIME 300
80 static CWCB clientWriteComplete
;
81 static CWCB clientWriteBodyComplete
;
82 static PF clientReadRequest
;
83 static PF connStateFree
;
84 static PF requestTimeout
;
85 static PF clientLifetimeTimeout
;
86 static int clientCheckTransferDone(clientHttpRequest
*);
87 static int clientGotNotEnough(clientHttpRequest
*);
88 static void checkFailureRatio(err_type
, hier_code
);
89 static void clientProcessMiss(clientHttpRequest
*);
90 static void clientBuildReplyHeader(clientHttpRequest
* http
, HttpReply
* rep
);
91 static clientHttpRequest
*parseHttpRequestAbort(ConnStateData
* conn
, const char *uri
);
92 static clientHttpRequest
*parseHttpRequest(ConnStateData
*, method_t
*, int *, char **, size_t *);
93 static RH clientRedirectDone
;
94 static void clientCheckNoCache(clientHttpRequest
*);
95 static void clientCheckNoCacheDone(int answer
, void *data
);
96 static STCB clientHandleIMSReply
;
97 static int clientGetsOldEntry(StoreEntry
* new, StoreEntry
* old
, request_t
* request
);
98 static int checkAccelOnly(clientHttpRequest
*);
100 static IDCB clientIdentDone
;
102 static int clientOnlyIfCached(clientHttpRequest
* http
);
103 static STCB clientSendMoreData
;
104 static STCB clientCacheHit
;
105 static void clientSetKeepaliveFlag(clientHttpRequest
*);
106 static void clientPackRangeHdr(const HttpReply
* rep
, const HttpHdrRangeSpec
* spec
, String boundary
, MemBuf
* mb
);
107 static void clientPackTermBound(String boundary
, MemBuf
* mb
);
108 static void clientInterpretRequestHeaders(clientHttpRequest
*);
109 static void clientProcessRequest(clientHttpRequest
*);
110 static void clientProcessExpired(void *data
);
111 static void clientProcessOnlyIfCachedMiss(clientHttpRequest
* http
);
112 static int clientCachable(clientHttpRequest
* http
);
113 static int clientHierarchical(clientHttpRequest
* http
);
114 static int clientCheckContentLength(request_t
* r
);
115 static DEFER httpAcceptDefer
;
116 static log_type
clientProcessRequest2(clientHttpRequest
* http
);
117 static int clientReplyBodyTooLarge(int clen
);
118 static int clientRequestBodyTooLarge(int clen
);
121 checkAccelOnly(clientHttpRequest
* http
)
123 /* return TRUE if someone makes a proxy request to us and
124 * we are in httpd-accel only mode */
125 if (!Config2
.Accel
.on
)
127 if (Config
.onoff
.accel_with_proxy
)
129 if (http
->request
->protocol
== PROTO_CACHEOBJ
)
131 if (http
->flags
.accel
)
138 clientIdentDone(const char *ident
, void *data
)
140 ConnStateData
*conn
= data
;
142 xstrncpy(conn
->ident
, ident
, sizeof(conn
->ident
));
144 xstrncpy(conn
->ident
, "-", sizeof(conn
->ident
));
149 clientAclChecklistCreate(const acl_access
* acl
, const clientHttpRequest
* http
)
152 ConnStateData
*conn
= http
->conn
;
153 ch
= aclChecklistCreate(acl
,
158 * hack for ident ACL. It needs to get full addresses, and a
159 * place to store the ident result on persistent connections...
162 cbdataLock(ch
->conn
);
168 clientAccessCheck(void *data
)
170 clientHttpRequest
*http
= data
;
171 if (checkAccelOnly(http
)) {
172 /* deny proxy requests in accel_only mode */
173 debug(33, 1) ("clientAccessCheck: proxy request denied in accel_only mode\n");
174 clientAccessCheckDone(ACCESS_DENIED
, http
);
177 http
->acl_checklist
= clientAclChecklistCreate(Config
.accessList
.http
, http
);
178 aclNBCheck(http
->acl_checklist
, clientAccessCheckDone
, http
);
182 * returns true if client specified that the object must come from the cache
183 * without contacting origin server
186 clientOnlyIfCached(clientHttpRequest
* http
)
188 const request_t
*r
= http
->request
;
190 return r
->cache_control
&&
191 EBIT_TEST(r
->cache_control
->mask
, CC_ONLY_IF_CACHED
);
195 clientCreateStoreEntry(clientHttpRequest
* h
, method_t m
, request_flags flags
)
199 * For erroneous requests, we might not have a h->request,
200 * so make a fake one.
202 if (h
->request
== NULL
)
203 h
->request
= requestLink(requestCreate(m
, PROTO_NONE
, null_string
));
204 e
= storeCreateEntry(h
->uri
, h
->log_uri
, flags
, m
);
205 h
->sc
= storeClientListAdd(e
, h
);
207 delaySetStoreClient(h
->sc
, delayClient(h
->request
));
209 storeClientCopy(h
->sc
, e
, 0, 0, CLIENT_SOCK_SZ
,
210 memAllocate(MEM_CLIENT_SOCK_BUF
), clientSendMoreData
, h
);
215 clientAccessCheckDone(int answer
, void *data
)
217 clientHttpRequest
*http
= data
;
220 ErrorState
*err
= NULL
;
221 char *proxy_auth_msg
= NULL
;
222 debug(33, 2) ("The request %s %s is %s, because it matched '%s'\n",
223 RequestMethodStr
[http
->request
->method
], http
->uri
,
224 answer
== ACCESS_ALLOWED
? "ALLOWED" : "DENIED",
225 AclMatchedName
? AclMatchedName
: "NO ACL's");
226 if (http
->acl_checklist
->auth_user
)
227 proxy_auth_msg
= http
->acl_checklist
->auth_user
->message
;
228 http
->acl_checklist
= NULL
;
229 if (answer
== ACCESS_ALLOWED
) {
230 safe_free(http
->uri
);
231 http
->uri
= xstrdup(urlCanonical(http
->request
));
232 assert(http
->redirect_state
== REDIRECT_NONE
);
233 http
->redirect_state
= REDIRECT_PENDING
;
234 redirectStart(http
, clientRedirectDone
, http
);
236 debug(33, 5) ("Access Denied: %s\n", http
->uri
);
237 debug(33, 5) ("AclMatchedName = %s\n",
238 AclMatchedName
? AclMatchedName
: "<null>");
239 debug(33, 5) ("Proxy Auth Message = %s\n",
240 proxy_auth_msg
? proxy_auth_msg
: "<null>");
242 * NOTE: get page_id here, based on AclMatchedName because
243 * if USE_DELAY_POOLS is enabled, then AclMatchedName gets
244 * clobbered in the clientCreateStoreEntry() call
245 * just below. Pedro Ribeiro <pribeiro@isel.pt>
247 page_id
= aclGetDenyInfoPage(&Config
.denyInfoList
, AclMatchedName
);
248 http
->log_type
= LOG_TCP_DENIED
;
249 http
->entry
= clientCreateStoreEntry(http
, http
->request
->method
,
251 if (answer
== ACCESS_REQ_PROXY_AUTH
|| aclIsProxyAuth(AclMatchedName
)) {
252 if (!http
->flags
.accel
) {
253 /* Proxy authorisation needed */
254 status
= HTTP_PROXY_AUTHENTICATION_REQUIRED
;
256 /* WWW authorisation needed */
257 status
= HTTP_UNAUTHORIZED
;
260 page_id
= ERR_CACHE_ACCESS_DENIED
;
262 status
= HTTP_FORBIDDEN
;
264 page_id
= ERR_ACCESS_DENIED
;
266 err
= errorCon(page_id
, status
);
267 err
->request
= requestLink(http
->request
);
268 err
->src_addr
= http
->conn
->peer
.sin_addr
;
269 err
->proxy_auth_msg
= proxy_auth_msg
;
270 err
->callback_data
= NULL
;
271 errorAppendEntry(http
->entry
, err
);
276 clientRedirectDone(void *data
, char *result
)
278 clientHttpRequest
*http
= data
;
279 request_t
*new_request
= NULL
;
280 request_t
*old_request
= http
->request
;
281 debug(33, 5) ("clientRedirectDone: '%s' result=%s\n", http
->uri
,
282 result
? result
: "NULL");
283 assert(http
->redirect_state
== REDIRECT_PENDING
);
284 http
->redirect_state
= REDIRECT_DONE
;
286 http_status status
= atoi(result
);
287 if (status
== 301 || status
== 302) {
289 if ((t
= strchr(result
, ':')) != NULL
) {
290 http
->redirect
.status
= status
;
291 http
->redirect
.location
= xstrdup(t
+ 1);
293 debug(33, 1) ("clientRedirectDone: bad input: %s\n", result
);
296 if (strcmp(result
, http
->uri
))
297 new_request
= urlParse(old_request
->method
, result
);
300 safe_free(http
->uri
);
301 http
->uri
= xstrdup(urlCanonical(new_request
));
302 new_request
->http_ver
= old_request
->http_ver
;
303 httpHeaderAppend(&new_request
->header
, &old_request
->header
);
304 new_request
->client_addr
= old_request
->client_addr
;
305 new_request
->my_addr
= old_request
->my_addr
;
306 new_request
->my_port
= old_request
->my_port
;
307 new_request
->flags
.redirected
= 1;
308 if (old_request
->user_ident
[0])
309 xstrncpy(new_request
->user_ident
, old_request
->user_ident
,
311 if (old_request
->body
) {
312 new_request
->body
= xmalloc(old_request
->body_sz
);
313 xmemcpy(new_request
->body
, old_request
->body
, old_request
->body_sz
);
314 new_request
->body_sz
= old_request
->body_sz
;
316 new_request
->content_length
= old_request
->content_length
;
317 new_request
->flags
.proxy_keepalive
= old_request
->flags
.proxy_keepalive
;
318 requestUnlink(old_request
);
319 http
->request
= requestLink(new_request
);
321 clientInterpretRequestHeaders(http
);
323 headersLog(0, 1, request
->method
, request
);
325 fd_note(http
->conn
->fd
, http
->uri
);
326 clientCheckNoCache(http
);
330 clientCheckNoCache(clientHttpRequest
* http
)
332 if (Config
.accessList
.noCache
&& http
->request
->flags
.cachable
) {
333 http
->acl_checklist
= clientAclChecklistCreate(Config
.accessList
.noCache
, http
);
334 aclNBCheck(http
->acl_checklist
, clientCheckNoCacheDone
, http
);
336 clientCheckNoCacheDone(http
->request
->flags
.cachable
, http
);
341 clientCheckNoCacheDone(int answer
, void *data
)
343 clientHttpRequest
*http
= data
;
344 http
->request
->flags
.cachable
= answer
;
345 http
->acl_checklist
= NULL
;
346 clientProcessRequest(http
);
350 clientProcessExpired(void *data
)
352 clientHttpRequest
*http
= data
;
353 char *url
= http
->uri
;
354 StoreEntry
*entry
= NULL
;
355 debug(33, 3) ("clientProcessExpired: '%s'\n", http
->uri
);
356 assert(http
->entry
->lastmod
>= 0);
358 * check if we are allowed to contact other servers
359 * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
360 * a stale entry *if* it matches client requirements
362 if (clientOnlyIfCached(http
)) {
363 clientProcessOnlyIfCachedMiss(http
);
366 http
->request
->flags
.refresh
= 1;
367 http
->old_entry
= http
->entry
;
368 http
->old_sc
= http
->sc
;
370 * Assert that 'http' is already a client of old_entry. If
371 * it is not, then the beginning of the object data might get
372 * freed from memory before we need to access it.
374 assert(http
->sc
->callback_data
== http
);
375 entry
= storeCreateEntry(url
,
377 http
->request
->flags
,
378 http
->request
->method
);
379 /* NOTE, don't call storeLockObject(), storeCreateEntry() does it */
380 http
->sc
= storeClientListAdd(entry
, http
);
382 /* delay_id is already set on original store client */
383 delaySetStoreClient(http
->sc
, delayClient(http
->request
));
385 http
->request
->lastmod
= http
->old_entry
->lastmod
;
386 debug(33, 5) ("clientProcessExpired: lastmod %d\n", (int) entry
->lastmod
);
388 http
->out
.offset
= 0;
389 fwdStart(http
->conn
->fd
, http
->entry
, http
->request
);
390 /* Register with storage manager to receive updates when data comes in. */
391 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
))
392 debug(33, 0) ("clientProcessExpired: found ENTRY_ABORTED object\n");
393 storeClientCopy(http
->sc
, entry
,
397 memAllocate(MEM_CLIENT_SOCK_BUF
),
398 clientHandleIMSReply
,
403 clientGetsOldEntry(StoreEntry
* new_entry
, StoreEntry
* old_entry
, request_t
* request
)
405 const http_status status
= new_entry
->mem_obj
->reply
->sline
.status
;
407 debug(33, 5) ("clientGetsOldEntry: YES, broken HTTP reply\n");
410 /* If the reply is a failure then send the old object as a last
412 if (status
>= 500 && status
< 600) {
413 debug(33, 3) ("clientGetsOldEntry: YES, failure reply=%d\n", status
);
416 /* If the reply is anything but "Not Modified" then
417 * we must forward it to the client */
418 if (HTTP_NOT_MODIFIED
!= status
) {
419 debug(33, 5) ("clientGetsOldEntry: NO, reply=%d\n", status
);
422 /* If the client did not send IMS in the request, then it
423 * must get the old object, not this "Not Modified" reply */
424 if (!request
->flags
.ims
) {
425 debug(33, 5) ("clientGetsOldEntry: YES, no client IMS\n");
428 /* If the client IMS time is prior to the entry LASTMOD time we
429 * need to send the old object */
430 if (modifiedSince(old_entry
, request
)) {
431 debug(33, 5) ("clientGetsOldEntry: YES, modified since %d\n",
435 debug(33, 5) ("clientGetsOldEntry: NO, new one is fine\n");
441 clientHandleIMSReply(void *data
, char *buf
, ssize_t size
)
443 clientHttpRequest
*http
= data
;
444 StoreEntry
*entry
= http
->entry
;
446 const char *url
= storeUrl(entry
);
447 int unlink_request
= 0;
448 StoreEntry
*oldentry
;
451 debug(33, 3) ("clientHandleIMSReply: %s, %d bytes\n", url
, (int) size
);
453 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
456 if (size
< 0 && !EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
457 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
460 mem
= entry
->mem_obj
;
461 status
= mem
->reply
->sline
.status
;
462 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
463 debug(33, 3) ("clientHandleIMSReply: ABORTED '%s'\n", url
);
464 /* We have an existing entry, but failed to validate it */
465 /* Its okay to send the old one anyway */
466 http
->log_type
= LOG_TCP_REFRESH_FAIL_HIT
;
467 storeUnregister(http
->sc
, entry
, http
);
468 storeUnlockObject(entry
);
469 entry
= http
->entry
= http
->old_entry
;
470 http
->sc
= http
->old_sc
;
471 } else if (STORE_PENDING
== entry
->store_status
&& 0 == status
) {
472 debug(33, 3) ("clientHandleIMSReply: Incomplete headers for '%s'\n", url
);
473 if (size
>= CLIENT_SOCK_SZ
) {
474 /* will not get any bigger than that */
475 debug(33, 3) ("clientHandleIMSReply: Reply is too large '%s', using old entry\n", url
);
476 /* use old entry, this repeats the code abovez */
477 http
->log_type
= LOG_TCP_REFRESH_FAIL_HIT
;
478 storeUnregister(http
->sc
, entry
, http
);
479 storeUnlockObject(entry
);
480 entry
= http
->entry
= http
->old_entry
;
481 http
->sc
= http
->old_sc
;
484 storeClientCopy(http
->sc
, entry
,
485 http
->out
.offset
+ size
,
489 clientHandleIMSReply
,
493 } else if (clientGetsOldEntry(entry
, http
->old_entry
, http
->request
)) {
494 /* We initiated the IMS request, the client is not expecting
495 * 304, so put the good one back. First, make sure the old entry
496 * headers have been loaded from disk. */
497 oldentry
= http
->old_entry
;
498 http
->log_type
= LOG_TCP_REFRESH_HIT
;
499 if (oldentry
->mem_obj
->request
== NULL
) {
500 oldentry
->mem_obj
->request
= requestLink(mem
->request
);
503 /* Don't memcpy() the whole reply structure here. For example,
504 * www.thegist.com (Netscape/1.13) returns a content-length for
505 * 304's which seems to be the length of the 304 HEADERS!!! and
506 * not the body they refer to. */
507 httpReplyUpdateOnNotModified(oldentry
->mem_obj
->reply
, mem
->reply
);
508 storeTimestampsSet(oldentry
);
509 storeUnregister(http
->sc
, entry
, http
);
510 http
->sc
= http
->old_sc
;
511 storeUnlockObject(entry
);
512 entry
= http
->entry
= oldentry
;
513 entry
->timestamp
= squid_curtime
;
514 if (unlink_request
) {
515 requestUnlink(entry
->mem_obj
->request
);
516 entry
->mem_obj
->request
= NULL
;
519 /* the client can handle this reply, whatever it is */
520 http
->log_type
= LOG_TCP_REFRESH_MISS
;
521 if (HTTP_NOT_MODIFIED
== mem
->reply
->sline
.status
) {
522 httpReplyUpdateOnNotModified(http
->old_entry
->mem_obj
->reply
,
524 storeTimestampsSet(http
->old_entry
);
525 http
->log_type
= LOG_TCP_REFRESH_HIT
;
527 storeUnregister(http
->old_sc
, http
->old_entry
, http
);
528 storeUnlockObject(http
->old_entry
);
531 http
->old_entry
= NULL
; /* done with old_entry */
533 assert(!EBIT_TEST(entry
->flags
, ENTRY_ABORTED
));
535 storeClientCopy(http
->sc
, entry
,
543 clientSendMoreData(data
, buf
, size
);
548 modifiedSince(StoreEntry
* entry
, request_t
* request
)
551 MemObject
*mem
= entry
->mem_obj
;
552 time_t mod_time
= entry
->lastmod
;
553 debug(33, 3) ("modifiedSince: '%s'\n", storeUrl(entry
));
555 mod_time
= entry
->timestamp
;
556 debug(33, 3) ("modifiedSince: mod_time = %d\n", (int) mod_time
);
559 /* Find size of the object */
560 object_length
= mem
->reply
->content_length
;
561 if (object_length
< 0)
562 object_length
= contentLen(entry
);
563 if (mod_time
> request
->ims
) {
564 debug(33, 3) ("--> YES: entry newer than client\n");
566 } else if (mod_time
< request
->ims
) {
567 debug(33, 3) ("--> NO: entry older than client\n");
569 } else if (request
->imslen
< 0) {
570 debug(33, 3) ("--> NO: same LMT, no client length\n");
572 } else if (request
->imslen
== object_length
) {
573 debug(33, 3) ("--> NO: same LMT, same length\n");
576 debug(33, 3) ("--> YES: same LMT, different length\n");
582 clientPurgeRequest(clientHttpRequest
* http
)
585 ErrorState
*err
= NULL
;
588 http_version_t version
;
589 debug(33, 3) ("Config2.onoff.enable_purge = %d\n", Config2
.onoff
.enable_purge
);
590 if (!Config2
.onoff
.enable_purge
) {
591 http
->log_type
= LOG_TCP_DENIED
;
592 err
= errorCon(ERR_ACCESS_DENIED
, HTTP_FORBIDDEN
);
593 err
->request
= requestLink(http
->request
);
594 err
->src_addr
= http
->conn
->peer
.sin_addr
;
595 http
->entry
= clientCreateStoreEntry(http
, http
->request
->method
, null_request_flags
);
596 errorAppendEntry(http
->entry
, err
);
599 http
->log_type
= LOG_TCP_MISS
;
600 /* Release both IP and object cache entries */
601 ipcacheInvalidate(http
->request
->host
);
602 if ((entry
= storeGetPublic(http
->uri
, METHOD_GET
)) == NULL
) {
603 status
= HTTP_NOT_FOUND
;
608 debug(33, 4) ("clientPurgeRequest: Not modified '%s'\n",
611 * Make a new entry to hold the reply to be written
614 http
->entry
= clientCreateStoreEntry(http
, http
->request
->method
, null_request_flags
);
615 httpReplyReset(r
= http
->entry
->mem_obj
->reply
);
616 httpBuildVersion(&version
, 1, 0);
617 httpReplySetHeaders(r
, version
, status
, NULL
, NULL
, 0, 0, -1);
618 httpReplySwapOut(r
, http
->entry
);
619 storeComplete(http
->entry
);
623 checkNegativeHit(StoreEntry
* e
)
625 if (!EBIT_TEST(e
->flags
, ENTRY_NEGCACHED
))
627 if (e
->expires
<= squid_curtime
)
629 if (e
->store_status
!= STORE_OK
)
635 clientUpdateCounters(clientHttpRequest
* http
)
637 int svc_time
= tvSubMsec(http
->start
, current_time
);
639 HierarchyLogEntry
*H
;
640 statCounter
.client_http
.requests
++;
641 if (isTcpHit(http
->log_type
))
642 statCounter
.client_http
.hits
++;
643 if (http
->log_type
== LOG_TCP_HIT
)
644 statCounter
.client_http
.disk_hits
++;
645 else if (http
->log_type
== LOG_TCP_MEM_HIT
)
646 statCounter
.client_http
.mem_hits
++;
647 if (http
->request
->err_type
!= ERR_NONE
)
648 statCounter
.client_http
.errors
++;
649 statHistCount(&statCounter
.client_http
.all_svc_time
, svc_time
);
651 * The idea here is not to be complete, but to get service times
652 * for only well-defined types. For example, we don't include
653 * LOG_TCP_REFRESH_FAIL_HIT because its not really a cache hit
654 * (we *tried* to validate it, but failed).
656 switch (http
->log_type
) {
657 case LOG_TCP_REFRESH_HIT
:
658 statHistCount(&statCounter
.client_http
.nh_svc_time
, svc_time
);
660 case LOG_TCP_IMS_HIT
:
661 statHistCount(&statCounter
.client_http
.nm_svc_time
, svc_time
);
664 case LOG_TCP_MEM_HIT
:
665 case LOG_TCP_OFFLINE_HIT
:
666 statHistCount(&statCounter
.client_http
.hit_svc_time
, svc_time
);
669 case LOG_TCP_CLIENT_REFRESH_MISS
:
670 statHistCount(&statCounter
.client_http
.miss_svc_time
, svc_time
);
673 /* make compiler warnings go away */
676 H
= &http
->request
->hier
;
679 statCounter
.cd
.times_used
++;
682 statCounter
.icp
.times_used
++;
684 if (0 != i
->stop
.tv_sec
&& 0 != i
->start
.tv_sec
)
685 statHistCount(&statCounter
.icp
.query_svc_time
,
686 tvSubUsec(i
->start
, i
->stop
));
688 statCounter
.icp
.query_timeouts
++;
691 statCounter
.netdb
.times_used
++;
699 httpRequestFree(void *data
)
701 clientHttpRequest
*http
= data
;
702 clientHttpRequest
**H
;
703 ConnStateData
*conn
= http
->conn
;
705 request_t
*request
= http
->request
;
706 MemObject
*mem
= NULL
;
707 debug(33, 3) ("httpRequestFree: %s\n", storeUrl(http
->entry
));
708 if (!clientCheckTransferDone(http
)) {
711 * DW: this seems odd here, is it really needed? It causes
712 * incomplete transfers to get logged with "000" status
713 * code because http->entry becomes NULL.
715 if ((e
= http
->entry
)) {
717 storeUnregister(http
->sc
, e
, http
);
718 storeUnlockObject(e
);
721 if (http
->entry
&& http
->entry
->ping_status
== PING_WAITING
)
722 storeReleaseRequest(http
->entry
);
724 assert(http
->log_type
< LOG_TYPE_MAX
);
726 mem
= http
->entry
->mem_obj
;
727 if (http
->out
.size
|| http
->log_type
) {
728 http
->al
.icp
.opcode
= ICP_INVALID
;
729 http
->al
.url
= http
->log_uri
;
730 debug(33, 9) ("httpRequestFree: al.url='%s'\n", http
->al
.url
);
732 http
->al
.http
.code
= mem
->reply
->sline
.status
;
733 http
->al
.http
.content_type
= strBuf(mem
->reply
->content_type
);
735 http
->al
.cache
.caddr
= conn
->log_addr
;
736 http
->al
.cache
.size
= http
->out
.size
;
737 http
->al
.cache
.code
= http
->log_type
;
738 http
->al
.cache
.msec
= tvSubMsec(http
->start
, current_time
);
743 packerToMemInit(&p
, &mb
);
744 httpHeaderPackInto(&request
->header
, &p
);
745 http
->al
.http
.method
= request
->method
;
746 http
->al
.http
.version
= request
->http_ver
;
747 http
->al
.headers
.request
= xstrdup(mb
.buf
);
748 http
->al
.hier
= request
->hier
;
749 if (request
->user_ident
[0])
750 http
->al
.cache
.ident
= request
->user_ident
;
752 http
->al
.cache
.ident
= conn
->ident
;
756 accessLogLog(&http
->al
);
757 clientUpdateCounters(http
);
758 clientdbUpdate(conn
->peer
.sin_addr
, http
->log_type
, PROTO_HTTP
, http
->out
.size
);
760 if (http
->acl_checklist
)
761 aclChecklistFree(http
->acl_checklist
);
763 checkFailureRatio(request
->err_type
, http
->al
.hier
.code
);
764 safe_free(http
->uri
);
765 safe_free(http
->log_uri
);
766 safe_free(http
->al
.headers
.request
);
767 safe_free(http
->al
.headers
.reply
);
768 safe_free(http
->redirect
.location
);
769 stringClean(&http
->range_iter
.boundary
);
770 if ((e
= http
->entry
)) {
772 storeUnregister(http
->sc
, e
, http
);
774 storeUnlockObject(e
);
776 /* old_entry might still be set if we didn't yet get the reply
777 * code in clientHandleIMSReply() */
778 if ((e
= http
->old_entry
)) {
779 http
->old_entry
= NULL
;
780 storeUnregister(http
->old_sc
, e
, http
);
782 storeUnlockObject(e
);
784 requestUnlink(http
->request
);
785 assert(http
!= http
->next
);
786 assert(http
->conn
->chr
!= NULL
);
787 H
= &http
->conn
->chr
;
796 dlinkDelete(&http
->active
, &ClientActiveRequests
);
800 /* This is a handler normally called by comm_close() */
802 connStateFree(int fd
, void *data
)
804 ConnStateData
*connState
= data
;
805 clientHttpRequest
*http
;
806 debug(33, 3) ("connStateFree: FD %d\n", fd
);
807 assert(connState
!= NULL
);
808 clientdbEstablished(connState
->peer
.sin_addr
, -1); /* decrement */
809 while ((http
= connState
->chr
) != NULL
) {
810 assert(http
->conn
== connState
);
811 assert(connState
->chr
!= connState
->chr
->next
);
812 httpRequestFree(http
);
814 if (connState
->in
.size
== CLIENT_REQ_BUF_SZ
)
815 memFree(connState
->in
.buf
, MEM_CLIENT_REQ_BUF
);
817 safe_free(connState
->in
.buf
);
818 /* XXX account connState->in.buf */
819 pconnHistCount(0, connState
->nrequests
);
820 cbdataFree(connState
);
822 /* prevent those nasty RST packets */
824 char buf
[SQUID_TCP_SO_RCVBUF
];
825 while (read(fd
, buf
, SQUID_TCP_SO_RCVBUF
) > 0);
831 clientInterpretRequestHeaders(clientHttpRequest
* http
)
833 request_t
*request
= http
->request
;
834 const HttpHeader
*req_hdr
= &request
->header
;
837 request
->imslen
= -1;
838 request
->ims
= httpHeaderGetTime(req_hdr
, HDR_IF_MODIFIED_SINCE
);
839 if (request
->ims
> 0)
840 request
->flags
.ims
= 1;
841 if (httpHeaderHas(req_hdr
, HDR_PRAGMA
)) {
842 String s
= httpHeaderGetList(req_hdr
, HDR_PRAGMA
);
843 if (strListIsMember(&s
, "no-cache", ','))
847 request
->cache_control
= httpHeaderGetCc(req_hdr
);
848 if (request
->cache_control
)
849 if (EBIT_TEST(request
->cache_control
->mask
, CC_NO_CACHE
))
851 /* Work around for supporting the Reload button in IE browsers
852 * when Squid is used as an accelerator or transparent proxy,
853 * by turning accelerated IMS request to no-cache requests.
854 * Now knows about IE 5.5 fix (is actually only fixed in SP1,
855 * but we can't tell whether we are talking to SP1 or not so
856 * all 5.5 versions are treated 'normally').
858 if (Config
.onoff
.ie_refresh
) {
859 if (http
->flags
.accel
&& request
->flags
.ims
) {
860 if ( (str
= httpHeaderGetStr(req_hdr
, HDR_USER_AGENT
)) ) {
861 if (strstr(str
, "MSIE 5.01") != NULL
)
863 else if (strstr(str
, "MSIE 5.0") != NULL
)
865 else if (strstr(str
, "MSIE 4.") != NULL
)
867 else if (strstr(str
, "MSIE 3.") != NULL
)
875 if (Config
.onoff
.reload_into_ims
)
876 request
->flags
.nocache_hack
= 1;
877 else if (refresh_nocache_hack
)
878 request
->flags
.nocache_hack
= 1;
881 request
->flags
.nocache
= 1;
883 /* ignore range header in non-GETs */
884 if (request
->method
== METHOD_GET
) {
885 request
->range
= httpHeaderGetRange(req_hdr
);
887 request
->flags
.range
= 1;
889 if (httpHeaderHas(req_hdr
, HDR_AUTHORIZATION
))
890 request
->flags
.auth
= 1;
891 if (request
->login
[0] != '\0')
892 request
->flags
.auth
= 1;
893 if (httpHeaderHas(req_hdr
, HDR_VIA
)) {
894 String s
= httpHeaderGetList(req_hdr
, HDR_VIA
);
896 * ThisCache cannot be a member of Via header, "1.0 ThisCache" can.
897 * Note ThisCache2 has a space prepended to the hostname so we don't
898 * accidentally match super-domains.
900 if (strListIsSubstr(&s
, ThisCache2
, ',')) {
901 debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
902 request
, (ObjPackMethod
) & httpRequestPack
);
903 request
->flags
.loopdetect
= 1;
906 fvdbCountVia(strBuf(s
));
910 #if USE_USERAGENT_LOG
911 if ((str
= httpHeaderGetStr(req_hdr
, HDR_USER_AGENT
)))
912 logUserAgent(fqdnFromAddr(http
->conn
->peer
.sin_addr
), str
);
915 if ((str
= httpHeaderGetStr(req_hdr
, HDR_REFERER
)))
916 logReferer(fqdnFromAddr(http
->conn
->peer
.sin_addr
), str
,
920 if (httpHeaderHas(req_hdr
, HDR_X_FORWARDED_FOR
)) {
921 String s
= httpHeaderGetList(req_hdr
, HDR_X_FORWARDED_FOR
);
922 fvdbCountForw(strBuf(s
));
926 if (request
->method
== METHOD_TRACE
) {
927 request
->max_forwards
= httpHeaderGetInt(req_hdr
, HDR_MAX_FORWARDS
);
929 if (clientCachable(http
))
930 request
->flags
.cachable
= 1;
931 if (clientHierarchical(http
))
932 request
->flags
.hierarchical
= 1;
933 debug(33, 5) ("clientInterpretRequestHeaders: REQ_NOCACHE = %s\n",
934 request
->flags
.nocache
? "SET" : "NOT SET");
935 debug(33, 5) ("clientInterpretRequestHeaders: REQ_CACHABLE = %s\n",
936 request
->flags
.cachable
? "SET" : "NOT SET");
937 debug(33, 5) ("clientInterpretRequestHeaders: REQ_HIERARCHICAL = %s\n",
938 request
->flags
.hierarchical
? "SET" : "NOT SET");
942 * clientSetKeepaliveFlag() sets request->flags.proxy_keepalive.
943 * This is the client-side persistent connection flag. We need
944 * to set this relatively early in the request processing
945 * to handle hacks for broken servers and clients.
948 clientSetKeepaliveFlag(clientHttpRequest
* http
)
950 request_t
*request
= http
->request
;
951 const HttpHeader
*req_hdr
= &request
->header
;
952 debug(33, 3) ("clientSetKeepaliveFlag: http_ver = %d.%d\n",
953 request
->http_ver
.major
, request
->http_ver
.minor
);
954 debug(33, 3) ("clientSetKeepaliveFlag: method = %s\n",
955 RequestMethodStr
[request
->method
]);
956 if (!Config
.onoff
.client_pconns
)
957 request
->flags
.proxy_keepalive
= 0;
958 else if (httpMsgIsPersistent(request
->http_ver
, req_hdr
))
959 request
->flags
.proxy_keepalive
= 1;
963 clientCheckContentLength(request_t
* r
)
968 /* PUT/POST requires a request entity */
969 return (r
->content_length
>= 0);
972 /* We do not want to see a request entity on GET/HEAD requests */
973 return (r
->content_length
<= 0);
975 /* For other types of requests we don't care */
982 clientCachable(clientHttpRequest
* http
)
984 const char *url
= http
->uri
;
985 request_t
*req
= http
->request
;
986 method_t method
= req
->method
;
987 if (req
->protocol
== PROTO_HTTP
)
988 return httpCachable(method
);
989 /* FTP is always cachable */
990 if (req
->protocol
== PROTO_GOPHER
)
991 return gopherCachable(url
);
992 if (req
->protocol
== PROTO_WAIS
)
994 if (method
== METHOD_CONNECT
)
996 if (method
== METHOD_TRACE
)
998 if (req
->protocol
== PROTO_CACHEOBJ
)
1003 /* Return true if we can query our neighbors for this object */
1005 clientHierarchical(clientHttpRequest
* http
)
1007 const char *url
= http
->uri
;
1008 request_t
*request
= http
->request
;
1009 method_t method
= request
->method
;
1010 const wordlist
*p
= NULL
;
1012 /* IMS needs a private key, so we can use the hierarchy for IMS only
1013 * if our neighbors support private keys */
1014 if (request
->flags
.ims
&& !neighbors_do_private_keys
)
1016 if (request
->flags
.auth
)
1018 if (method
== METHOD_TRACE
)
1020 if (method
!= METHOD_GET
)
1022 /* scan hierarchy_stoplist */
1023 for (p
= Config
.hierarchy_stoplist
; p
; p
= p
->next
)
1024 if (strstr(url
, p
->key
))
1026 if (request
->flags
.loopdetect
)
1028 if (request
->protocol
== PROTO_HTTP
)
1029 return httpCachable(method
);
1030 if (request
->protocol
== PROTO_GOPHER
)
1031 return gopherCachable(url
);
1032 if (request
->protocol
== PROTO_WAIS
)
1034 if (request
->protocol
== PROTO_CACHEOBJ
)
1040 isTcpHit(log_type code
)
1042 /* this should be a bitmap for better optimization */
1043 if (code
== LOG_TCP_HIT
)
1045 if (code
== LOG_TCP_IMS_HIT
)
1047 if (code
== LOG_TCP_REFRESH_FAIL_HIT
)
1049 if (code
== LOG_TCP_REFRESH_HIT
)
1051 if (code
== LOG_TCP_NEGATIVE_HIT
)
1053 if (code
== LOG_TCP_MEM_HIT
)
1055 if (code
== LOG_TCP_OFFLINE_HIT
)
1061 * returns true if If-Range specs match reply, false otherwise
1064 clientIfRangeMatch(clientHttpRequest
* http
, HttpReply
* rep
)
1066 const TimeOrTag spec
= httpHeaderGetTimeOrTag(&http
->request
->header
, HDR_IF_RANGE
);
1067 /* check for parsing falure */
1072 ETag rep_tag
= httpHeaderGetETag(&rep
->header
, HDR_ETAG
);
1073 debug(33, 3) ("clientIfRangeMatch: ETags: %s and %s\n",
1074 spec
.tag
.str
, rep_tag
.str
? rep_tag
.str
: "<none>");
1076 return 0; /* entity has no etag to compare with! */
1077 if (spec
.tag
.weak
|| rep_tag
.weak
) {
1078 debug(33, 1) ("clientIfRangeMatch: Weak ETags are not allowed in If-Range: %s ? %s\n",
1079 spec
.tag
.str
, rep_tag
.str
);
1080 return 0; /* must use strong validator for sub-range requests */
1082 return etagIsEqual(&rep_tag
, &spec
.tag
);
1084 /* got modification time? */
1085 if (spec
.time
>= 0) {
1086 return http
->entry
->lastmod
<= spec
.time
;
1088 assert(0); /* should not happen */
1092 /* returns expected content length for multi-range replies
1093 * note: assumes that httpHdrRangeCanonize has already been called
1094 * warning: assumes that HTTP headers for individual ranges at the
1095 * time of the actuall assembly will be exactly the same as
1096 * the headers when clientMRangeCLen() is called */
1098 clientMRangeCLen(clientHttpRequest
* http
)
1101 HttpHdrRangePos pos
= HttpHdrRangeInitPos
;
1102 const HttpHdrRangeSpec
*spec
;
1105 assert(http
->entry
->mem_obj
);
1108 while ((spec
= httpHdrRangeGetSpec(http
->request
->range
, &pos
))) {
1110 /* account for headers for this range */
1112 clientPackRangeHdr(http
->entry
->mem_obj
->reply
,
1113 spec
, http
->range_iter
.boundary
, &mb
);
1116 /* account for range content */
1117 clen
+= spec
->length
;
1119 debug(33, 6) ("clientMRangeCLen: (clen += %d + %d) == %d\n",
1120 mb
.size
, spec
->length
, clen
);
1122 /* account for the terminating boundary */
1124 clientPackTermBound(http
->range_iter
.boundary
, &mb
);
1131 /* adds appropriate Range headers if needed */
1133 clientBuildRangeHeader(clientHttpRequest
* http
, HttpReply
* rep
)
1135 HttpHeader
*hdr
= rep
? &rep
->header
: 0;
1136 const char *range_err
= NULL
;
1137 request_t
*request
= http
->request
;
1138 assert(request
->range
);
1139 /* check if we still want to do ranges */
1141 range_err
= "no [parse-able] reply";
1142 else if (rep
->sline
.status
!= HTTP_OK
)
1143 range_err
= "wrong status code";
1144 else if (httpHeaderHas(hdr
, HDR_CONTENT_RANGE
))
1145 range_err
= "origin server does ranges";
1146 else if (rep
->content_length
< 0)
1147 range_err
= "unknown length";
1148 else if (rep
->content_length
!= http
->entry
->mem_obj
->reply
->content_length
)
1149 range_err
= "INCONSISTENT length"; /* a bug? */
1150 else if (httpHeaderHas(&http
->request
->header
, HDR_IF_RANGE
) && !clientIfRangeMatch(http
, rep
))
1151 range_err
= "If-Range match failed";
1152 else if (!httpHdrRangeCanonize(http
->request
->range
, rep
->content_length
))
1153 range_err
= "canonization failed";
1154 else if (httpHdrRangeIsComplex(http
->request
->range
))
1155 range_err
= "too complex range header";
1156 else if (!request
->flags
.cachable
) /* from we_do_ranges in http.c */
1157 range_err
= "non-cachable request";
1158 /* get rid of our range specs on error */
1160 debug(33, 3) ("clientBuildRangeHeader: will not do ranges: %s.\n", range_err
);
1161 httpHdrRangeDestroy(http
->request
->range
);
1162 http
->request
->range
= NULL
;
1164 const int spec_count
= http
->request
->range
->specs
.count
;
1165 int actual_clen
= -1;
1167 debug(33, 3) ("clientBuildRangeHeader: range spec count: %d virgin clen: %d\n",
1168 spec_count
, rep
->content_length
);
1169 assert(spec_count
> 0);
1170 /* ETags should not be returned with Partial Content replies? */
1171 httpHeaderDelById(hdr
, HDR_ETAG
);
1172 /* append appropriate header(s) */
1173 if (spec_count
== 1) {
1174 HttpHdrRangePos pos
= HttpHdrRangeInitPos
;
1175 const HttpHdrRangeSpec
*spec
= httpHdrRangeGetSpec(http
->request
->range
, &pos
);
1177 /* append Content-Range */
1178 httpHeaderAddContRange(hdr
, *spec
, rep
->content_length
);
1179 /* set new Content-Length to the actual number of bytes
1180 * transmitted in the message-body */
1181 actual_clen
= spec
->length
;
1184 /* generate boundary string */
1185 http
->range_iter
.boundary
= httpHdrRangeBoundaryStr(http
);
1186 /* delete old Content-Type, add ours */
1187 httpHeaderDelById(hdr
, HDR_CONTENT_TYPE
);
1188 httpHeaderPutStrf(hdr
, HDR_CONTENT_TYPE
,
1189 "multipart/byteranges; boundary=\"%s\"",
1190 strBuf(http
->range_iter
.boundary
));
1191 /* Content-Length is not required in multipart responses
1192 * but it is always nice to have one */
1193 actual_clen
= clientMRangeCLen(http
);
1196 /* replace Content-Length header */
1197 assert(actual_clen
>= 0);
1198 httpHeaderDelById(hdr
, HDR_CONTENT_LENGTH
);
1199 httpHeaderPutInt(hdr
, HDR_CONTENT_LENGTH
, actual_clen
);
1200 debug(33, 3) ("clientBuildRangeHeader: actual content length: %d\n", actual_clen
);
1205 * filters out unwanted entries from original reply header
1206 * adds extra entries if we have more info than origin server
1207 * adds Squid specific entries
1210 clientBuildReplyHeader(clientHttpRequest
* http
, HttpReply
* rep
)
1212 HttpHeader
*hdr
= &rep
->header
;
1213 int is_hit
= isTcpHit(http
->log_type
);
1214 request_t
*request
= http
->request
;
1215 #if DONT_FILTER_THESE
1216 /* but you might want to if you run Squid as an HTTP accelerator */
1217 /* httpHeaderDelById(hdr, HDR_ACCEPT_RANGES); */
1218 httpHeaderDelById(hdr
, HDR_ETAG
);
1220 httpHeaderDelById(hdr
, HDR_PROXY_CONNECTION
);
1221 /* here: Keep-Alive is a field-name, not a connection directive! */
1222 httpHeaderDelByName(hdr
, "Keep-Alive");
1223 /* remove Set-Cookie if a hit */
1225 httpHeaderDelById(hdr
, HDR_SET_COOKIE
);
1226 /* handle Connection header */
1227 if (httpHeaderHas(hdr
, HDR_CONNECTION
)) {
1228 /* anything that matches Connection list member will be deleted */
1229 String strConnection
= httpHeaderGetList(hdr
, HDR_CONNECTION
);
1230 const HttpHeaderEntry
*e
;
1231 HttpHeaderPos pos
= HttpHeaderInitPos
;
1233 * think: on-average-best nesting of the two loops (hdrEntry
1234 * and strListItem) @?@
1237 * maybe we should delete standard stuff ("keep-alive","close")
1238 * from strConnection first?
1240 while ((e
= httpHeaderGetEntry(hdr
, &pos
))) {
1241 if (strListIsMember(&strConnection
, strBuf(e
->name
), ','))
1242 httpHeaderDelAt(hdr
, pos
);
1244 httpHeaderDelById(hdr
, HDR_CONNECTION
);
1245 stringClean(&strConnection
);
1249 clientBuildRangeHeader(http
, rep
);
1251 * Add a estimated Age header on cache hits.
1255 * Remove any existing Age header sent by upstream caches
1256 * (note that the existing header is passed along unmodified
1259 httpHeaderDelById(hdr
, HDR_AGE
);
1261 * This adds the calculated object age. Note that the details of the
1262 * age calculation is performed by adjusting the timestamp in
1263 * storeTimestampsSet(), not here.
1265 * BROWSER WORKAROUND: IE sometimes hangs when receiving a 0 Age
1266 * header, so don't use it unless there is a age to report. Please
1267 * note that Age is only used to make a conservative estimation of
1268 * the objects age, so a Age: 0 header does not add any useful
1269 * information to the reply in any case.
1271 if (http
->entry
->timestamp
> -1)
1272 if (http
->entry
->timestamp
< squid_curtime
)
1273 httpHeaderPutInt(hdr
, HDR_AGE
,
1274 squid_curtime
- http
->entry
->timestamp
);
1276 /* Append X-Cache */
1277 httpHeaderPutStrf(hdr
, HDR_X_CACHE
, "%s from %s",
1278 is_hit
? "HIT" : "MISS", getMyHostname());
1279 #if USE_CACHE_DIGESTS
1280 /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1281 httpHeaderPutStrf(hdr
, HDR_X_CACHE_LOOKUP
, "%s from %s:%d",
1282 http
->lookup_type
? http
->lookup_type
: "NONE",
1283 getMyHostname(), ntohs(Config
.Sockaddr
.http
->s
.sin_port
));
1285 if (httpReplyBodySize(request
->method
, rep
) < 0) {
1286 debug(33, 3) ("clientBuildReplyHeader: can't keep-alive, unknown body size\n");
1287 request
->flags
.proxy_keepalive
= 0;
1289 /* Signal keep-alive if needed */
1290 httpHeaderPutStr(hdr
,
1291 http
->flags
.accel
? HDR_CONNECTION
: HDR_PROXY_CONNECTION
,
1292 request
->flags
.proxy_keepalive
? "keep-alive" : "close");
1293 #if ADD_X_REQUEST_URI
1295 * Knowing the URI of the request is useful when debugging persistent
1296 * connections in a client; we cannot guarantee the order of http headers,
1297 * but X-Request-URI is likely to be the very last header to ease use from a
1298 * debugger [hdr->entries.count-1].
1300 httpHeaderPutStr(hdr
, HDR_X_REQUEST_URI
,
1301 http
->entry
->mem_obj
->url
? http
->entry
->mem_obj
->url
: http
->uri
);
1306 clientBuildReply(clientHttpRequest
* http
, const char *buf
, size_t size
)
1308 HttpReply
*rep
= httpReplyCreate();
1309 size_t k
= headersEnd(buf
, size
);
1310 if (k
&& httpReplyParse(rep
, buf
, k
)) {
1311 /* enforce 1.0 reply version */
1312 httpBuildVersion(&rep
->sline
.version
, 1, 0);
1313 /* do header conversions */
1314 clientBuildReplyHeader(http
, rep
);
1315 /* if we do ranges, change status to "Partial Content" */
1316 if (http
->request
->range
)
1317 httpStatusLineSet(&rep
->sline
, rep
->sline
.version
,
1318 HTTP_PARTIAL_CONTENT
, NULL
);
1320 /* parsing failure, get rid of the invalid reply */
1321 httpReplyDestroy(rep
);
1323 /* if we were going to do ranges, backoff */
1324 if (http
->request
->range
) {
1325 /* this will fail and destroy request->range */
1326 clientBuildRangeHeader(http
, rep
);
1333 * clientCacheHit should only be called until the HTTP reply headers
1334 * have been parsed. Normally this should be a single call, but
1335 * it might take more than one. As soon as we have the headers,
1336 * we hand off to clientSendMoreData, clientProcessExpired, or
1337 * clientProcessMiss.
1340 clientCacheHit(void *data
, char *buf
, ssize_t size
)
1342 clientHttpRequest
*http
= data
;
1343 StoreEntry
*e
= http
->entry
;
1345 request_t
*r
= http
->request
;
1346 debug(33, 3) ("clientCacheHit: %s, %d bytes\n", http
->uri
, (int) size
);
1347 if (http
->entry
== NULL
) {
1348 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1349 debug(33, 3) ("clientCacheHit: request aborted\n");
1351 } else if (size
< 0) {
1352 /* swap in failure */
1353 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1354 debug(33, 3) ("clientCacheHit: swapin failure for %s\n", http
->uri
);
1355 http
->log_type
= LOG_TCP_SWAPFAIL_MISS
;
1356 if ((e
= http
->entry
)) {
1358 storeUnregister(http
->sc
, e
, http
);
1360 storeUnlockObject(e
);
1362 clientProcessMiss(http
);
1367 assert(!EBIT_TEST(e
->flags
, ENTRY_ABORTED
));
1368 if (mem
->reply
->sline
.status
== 0) {
1370 * we don't have full reply headers yet; either wait for more or
1371 * punt to clientProcessMiss.
1373 if (e
->mem_status
== IN_MEMORY
|| e
->store_status
== STORE_OK
) {
1374 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1375 clientProcessMiss(http
);
1376 } else if (size
== CLIENT_SOCK_SZ
&& http
->out
.offset
== 0) {
1377 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1378 clientProcessMiss(http
);
1380 debug(33, 3) ("clientCacheHit: waiting for HTTP reply headers\n");
1381 storeClientCopy(http
->sc
, e
,
1382 http
->out
.offset
+ size
,
1392 * Got the headers, now grok them
1394 assert(http
->log_type
== LOG_TCP_HIT
);
1395 if (checkNegativeHit(e
)) {
1396 http
->log_type
= LOG_TCP_NEGATIVE_HIT
;
1397 clientSendMoreData(data
, buf
, size
);
1398 } else if (r
->method
== METHOD_HEAD
) {
1400 * RFC 2068 seems to indicate there is no "conditional HEAD"
1401 * request. We cannot validate a cached object for a HEAD
1402 * request, nor can we return 304.
1404 if (e
->mem_status
== IN_MEMORY
)
1405 http
->log_type
= LOG_TCP_MEM_HIT
;
1406 clientSendMoreData(data
, buf
, size
);
1407 } else if (refreshCheckHTTP(e
, r
) && !http
->flags
.internal
) {
1408 debug(33, 5) ("clientCacheHit: in refreshCheck() block\n");
1410 * We hold a stale copy; it needs to be validated
1413 * The 'need_validation' flag is used to prevent forwarding
1414 * loops between siblings. If our copy of the object is stale,
1415 * then we should probably only use parents for the validation
1416 * request. Otherwise two siblings could generate a loop if
1417 * both have a stale version of the object.
1419 r
->flags
.need_validation
= 1;
1420 if (e
->lastmod
< 0) {
1422 * Previous reply didn't have a Last-Modified header,
1423 * we cannot revalidate it.
1425 http
->log_type
= LOG_TCP_MISS
;
1426 clientProcessMiss(http
);
1427 } else if (r
->flags
.nocache
) {
1429 * This did not match a refresh pattern that overrides no-cache
1430 * we should honour the client no-cache header.
1432 http
->log_type
= LOG_TCP_CLIENT_REFRESH_MISS
;
1433 clientProcessMiss(http
);
1434 } else if (r
->protocol
== PROTO_HTTP
) {
1436 * Object needs to be revalidated
1437 * XXX This could apply to FTP as well, if Last-Modified is known.
1439 http
->log_type
= LOG_TCP_REFRESH_MISS
;
1440 clientProcessExpired(http
);
1443 * We don't know how to re-validate other protocols. Handle
1444 * them as if the object has expired.
1446 http
->log_type
= LOG_TCP_MISS
;
1447 clientProcessMiss(http
);
1449 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1450 } else if (r
->flags
.ims
) {
1452 * Handle If-Modified-Since requests from the client
1454 if (mem
->reply
->sline
.status
!= HTTP_OK
) {
1455 debug(33, 4) ("clientCacheHit: Reply code %d != 200\n",
1456 mem
->reply
->sline
.status
);
1457 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1458 http
->log_type
= LOG_TCP_MISS
;
1459 clientProcessMiss(http
);
1460 } else if (modifiedSince(e
, http
->request
)) {
1461 http
->log_type
= LOG_TCP_IMS_HIT
;
1462 clientSendMoreData(data
, buf
, size
);
1464 time_t timestamp
= e
->timestamp
;
1465 MemBuf mb
= httpPacked304Reply(e
->mem_obj
->reply
);
1466 http
->log_type
= LOG_TCP_IMS_HIT
;
1467 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1468 storeUnregister(http
->sc
, e
, http
);
1470 storeUnlockObject(e
);
1471 e
= clientCreateStoreEntry(http
, http
->request
->method
, null_request_flags
);
1473 * Copy timestamp from the original entry so the 304
1474 * reply has a meaningful Age: header.
1476 e
->timestamp
= timestamp
;
1478 httpReplyParse(e
->mem_obj
->reply
, mb
.buf
, mb
.size
);
1479 storeAppend(e
, mb
.buf
, mb
.size
);
1485 * plain ol' cache hit
1487 if (e
->mem_status
== IN_MEMORY
)
1488 http
->log_type
= LOG_TCP_MEM_HIT
;
1489 else if (Config
.onoff
.offline
)
1490 http
->log_type
= LOG_TCP_OFFLINE_HIT
;
1491 clientSendMoreData(data
, buf
, size
);
1495 /* put terminating boundary for multiparts */
1497 clientPackTermBound(String boundary
, MemBuf
* mb
)
1499 memBufPrintf(mb
, "\r\n--%s--\r\n", strBuf(boundary
));
1500 debug(33, 6) ("clientPackTermBound: buf offset: %d\n", mb
->size
);
1503 /* appends a "part" HTTP header (as in a multi-part/range reply) to the buffer */
1505 clientPackRangeHdr(const HttpReply
* rep
, const HttpHdrRangeSpec
* spec
, String boundary
, MemBuf
* mb
)
1513 debug(33, 5) ("clientPackRangeHdr: appending boundary: %s\n", strBuf(boundary
));
1514 /* rfc2046 requires to _prepend_ boundary with <crlf>! */
1515 memBufPrintf(mb
, "\r\n--%s\r\n", strBuf(boundary
));
1517 /* stuff the header with required entries and pack it */
1518 httpHeaderInit(&hdr
, hoReply
);
1519 if (httpHeaderHas(&rep
->header
, HDR_CONTENT_TYPE
))
1520 httpHeaderPutStr(&hdr
, HDR_CONTENT_TYPE
, httpHeaderGetStr(&rep
->header
, HDR_CONTENT_TYPE
));
1521 httpHeaderAddContRange(&hdr
, *spec
, rep
->content_length
);
1522 packerToMemInit(&p
, mb
);
1523 httpHeaderPackInto(&hdr
, &p
);
1525 httpHeaderClean(&hdr
);
1527 /* append <crlf> (we packed a header, not a reply) */
1528 memBufPrintf(mb
, crlf
);
1532 * extracts a "range" from *buf and appends them to mb, updating
1533 * all offsets and such.
1536 clientPackRange(clientHttpRequest
* http
,
1537 HttpHdrRangeIter
* i
,
1542 const ssize_t copy_sz
= i
->debt_size
<= *size
? i
->debt_size
: *size
;
1543 off_t body_off
= http
->out
.offset
- i
->prefix_size
;
1547 * intersection of "have" and "need" ranges must not be empty
1549 assert(body_off
< i
->spec
->offset
+ i
->spec
->length
);
1550 assert(body_off
+ *size
> i
->spec
->offset
);
1552 * put boundary and headers at the beginning of a range in a
1555 if (http
->request
->range
->specs
.count
> 1 && i
->debt_size
== i
->spec
->length
) {
1556 assert(http
->entry
->mem_obj
);
1558 http
->entry
->mem_obj
->reply
, /* original reply */
1559 i
->spec
, /* current range */
1560 i
->boundary
, /* boundary, the same for all */
1567 debug(33, 3) ("clientPackRange: appending %d bytes\n", copy_sz
);
1568 memBufAppend(mb
, *buf
, copy_sz
);
1573 i
->debt_size
-= copy_sz
;
1574 body_off
+= copy_sz
;
1576 http
->out
.offset
= body_off
+ i
->prefix_size
; /* sync */
1580 assert(*size
>= 0 && i
->debt_size
>= 0);
1583 /* returns true if there is still data available to pack more ranges
1584 * increments iterator "i"
1585 * used by clientPackMoreRanges */
1587 clientCanPackMoreRanges(const clientHttpRequest
* http
, HttpHdrRangeIter
* i
, ssize_t size
)
1589 /* first update "i" if needed */
1590 if (!i
->debt_size
) {
1591 if ((i
->spec
= httpHdrRangeGetSpec(http
->request
->range
, &i
->pos
)))
1592 i
->debt_size
= i
->spec
->length
;
1594 assert(!i
->debt_size
== !i
->spec
); /* paranoid sync condition */
1595 /* continue condition: need_more_data && have_more_data */
1596 return i
->spec
&& size
> 0;
1599 /* extracts "ranges" from buf and appends them to mb, updating all offsets and such */
1600 /* returns true if we need more data */
1602 clientPackMoreRanges(clientHttpRequest
* http
, const char *buf
, ssize_t size
, MemBuf
* mb
)
1604 HttpHdrRangeIter
*i
= &http
->range_iter
;
1605 /* offset in range specs does not count the prefix of an http msg */
1606 off_t body_off
= http
->out
.offset
- i
->prefix_size
;
1608 /* check: reply was parsed and range iterator was initialized */
1609 assert(i
->prefix_size
> 0);
1610 /* filter out data according to range specs */
1611 while (clientCanPackMoreRanges(http
, i
, size
)) {
1612 off_t start
; /* offset of still missing data */
1614 start
= i
->spec
->offset
+ i
->spec
->length
- i
->debt_size
;
1615 debug(33, 3) ("clientPackMoreRanges: in: offset: %d size: %d\n",
1616 (int) body_off
, size
);
1617 debug(33, 3) ("clientPackMoreRanges: out: start: %d spec[%d]: [%d, %d), len: %d debt: %d\n",
1618 (int) start
, (int) i
->pos
, i
->spec
->offset
, (int) (i
->spec
->offset
+ i
->spec
->length
), i
->spec
->length
, i
->debt_size
);
1619 assert(body_off
<= start
); /* we did not miss it */
1620 /* skip up to start */
1621 if (body_off
+ size
> start
) {
1622 const size_t skip_size
= start
- body_off
;
1627 /* has not reached start yet */
1632 /* put next chunk if any */
1634 http
->out
.offset
= body_off
+ i
->prefix_size
; /* sync */
1635 clientPackRange(http
, i
, &buf
, &size
, mb
);
1636 body_off
= http
->out
.offset
- i
->prefix_size
; /* sync */
1639 assert(!i
->debt_size
== !i
->spec
); /* paranoid sync condition */
1640 debug(33, 3) ("clientPackMoreRanges: buf exhausted: in: offset: %d size: %d need_more: %d\n",
1641 (int) body_off
, size
, i
->debt_size
);
1643 debug(33, 3) ("clientPackMoreRanges: need more: spec[%d]: [%d, %d), len: %d\n",
1644 (int) i
->pos
, i
->spec
->offset
, (int) (i
->spec
->offset
+ i
->spec
->length
), i
->spec
->length
);
1645 /* skip the data we do not need if possible */
1646 if (i
->debt_size
== i
->spec
->length
) /* at the start of the cur. spec */
1647 body_off
= i
->spec
->offset
;
1649 assert(body_off
== i
->spec
->offset
+ i
->spec
->length
- i
->debt_size
);
1650 } else if (http
->request
->range
->specs
.count
> 1) {
1651 /* put terminating boundary for multiparts */
1652 clientPackTermBound(i
->boundary
, mb
);
1654 http
->out
.offset
= body_off
+ i
->prefix_size
; /* sync */
1655 return i
->debt_size
> 0;
1659 clientReplyBodyTooLarge(int clen
)
1661 if (0 == Config
.maxReplyBodySize
)
1662 return 0; /* disabled */
1664 return 0; /* unknown */
1665 if (clen
> Config
.maxReplyBodySize
)
1666 return 1; /* too large */
1671 clientRequestBodyTooLarge(int clen
)
1673 if (0 == Config
.maxRequestBodySize
)
1674 return 0; /* disabled */
1676 return 0; /* unknown, bug? */
1677 if (clen
> Config
.maxRequestBodySize
)
1678 return 1; /* too large */
1683 * accepts chunk of a http message in buf, parses prefix, filters headers and
1684 * such, writes processed message to the client's socket
1687 clientSendMoreData(void *data
, char *buf
, ssize_t size
)
1689 clientHttpRequest
*http
= data
;
1690 StoreEntry
*entry
= http
->entry
;
1691 ConnStateData
*conn
= http
->conn
;
1693 HttpReply
*rep
= NULL
;
1694 const char *body_buf
= buf
;
1695 ssize_t body_size
= size
;
1697 ssize_t check_size
= 0;
1698 debug(33, 5) ("clientSendMoreData: %s, %d bytes\n", http
->uri
, (int) size
);
1699 assert(size
<= CLIENT_SOCK_SZ
);
1700 assert(http
->request
!= NULL
);
1701 dlinkDelete(&http
->active
, &ClientActiveRequests
);
1702 dlinkAdd(http
, &http
->active
, &ClientActiveRequests
);
1703 debug(33, 5) ("clientSendMoreData: FD %d '%s', out.offset=%d \n",
1704 fd
, storeUrl(entry
), (int) http
->out
.offset
);
1705 if (conn
->chr
!= http
) {
1706 /* there is another object in progress, defer this one */
1707 debug(33, 1) ("clientSendMoreData: Deferring %s\n", storeUrl(entry
));
1708 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1710 } else if (entry
&& EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
1711 /* call clientWriteComplete so the client socket gets closed */
1712 clientWriteComplete(fd
, NULL
, 0, COMM_OK
, http
);
1713 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1715 } else if (size
< 0) {
1716 /* call clientWriteComplete so the client socket gets closed */
1717 clientWriteComplete(fd
, NULL
, 0, COMM_OK
, http
);
1718 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1720 } else if (size
== 0) {
1721 /* call clientWriteComplete so the client socket gets closed */
1722 clientWriteComplete(fd
, NULL
, 0, COMM_OK
, http
);
1723 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1726 if (http
->out
.offset
== 0) {
1727 if (Config
.onoff
.log_mime_hdrs
) {
1729 if ((k
= headersEnd(buf
, size
))) {
1730 safe_free(http
->al
.headers
.reply
);
1731 http
->al
.headers
.reply
= xcalloc(k
+ 1, 1);
1732 xstrncpy(http
->al
.headers
.reply
, buf
, k
);
1735 rep
= clientBuildReply(http
, buf
, size
);
1736 if (rep
&& clientReplyBodyTooLarge(rep
->content_length
)) {
1737 ErrorState
*err
= errorCon(ERR_TOO_BIG
, HTTP_FORBIDDEN
);
1738 err
->request
= requestLink(http
->request
);
1739 storeUnregister(http
->sc
, http
->entry
, http
);
1741 storeUnlockObject(http
->entry
);
1742 http
->entry
= clientCreateStoreEntry(http
, http
->request
->method
,
1743 null_request_flags
);
1744 errorAppendEntry(http
->entry
, err
);
1745 httpReplyDestroy(rep
);
1748 body_size
= size
- rep
->hdr_sz
;
1749 assert(body_size
>= 0);
1750 body_buf
= buf
+ rep
->hdr_sz
;
1751 http
->range_iter
.prefix_size
= rep
->hdr_sz
;
1752 debug(33, 3) ("clientSendMoreData: Appending %d bytes after %d bytes of headers\n",
1753 body_size
, rep
->hdr_sz
);
1754 } else if (size
< CLIENT_SOCK_SZ
&& entry
->store_status
== STORE_PENDING
) {
1755 /* wait for more to arrive */
1756 storeClientCopy(http
->sc
, entry
,
1757 http
->out
.offset
+ size
,
1765 /* reset range iterator */
1766 http
->range_iter
.pos
= HttpHdrRangeInitPos
;
1767 } else if (!http
->request
->range
) {
1768 /* Avoid copying to MemBuf for non-range requests */
1769 /* Note, if we're here, then 'rep' is known to be NULL */
1770 http
->out
.offset
+= body_size
;
1771 comm_write(fd
, buf
, size
, clientWriteBodyComplete
, http
, NULL
);
1772 /* NULL because clientWriteBodyComplete frees it */
1775 if (http
->request
->method
== METHOD_HEAD
) {
1777 /* do not forward body for HEAD replies */
1779 http
->flags
.done_copying
= 1;
1782 * If we are here, then store_status == STORE_OK and it
1783 * seems we have a HEAD repsponse which is missing the
1784 * empty end-of-headers line (home.mira.net, phttpd/0.99.72
1785 * does this). Because clientBuildReply() fails we just
1786 * call this reply a body, set the done_copying flag and
1789 http
->flags
.done_copying
= 1;
1792 /* write headers and/or body if any */
1793 assert(rep
|| (body_buf
&& body_size
));
1794 /* init mb; put status line and headers if any */
1796 mb
= httpReplyPack(rep
);
1797 http
->out
.offset
+= rep
->hdr_sz
;
1798 check_size
+= rep
->hdr_sz
;
1800 headersLog(0, 0, http
->request
->method
, rep
);
1802 httpReplyDestroy(rep
);
1807 /* append body if any */
1808 if (http
->request
->range
) {
1809 /* Only GET requests should have ranges */
1810 assert(http
->request
->method
== METHOD_GET
);
1811 /* clientPackMoreRanges() updates http->out.offset */
1812 /* force the end of the transfer if we are done */
1813 if (!clientPackMoreRanges(http
, body_buf
, body_size
, &mb
))
1814 http
->flags
.done_copying
= 1;
1815 } else if (body_buf
&& body_size
) {
1816 http
->out
.offset
+= body_size
;
1817 check_size
+= body_size
;
1818 memBufAppend(&mb
, body_buf
, body_size
);
1820 if (!http
->request
->range
&& http
->request
->method
== METHOD_GET
)
1821 assert(check_size
== size
);
1823 comm_write_mbuf(fd
, mb
, clientWriteComplete
, http
);
1824 /* if we don't do it, who will? */
1825 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1829 * clientWriteBodyComplete is called for MEM_CLIENT_SOCK_BUF's
1830 * written directly to the client socket, versus copying to a MemBuf
1831 * and going through comm_write_mbuf. Most non-range responses after
1832 * the headers probably go through here.
1835 clientWriteBodyComplete(int fd
, char *buf
, size_t size
, int errflag
, void *data
)
1838 * NOTE: clientWriteComplete doesn't currently use its "buf"
1839 * (second) argument, so we pass in NULL.
1841 clientWriteComplete(fd
, NULL
, size
, errflag
, data
);
1842 memFree(buf
, MEM_CLIENT_SOCK_BUF
);
1846 clientKeepaliveNextRequest(clientHttpRequest
* http
)
1848 ConnStateData
*conn
= http
->conn
;
1850 debug(33, 3) ("clientKeepaliveNextRequest: FD %d\n", conn
->fd
);
1851 conn
->defer
.until
= 0; /* Kick it to read a new request */
1852 httpRequestFree(http
);
1853 if ((http
= conn
->chr
) == NULL
) {
1854 debug(33, 5) ("clientKeepaliveNextRequest: FD %d reading next req\n",
1856 fd_note(conn
->fd
, "Reading next request");
1858 * Set the timeout BEFORE calling clientReadRequest().
1860 commSetTimeout(conn
->fd
, Config
.Timeout
.pconn
, requestTimeout
, conn
);
1862 * CYGWIN has a problem and is blocking on read() requests when there
1863 * is no data present.
1864 * This hack may hit performance a little, but it's better than
1867 #ifdef _SQUID_CYGWIN_
1868 commSetSelect(conn
->fd
, COMM_SELECT_READ
, clientReadRequest
, conn
, 0);
1870 clientReadRequest(conn
->fd
, conn
); /* Read next request */
1873 * Note, the FD may be closed at this point.
1875 } else if ((entry
= http
->entry
) == NULL
) {
1877 * this request is in progress, maybe doing an ACL or a redirect,
1878 * execution will resume after the operation completes.
1881 debug(33, 1) ("clientKeepaliveNextRequest: FD %d Sending next\n",
1884 if (0 == storeClientCopyPending(http
->sc
, entry
, http
)) {
1885 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
))
1886 debug(33, 0) ("clientKeepaliveNextRequest: ENTRY_ABORTED\n");
1887 storeClientCopy(http
->sc
, entry
,
1891 memAllocate(MEM_CLIENT_SOCK_BUF
),
1899 clientWriteComplete(int fd
, char *bufnotused
, size_t size
, int errflag
, void *data
)
1901 clientHttpRequest
*http
= data
;
1902 StoreEntry
*entry
= http
->entry
;
1904 http
->out
.size
+= size
;
1905 debug(33, 5) ("clientWriteComplete: FD %d, sz %d, err %d, off %d, len %d\n",
1906 fd
, size
, errflag
, (int) http
->out
.offset
, entry
? objectLen(entry
) : 0);
1908 kb_incr(&statCounter
.client_http
.kbytes_out
, size
);
1909 if (isTcpHit(http
->log_type
))
1910 kb_incr(&statCounter
.client_http
.hit_kbytes_out
, size
);
1914 * just close the socket, httpRequestFree will abort if needed
1917 } else if (NULL
== entry
) {
1918 comm_close(fd
); /* yuk */
1919 } else if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
)) {
1921 } else if ((done
= clientCheckTransferDone(http
)) != 0 || size
== 0) {
1922 debug(33, 5) ("clientWriteComplete: FD %d transfer is DONE\n", fd
);
1923 /* We're finished case */
1924 if (httpReplyBodySize(http
->request
->method
, entry
->mem_obj
->reply
) < 0) {
1925 debug(33, 5) ("clientWriteComplete: closing, content_length < 0\n");
1928 debug(33, 5) ("clientWriteComplete: closing, !done\n");
1930 } else if (clientGotNotEnough(http
)) {
1931 debug(33, 5) ("clientWriteComplete: client didn't get all it expected\n");
1933 } else if (http
->request
->flags
.proxy_keepalive
) {
1934 debug(33, 5) ("clientWriteComplete: FD %d Keeping Alive\n", fd
);
1935 clientKeepaliveNextRequest(http
);
1939 } else if (clientReplyBodyTooLarge((int) http
->out
.offset
)) {
1942 /* More data will be coming from primary server; register with
1943 * storage manager. */
1944 if (EBIT_TEST(entry
->flags
, ENTRY_ABORTED
))
1945 debug(33, 0) ("clientWriteComplete 2: ENTRY_ABORTED\n");
1946 storeClientCopy(http
->sc
, entry
,
1950 memAllocate(MEM_CLIENT_SOCK_BUF
),
1957 * client issued a request with an only-if-cached cache-control directive;
1958 * we did not find a cached object that can be returned without
1959 * contacting other servers;
1960 * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
1963 clientProcessOnlyIfCachedMiss(clientHttpRequest
* http
)
1965 char *url
= http
->uri
;
1966 request_t
*r
= http
->request
;
1967 ErrorState
*err
= NULL
;
1968 debug(33, 4) ("clientProcessOnlyIfCachedMiss: '%s %s'\n",
1969 RequestMethodStr
[r
->method
], url
);
1970 http
->al
.http
.code
= HTTP_GATEWAY_TIMEOUT
;
1971 err
= errorCon(ERR_ONLY_IF_CACHED_MISS
, HTTP_GATEWAY_TIMEOUT
);
1972 err
->request
= requestLink(r
);
1973 err
->src_addr
= http
->conn
->peer
.sin_addr
;
1975 storeUnregister(http
->sc
, http
->entry
, http
);
1977 storeUnlockObject(http
->entry
);
1979 http
->entry
= clientCreateStoreEntry(http
, r
->method
, null_request_flags
);
1980 errorAppendEntry(http
->entry
, err
);
1984 * Return true if we should force a cache miss on this range request.
1985 * entry must be non-NULL.
1988 clientCheckRangeForceMiss(StoreEntry
* entry
, HttpHdrRange
* range
)
1991 * If the range_offset_limit is NOT in effect, there
1992 * is no reason to force a miss.
1994 if (0 == httpHdrRangeOffsetLimit(range
))
1997 * Here, we know it's possibly a hit. If we already have the
1998 * whole object cached, we won't force a miss.
2000 if (STORE_OK
== entry
->store_status
)
2001 return 0; /* we have the whole object */
2003 * Now we have a hit on a PENDING object. We need to see
2004 * if the part we want is already cached. If so, we don't
2007 assert(NULL
!= entry
->mem_obj
);
2008 if (httpHdrRangeFirstOffset(range
) <= entry
->mem_obj
->inmem_hi
)
2011 * Even though we have a PENDING copy of the object, we
2012 * don't want to wait to reach the first range offset,
2013 * so we force a miss for a new range request to the
2020 clientProcessRequest2(clientHttpRequest
* http
)
2022 request_t
*r
= http
->request
;
2024 e
= http
->entry
= storeGetPublic(http
->uri
, r
->method
);
2025 if (r
->method
== METHOD_HEAD
&& e
== NULL
) {
2026 /* We can generate a HEAD reply from a cached GET object */
2027 e
= http
->entry
= storeGetPublic(http
->uri
, METHOD_GET
);
2029 /* Release negatively cached IP-cache entries on reload */
2030 if (r
->flags
.nocache
)
2031 ipcacheInvalidate(r
->host
);
2033 else if (r
->flags
.nocache_hack
)
2034 ipcacheInvalidate(r
->host
);
2036 #if USE_CACHE_DIGESTS
2037 http
->lookup_type
= e
? "HIT" : "MISS";
2040 /* this object isn't in the cache */
2041 debug(33, 3) ("clientProcessRequest2: storeGet() MISS\n");
2042 return LOG_TCP_MISS
;
2044 if (Config
.onoff
.offline
) {
2045 debug(33, 3) ("clientProcessRequest2: offline HIT\n");
2049 if (http
->redirect
.status
) {
2050 /* force this to be a miss */
2052 return LOG_TCP_MISS
;
2054 if (!storeEntryValidToSend(e
)) {
2055 debug(33, 3) ("clientProcessRequest2: !storeEntryValidToSend MISS\n");
2057 return LOG_TCP_MISS
;
2059 if (EBIT_TEST(e
->flags
, ENTRY_SPECIAL
)) {
2060 /* Special entries are always hits, no matter what the client says */
2061 debug(33, 3) ("clientProcessRequest2: ENTRY_SPECIAL HIT\n");
2066 if (e
->store_status
== STORE_PENDING
) {
2067 if (r
->flags
.nocache
|| r
->flags
.nocache_hack
) {
2068 debug(33, 3) ("Clearing no-cache for STORE_PENDING request\n\t%s\n",
2070 r
->flags
.nocache
= 0;
2071 r
->flags
.nocache_hack
= 0;
2075 if (r
->flags
.nocache
) {
2076 debug(33, 3) ("clientProcessRequest2: no-cache REFRESH MISS\n");
2078 ipcacheInvalidate(r
->host
);
2079 return LOG_TCP_CLIENT_REFRESH_MISS
;
2081 if (NULL
== r
->range
) {
2083 } else if (httpHdrRangeWillBeComplex(r
->range
)) {
2085 * Some clients break if we return "200 OK" for a Range
2086 * request. We would have to return "200 OK" for a _complex_
2087 * Range request that is also a HIT. Thus, let's prevent HITs
2088 * on complex Range requests
2090 debug(33, 3) ("clientProcessRequest2: complex range MISS\n");
2092 return LOG_TCP_MISS
;
2093 } else if (clientCheckRangeForceMiss(e
, r
->range
)) {
2094 debug(33, 3) ("clientProcessRequest2: forcing miss due to range_offset_limit\n");
2096 return LOG_TCP_MISS
;
2098 debug(33, 3) ("clientProcessRequest2: default HIT\n");
2104 clientProcessRequest(clientHttpRequest
* http
)
2106 char *url
= http
->uri
;
2107 request_t
*r
= http
->request
;
2108 int fd
= http
->conn
->fd
;
2110 http_version_t version
;
2111 debug(33, 4) ("clientProcessRequest: %s '%s'\n",
2112 RequestMethodStr
[r
->method
],
2114 if (r
->method
== METHOD_CONNECT
) {
2115 http
->log_type
= LOG_TCP_MISS
;
2116 sslStart(fd
, url
, r
, &http
->out
.size
, &http
->al
.http
.code
);
2118 } else if (r
->method
== METHOD_PURGE
) {
2119 clientPurgeRequest(http
);
2121 } else if (r
->method
== METHOD_TRACE
) {
2122 if (r
->max_forwards
== 0) {
2123 http
->entry
= clientCreateStoreEntry(http
, r
->method
, null_request_flags
);
2124 storeReleaseRequest(http
->entry
);
2125 storeBuffer(http
->entry
);
2126 rep
= httpReplyCreate();
2127 httpBuildVersion(&version
, 1, 0);
2128 httpReplySetHeaders(rep
, version
, HTTP_OK
, NULL
, "text/plain",
2129 httpRequestPrefixLen(r
), 0, squid_curtime
);
2130 httpReplySwapOut(rep
, http
->entry
);
2131 httpReplyDestroy(rep
);
2132 httpRequestSwapOut(r
, http
->entry
);
2133 storeComplete(http
->entry
);
2137 http
->log_type
= LOG_TCP_MISS
;
2138 } else if (r
->content_length
>= 0) {
2140 * Need to initialize pump even if content-length: 0
2142 http
->log_type
= LOG_TCP_MISS
;
2143 /* XXX oof, POST can be cached! */
2144 pumpInit(fd
, r
, http
->uri
);
2146 http
->log_type
= clientProcessRequest2(http
);
2148 debug(33, 4) ("clientProcessRequest: %s for '%s'\n",
2149 log_tags
[http
->log_type
],
2151 http
->out
.offset
= 0;
2152 if (NULL
!= http
->entry
) {
2153 storeLockObject(http
->entry
);
2154 storeCreateMemObject(http
->entry
, http
->uri
, http
->log_uri
);
2155 http
->entry
->mem_obj
->method
= r
->method
;
2156 http
->sc
= storeClientListAdd(http
->entry
, http
);
2158 delaySetStoreClient(http
->sc
, delayClient(r
));
2160 storeClientCopy(http
->sc
, http
->entry
,
2164 memAllocate(MEM_CLIENT_SOCK_BUF
),
2168 /* MISS CASE, http->log_type is already set! */
2169 clientProcessMiss(http
);
2174 * Prepare to fetch the object as it's a cache miss of some kind.
2177 clientProcessMiss(clientHttpRequest
* http
)
2179 char *url
= http
->uri
;
2180 request_t
*r
= http
->request
;
2181 ErrorState
*err
= NULL
;
2182 debug(33, 4) ("clientProcessMiss: '%s %s'\n",
2183 RequestMethodStr
[r
->method
], url
);
2185 * We might have a left-over StoreEntry from a failed cache hit
2189 if (EBIT_TEST(http
->entry
->flags
, ENTRY_SPECIAL
)) {
2190 debug(33, 0) ("clientProcessMiss: miss on a special object (%s).\n", url
);
2191 debug(33, 0) ("\tlog_type = %s\n", log_tags
[http
->log_type
]);
2192 storeEntryDump(http
->entry
, 1);
2194 storeUnregister(http
->sc
, http
->entry
, http
);
2196 storeUnlockObject(http
->entry
);
2199 if (clientOnlyIfCached(http
)) {
2200 clientProcessOnlyIfCachedMiss(http
);
2204 * Deny loops when running in accelerator/transproxy mode.
2206 if (http
->flags
.accel
&& r
->flags
.loopdetect
) {
2207 http
->al
.http
.code
= HTTP_FORBIDDEN
;
2208 err
= errorCon(ERR_ACCESS_DENIED
, HTTP_FORBIDDEN
);
2209 err
->request
= requestLink(r
);
2210 err
->src_addr
= http
->conn
->peer
.sin_addr
;
2211 http
->entry
= clientCreateStoreEntry(http
, r
->method
, null_request_flags
);
2212 errorAppendEntry(http
->entry
, err
);
2215 assert(http
->out
.offset
== 0);
2216 http
->entry
= clientCreateStoreEntry(http
, r
->method
, r
->flags
);
2217 if (http
->redirect
.status
) {
2218 HttpReply
*rep
= httpReplyCreate();
2219 #if LOG_TCP_REDIRECTS
2220 http
->log_type
= LOG_TCP_REDIRECT
;
2222 storeReleaseRequest(http
->entry
);
2223 httpRedirectReply(rep
, http
->redirect
.status
, http
->redirect
.location
);
2224 httpReplySwapOut(rep
, http
->entry
);
2225 httpReplyDestroy(rep
);
2226 storeComplete(http
->entry
);
2229 if (http
->flags
.internal
)
2230 r
->protocol
= PROTO_INTERNAL
;
2231 fwdStart(http
->conn
->fd
, http
->entry
, r
);
2234 static clientHttpRequest
*
2235 parseHttpRequestAbort(ConnStateData
* conn
, const char *uri
)
2237 clientHttpRequest
*http
= memAllocate(MEM_CLIENTHTTPREQUEST
);
2238 cbdataAdd(http
, memFree
, MEM_CLIENTHTTPREQUEST
);
2240 http
->start
= current_time
;
2241 http
->req_sz
= conn
->in
.offset
;
2242 http
->uri
= xstrdup(uri
);
2243 http
->log_uri
= xstrndup(uri
, MAX_URL
);
2244 http
->range_iter
.boundary
= StringNull
;
2245 dlinkAdd(http
, &http
->active
, &ClientActiveRequests
);
2250 * parseHttpRequest()
2253 * NULL on error or incomplete request
2254 * a clientHttpRequest structure on success
2256 static clientHttpRequest
*
2257 parseHttpRequest(ConnStateData
* conn
, method_t
* method_p
, int *status
,
2258 char **prefix_p
, size_t * req_line_sz_p
)
2263 char *req_hdr
= NULL
;
2264 http_version_t http_ver
;
2268 size_t header_sz
; /* size of headers, not including first line */
2269 size_t prefix_sz
; /* size of whole request (req-line + headers) */
2273 clientHttpRequest
*http
= NULL
;
2275 struct natlookup natLookup
;
2276 static int natfd
= -1;
2277 static int siocgnatl_cmd
= SIOCGNATL
& 0xff;
2281 size_t sock_sz
= sizeof(conn
->me
);
2284 if ((req_sz
= headersEnd(conn
->in
.buf
, conn
->in
.offset
)) == 0) {
2285 debug(33, 5) ("Incomplete request, waiting for end of headers\n");
2288 *method_p
= METHOD_NONE
;
2291 assert(req_sz
<= conn
->in
.offset
);
2292 /* Use memcpy, not strdup! */
2293 inbuf
= xmalloc(req_sz
+ 1);
2294 xmemcpy(inbuf
, conn
->in
.buf
, req_sz
);
2295 *(inbuf
+ req_sz
) = '\0';
2297 /* pre-set these values to make aborting simpler */
2299 *method_p
= METHOD_NONE
;
2302 /* Barf on NULL characters in the headers */
2303 if (strlen(inbuf
) != req_sz
) {
2304 debug(33, 1) ("parseHttpRequest: Requestheader contains NULL characters\n");
2305 return parseHttpRequestAbort(conn
, "error:invalid-request");
2307 /* Look for request method */
2308 if ((mstr
= strtok(inbuf
, "\t ")) == NULL
) {
2309 debug(33, 1) ("parseHttpRequest: Can't get request method\n");
2310 return parseHttpRequestAbort(conn
, "error:invalid-request-method");
2312 method
= urlParseMethod(mstr
);
2313 if (method
== METHOD_NONE
) {
2314 debug(33, 1) ("parseHttpRequest: Unsupported method '%s'\n", mstr
);
2315 return parseHttpRequestAbort(conn
, "error:unsupported-request-method");
2317 debug(33, 5) ("parseHttpRequest: Method is '%s'\n", mstr
);
2320 /* look for URL+HTTP/x.x */
2321 if ((url
= strtok(NULL
, "\n")) == NULL
) {
2322 debug(33, 1) ("parseHttpRequest: Missing URL\n");
2323 return parseHttpRequestAbort(conn
, "error:missing-url");
2325 while (xisspace(*url
))
2327 t
= url
+ strlen(url
);
2332 if (xisspace(*t
) && !strncmp(t
+ 1, "HTTP/", 5)) {
2337 while (t
> url
&& xisspace(*t
))
2339 debug(33, 5) ("parseHttpRequest: URI is '%s'\n", url
);
2340 if (token
== NULL
) {
2341 debug(33, 3) ("parseHttpRequest: Missing HTTP identifier\n");
2342 #if RELAXED_HTTP_PARSER
2343 httpBuildVersion(&http_ver
, 0, 9); /* wild guess */
2345 return parseHttpRequestAbort(conn
, "error:missing-http-ident");
2348 if (sscanf(token
+ 5, "%d.%d", &http_ver
.major
, &http_ver
.minor
) != 2) {
2349 debug(33, 3) ("parseHttpRequest: Invalid HTTP identifier.\n");
2350 return parseHttpRequestAbort(conn
, "error: invalid HTTP-ident");
2352 debug(33, 6) ("parseHttpRequest: Client HTTP version %d.%d.\n", http_ver
.major
, http_ver
.minor
);
2356 * Process headers after request line
2358 req_hdr
= strtok(NULL
, null_string
);
2359 header_sz
= req_sz
- (req_hdr
- inbuf
);
2360 if (0 == header_sz
) {
2361 debug(33, 3) ("parseHttpRequest: header_sz == 0\n");
2365 assert(header_sz
> 0);
2366 debug(33, 3) ("parseHttpRequest: req_hdr = {%s}\n", req_hdr
);
2367 end
= req_hdr
+ header_sz
;
2368 debug(33, 3) ("parseHttpRequest: end = {%s}\n", end
);
2370 prefix_sz
= end
- inbuf
;
2371 *req_line_sz_p
= req_hdr
- inbuf
;
2372 debug(33, 3) ("parseHttpRequest: prefix_sz = %d, req_line_sz = %d\n",
2373 (int) prefix_sz
, (int) *req_line_sz_p
);
2374 assert(prefix_sz
<= conn
->in
.offset
);
2376 /* Ok, all headers are received */
2377 http
= memAllocate(MEM_CLIENTHTTPREQUEST
);
2378 cbdataAdd(http
, memFree
, MEM_CLIENTHTTPREQUEST
);
2379 http
->http_ver
= http_ver
;
2381 http
->start
= current_time
;
2382 http
->req_sz
= prefix_sz
;
2383 http
->range_iter
.boundary
= StringNull
;
2384 *prefix_p
= xmalloc(prefix_sz
+ 1);
2385 xmemcpy(*prefix_p
, conn
->in
.buf
, prefix_sz
);
2386 *(*prefix_p
+ prefix_sz
) = '\0';
2387 dlinkAdd(http
, &http
->active
, &ClientActiveRequests
);
2389 debug(33, 5) ("parseHttpRequest: Request Header is\n%s\n", (*prefix_p
) + *req_line_sz_p
);
2390 if ((t
= strchr(url
, '#'))) /* remove HTML anchors */
2393 /* handle internal objects */
2394 if (internalCheck(url
)) {
2395 /* prepend our name & port */
2396 http
->uri
= xstrdup(internalLocalUri(NULL
, url
));
2397 http
->flags
.internal
= 1;
2398 http
->flags
.accel
= 1;
2400 /* see if we running in Config2.Accel.on, if so got to convert it to URL */
2401 else if (Config2
.Accel
.on
&& *url
== '/') {
2402 /* prepend the accel prefix */
2403 if (opt_accel_uses_host
&& (t
= mime_get_header(req_hdr
, "Host"))) {
2404 int vport
= (int) Config
.Accel
.port
;
2406 /* If a Host: header was specified, use it to build the URL
2407 * instead of the one in the Config file. */
2409 * XXX Use of the Host: header here opens a potential
2410 * security hole. There are no checks that the Host: value
2411 * corresponds to one of your servers. It might, for example,
2412 * refer to www.playboy.com. The 'dst' and/or 'dst_domain' ACL
2413 * types should be used to prevent httpd-accelerators
2414 * handling requests for non-local servers */
2416 if ((q
= strchr(t
, ':'))) {
2420 url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
+
2422 http
->uri
= xcalloc(url_sz
, 1);
2423 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
2425 } else if (vhost_mode
) {
2427 /* Put the local socket IP address as the hostname */
2428 url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
;
2429 http
->uri
= xcalloc(url_sz
, 1);
2431 vport
= (int) ntohs(http
->conn
->me
.sin_port
);
2433 vport
= (int) Config
.Accel
.port
;
2435 natLookup
.nl_inport
= http
->conn
->me
.sin_port
;
2436 natLookup
.nl_outport
= http
->conn
->peer
.sin_port
;
2437 natLookup
.nl_inip
= http
->conn
->me
.sin_addr
;
2438 natLookup
.nl_outip
= http
->conn
->peer
.sin_addr
;
2439 natLookup
.nl_flags
= IPN_TCP
;
2441 natfd
= open(IPL_NAT
, O_RDONLY
, 0);
2443 debug(50, 1) ("parseHttpRequest: NAT open failed: %s\n",
2445 return parseHttpRequestAbort(conn
, "error:nat-open-failed");
2448 * IP-Filter changed the type for SIOCGNATL between
2449 * 3.3 and 3.4. It also changed the cmd value for
2450 * SIOCGNATL, so at least we can detect it. We could
2451 * put something in configure and use ifdefs here, but
2452 * this seems simpler.
2454 if (63 == siocgnatl_cmd
) {
2455 struct natlookup
*nlp
= &natLookup
;
2456 x
= ioctl(natfd
, SIOCGNATL
, &nlp
);
2458 x
= ioctl(natfd
, SIOCGNATL
, &natLookup
);
2461 if (errno
!= ESRCH
) {
2462 debug(50, 1) ("parseHttpRequest: NAT lookup failed: ioctl(SIOCGNATL)\n");
2465 return parseHttpRequestAbort(conn
, "error:nat-lookup-failed");
2467 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
2468 inet_ntoa(http
->conn
->me
.sin_addr
),
2471 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
2472 inet_ntoa(natLookup
.nl_realip
),
2476 /* If the call fails the address structure will be unchanged */
2477 getsockopt(conn
->fd
, SOL_IP
, SO_ORIGINAL_DST
, &conn
->me
, &sock_sz
);
2478 debug(33, 5) ("parseHttpRequest: addr = %s", inet_ntoa(conn
->me
.sin_addr
) );
2480 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
2481 inet_ntoa(http
->conn
->me
.sin_addr
),
2484 debug(33, 5) ("VHOST REWRITE: '%s'\n", http
->uri
);
2486 url_sz
= strlen(Config2
.Accel
.prefix
) + strlen(url
) +
2487 Config
.appendDomainLen
+ 1;
2488 http
->uri
= xcalloc(url_sz
, 1);
2489 snprintf(http
->uri
, url_sz
, "%s%s", Config2
.Accel
.prefix
, url
);
2491 http
->flags
.accel
= 1;
2493 /* URL may be rewritten later, so make extra room */
2494 url_sz
= strlen(url
) + Config
.appendDomainLen
+ 5;
2495 http
->uri
= xcalloc(url_sz
, 1);
2496 strcpy(http
->uri
, url
);
2497 http
->flags
.accel
= 0;
2499 if (!stringHasCntl(http
->uri
))
2500 http
->log_uri
= xstrndup(http
->uri
, MAX_URL
);
2502 http
->log_uri
= xstrndup(rfc1738_escape_unescaped(http
->uri
), MAX_URL
);
2503 debug(33, 5) ("parseHttpRequest: Complete request received\n");
2510 clientReadDefer(int fdnotused
, void *data
)
2512 ConnStateData
*conn
= data
;
2513 return conn
->defer
.until
> squid_curtime
;
2517 clientReadRequest(int fd
, void *data
)
2519 ConnStateData
*conn
= data
;
2520 int parser_return_code
= 0;
2522 request_t
*request
= NULL
;
2526 clientHttpRequest
*http
= NULL
;
2527 clientHttpRequest
**H
= NULL
;
2528 char *prefix
= NULL
;
2529 ErrorState
*err
= NULL
;
2530 fde
*F
= &fd_table
[fd
];
2531 int len
= conn
->in
.size
- conn
->in
.offset
- 1;
2532 debug(33, 4) ("clientReadRequest: FD %d: reading request...\n", fd
);
2533 statCounter
.syscalls
.sock
.reads
++;
2534 size
= read(fd
, conn
->in
.buf
+ conn
->in
.offset
, len
);
2536 fd_bytes(fd
, size
, FD_READ
);
2537 kb_incr(&statCounter
.client_http
.kbytes_in
, size
);
2540 * Don't reset the timeout value here. The timeout value will be
2541 * set to Config.Timeout.request by httpAccept() and
2542 * clientWriteComplete(), and should apply to the request as a
2543 * whole, not individual read() calls. Plus, it breaks our
2544 * lame half-close detection
2546 commSetSelect(fd
, COMM_SELECT_READ
, clientReadRequest
, conn
, 0);
2548 if (conn
->chr
== NULL
) {
2549 /* no current or pending requests */
2552 } else if (!Config
.onoff
.half_closed_clients
) {
2553 /* admin doesn't want to support half-closed client sockets */
2557 /* It might be half-closed, we can't tell */
2558 debug(33, 5) ("clientReadRequest: FD %d closed?\n", fd
);
2559 F
->flags
.socket_eof
= 1;
2560 conn
->defer
.until
= squid_curtime
+ 1;
2562 fd_note(fd
, "half-closed");
2564 } else if (size
< 0) {
2565 if (!ignoreErrno(errno
)) {
2566 debug(50, 2) ("clientReadRequest: FD %d: %s\n", fd
, xstrerror());
2569 } else if (conn
->in
.offset
== 0) {
2570 debug(50, 2) ("clientReadRequest: FD %d: no data to process (%s)\n", fd
, xstrerror());
2573 /* Continue to process previously read data */
2576 conn
->in
.offset
+= size
;
2577 /* Skip leading (and trailing) whitespace */
2578 while (conn
->in
.offset
> 0) {
2581 while (conn
->in
.offset
> 0 && xisspace(conn
->in
.buf
[0])) {
2582 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ 1, conn
->in
.offset
- 1);
2585 conn
->in
.buf
[conn
->in
.offset
] = '\0'; /* Terminate the string */
2586 if (conn
->in
.offset
== 0)
2588 /* Limit the number of concurrent requests to 2 */
2589 for (H
= &conn
->chr
, nrequests
= 0; *H
; H
= &(*H
)->next
, nrequests
++);
2590 if (nrequests
>= 2) {
2591 debug(33, 3) ("clientReadRequest: FD %d max concurrent requests reached\n", fd
);
2592 debug(33, 5) ("clientReadRequest: FD %d defering new request until one is done\n", fd
);
2593 conn
->defer
.until
= squid_curtime
+ 100; /* Reset when a request is complete */
2596 /* Process request */
2597 http
= parseHttpRequest(conn
,
2599 &parser_return_code
,
2605 assert(http
->req_sz
> 0);
2606 conn
->in
.offset
-= http
->req_sz
;
2607 assert(conn
->in
.offset
>= 0);
2608 debug(33, 5) ("conn->in.offset = %d\n", (int) conn
->in
.offset
);
2610 * If we read past the end of this request, move the remaining
2611 * data to the beginning
2613 if (conn
->in
.offset
> 0)
2614 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ http
->req_sz
, conn
->in
.offset
);
2615 /* add to the client request queue */
2616 for (H
= &conn
->chr
; *H
; H
= &(*H
)->next
);
2620 * I wanted to lock 'http' here since its callback data for
2621 * clientLifetimeTimeout(), but there's no logical place to
2622 * cbdataUnlock if the timeout never happens. Maybe its safe
2623 * enough to assume that if the FD is open, and the timeout
2624 * triggers, that 'http' is valid.
2626 commSetTimeout(fd
, Config
.Timeout
.lifetime
, clientLifetimeTimeout
, http
);
2627 if (parser_return_code
< 0) {
2628 debug(33, 1) ("clientReadRequest: FD %d Invalid Request\n", fd
);
2629 err
= errorCon(ERR_INVALID_REQ
, HTTP_BAD_REQUEST
);
2630 err
->request_hdrs
= xstrdup(conn
->in
.buf
);
2631 http
->entry
= clientCreateStoreEntry(http
, method
, null_request_flags
);
2632 errorAppendEntry(http
->entry
, err
);
2636 if ((request
= urlParse(method
, http
->uri
)) == NULL
) {
2637 debug(33, 5) ("Invalid URL: %s\n", http
->uri
);
2638 err
= errorCon(ERR_INVALID_URL
, HTTP_BAD_REQUEST
);
2639 err
->src_addr
= conn
->peer
.sin_addr
;
2640 err
->url
= xstrdup(http
->uri
);
2641 http
->al
.http
.code
= err
->http_status
;
2642 http
->entry
= clientCreateStoreEntry(http
, method
, null_request_flags
);
2643 errorAppendEntry(http
->entry
, err
);
2647 /* compile headers */
2648 /* we should skip request line! */
2649 if (!httpRequestParseHeader(request
, prefix
+ req_line_sz
))
2650 debug(33, 1) ("Failed to parse request headers: %s\n%s\n",
2652 /* continue anyway? */
2654 request
->flags
.accelerated
= http
->flags
.accel
;
2655 if (!http
->flags
.internal
) {
2656 if (internalCheck(strBuf(request
->urlpath
))) {
2657 if (internalHostnameIs(request
->host
) &&
2658 request
->port
== ntohs(Config
.Sockaddr
.http
->s
.sin_port
)) {
2659 http
->flags
.internal
= 1;
2660 } else if (internalStaticCheck(strBuf(request
->urlpath
))) {
2661 xstrncpy(request
->host
, internalHostname(), SQUIDHOSTNAMELEN
);
2662 request
->port
= ntohs(Config
.Sockaddr
.http
->s
.sin_port
);
2663 http
->flags
.internal
= 1;
2668 * cache the Content-length value in request_t.
2670 request
->content_length
= httpHeaderGetInt(&request
->header
,
2671 HDR_CONTENT_LENGTH
);
2672 request
->flags
.internal
= http
->flags
.internal
;
2674 safe_free(http
->log_uri
);
2675 http
->log_uri
= xstrdup(urlCanonicalClean(request
));
2676 request
->client_addr
= conn
->peer
.sin_addr
;
2677 request
->my_addr
= conn
->me
.sin_addr
;
2678 request
->my_port
= ntohs(conn
->me
.sin_port
);
2679 request
->http_ver
= http
->http_ver
;
2680 if (!urlCheckRequest(request
)) {
2681 err
= errorCon(ERR_UNSUP_REQ
, HTTP_NOT_IMPLEMENTED
);
2682 err
->src_addr
= conn
->peer
.sin_addr
;
2683 err
->request
= requestLink(request
);
2684 request
->flags
.proxy_keepalive
= 0;
2685 http
->al
.http
.code
= err
->http_status
;
2686 http
->entry
= clientCreateStoreEntry(http
, request
->method
, null_request_flags
);
2687 errorAppendEntry(http
->entry
, err
);
2690 if (0 == clientCheckContentLength(request
)) {
2691 err
= errorCon(ERR_INVALID_REQ
, HTTP_LENGTH_REQUIRED
);
2692 err
->src_addr
= conn
->peer
.sin_addr
;
2693 err
->request
= requestLink(request
);
2694 http
->al
.http
.code
= err
->http_status
;
2695 http
->entry
= clientCreateStoreEntry(http
, request
->method
, null_request_flags
);
2696 errorAppendEntry(http
->entry
, err
);
2699 http
->request
= requestLink(request
);
2701 * We need to set the keepalive flag before doing some
2702 * hacks for POST/PUT requests below. Maybe we could
2703 * set keepalive flag even earlier.
2705 clientSetKeepaliveFlag(http
);
2707 * break here if the request has a content-length
2708 * because there is a reqeust body following and we
2709 * don't want to parse it as though it was new request.
2711 if (request
->content_length
>= 0) {
2712 int copy_len
= XMIN(conn
->in
.offset
, request
->content_length
);
2714 assert(conn
->in
.offset
>= copy_len
);
2715 request
->body_sz
= copy_len
;
2716 request
->body
= xmalloc(request
->body_sz
);
2717 xmemcpy(request
->body
, conn
->in
.buf
, request
->body_sz
);
2718 conn
->in
.offset
-= copy_len
;
2719 if (conn
->in
.offset
)
2720 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ copy_len
, conn
->in
.offset
);
2723 * if we didn't get the full body now, then more will
2724 * be arriving on the client socket. Lets cancel
2725 * the read handler until this request gets forwarded.
2727 if (request
->body_sz
< request
->content_length
)
2728 commSetSelect(fd
, COMM_SELECT_READ
, NULL
, NULL
, 0);
2729 if (request
->content_length
< 0)
2731 else if (clientRequestBodyTooLarge(request
->content_length
)) {
2732 err
= errorCon(ERR_TOO_BIG
, HTTP_REQUEST_ENTITY_TOO_LARGE
);
2733 err
->request
= requestLink(request
);
2734 http
->entry
= clientCreateStoreEntry(http
,
2735 METHOD_NONE
, null_request_flags
);
2736 errorAppendEntry(http
->entry
, err
);
2740 clientAccessCheck(http
);
2741 continue; /* while offset > 0 */
2742 } else if (parser_return_code
== 0) {
2744 * Partial request received; reschedule until parseHttpRequest()
2745 * is happy with the input
2747 k
= conn
->in
.size
- 1 - conn
->in
.offset
;
2749 if (conn
->in
.offset
>= Config
.maxRequestHeaderSize
) {
2750 /* The request is too large to handle */
2751 debug(33, 1) ("Request header is too large (%d bytes)\n",
2752 (int) conn
->in
.offset
);
2753 debug(33, 1) ("Config 'request_header_max_size'= %d bytes.\n",
2754 Config
.maxRequestHeaderSize
);
2755 err
= errorCon(ERR_TOO_BIG
, HTTP_REQUEST_ENTITY_TOO_LARGE
);
2756 http
= parseHttpRequestAbort(conn
, "error:request-too-large");
2757 /* add to the client request queue */
2758 for (H
= &conn
->chr
; *H
; H
= &(*H
)->next
);
2760 http
->entry
= clientCreateStoreEntry(http
, METHOD_NONE
, null_request_flags
);
2761 errorAppendEntry(http
->entry
, err
);
2764 /* Grow the request memory area to accomodate for a large request */
2765 conn
->in
.size
+= CLIENT_REQ_BUF_SZ
;
2766 if (conn
->in
.size
== 2 * CLIENT_REQ_BUF_SZ
) {
2767 p
= conn
->in
.buf
; /* get rid of fixed size Pooled buffer */
2768 conn
->in
.buf
= xcalloc(2, CLIENT_REQ_BUF_SZ
);
2769 xmemcpy(conn
->in
.buf
, p
, CLIENT_REQ_BUF_SZ
);
2770 memFree(p
, MEM_CLIENT_REQ_BUF
);
2772 conn
->in
.buf
= xrealloc(conn
->in
.buf
, conn
->in
.size
);
2773 /* XXX account conn->in.buf */
2774 debug(33, 3) ("Handling a large request, offset=%d inbufsize=%d\n",
2775 (int) conn
->in
.offset
, conn
->in
.size
);
2776 k
= conn
->in
.size
- 1 - conn
->in
.offset
;
2783 /* general lifetime handler for HTTP requests */
2785 requestTimeout(int fd
, void *data
)
2787 #if THIS_CONFUSES_PERSISTENT_CONNECTION_AWARE_BROWSERS_AND_USERS
2788 ConnStateData
*conn
= data
;
2790 debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd
);
2791 if (fd_table
[fd
].rwstate
) {
2793 * Some data has been sent to the client, just close the FD
2796 } else if (conn
->nrequests
) {
2798 * assume its a persistent connection; just close it
2805 err
= errorCon(ERR_LIFETIME_EXP
, HTTP_REQUEST_TIMEOUT
);
2806 err
->url
= xstrdup("N/A");
2808 * Normally we shouldn't call errorSend() in client_side.c, but
2809 * it should be okay in this case. Presumably if we get here
2810 * this is the first request for the connection, and no data
2811 * has been written yet
2813 assert(conn
->chr
== NULL
);
2816 * if we don't close() here, we still need a timeout handler!
2818 commSetTimeout(fd
, 30, requestTimeout
, conn
);
2820 * Aha, but we don't want a read handler!
2822 commSetSelect(fd
, COMM_SELECT_READ
, NULL
, NULL
, 0);
2826 * Just close the connection to not confuse browsers
2827 * using persistent connections. Some browsers opens
2828 * an connection and then does not use it until much
2829 * later (presumeably because the request triggering
2830 * the open has already been completed on another
2833 debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd
);
2839 clientLifetimeTimeout(int fd
, void *data
)
2841 clientHttpRequest
*http
= data
;
2842 ConnStateData
*conn
= http
->conn
;
2843 debug(33, 1) ("WARNING: Closing client %s connection due to lifetime timeout\n",
2844 inet_ntoa(conn
->peer
.sin_addr
));
2845 debug(33, 1) ("\t%s\n", http
->uri
);
2850 httpAcceptDefer(int fdunused
, void *dataunused
)
2852 static time_t last_warn
= 0;
2853 if (fdNFree() >= RESERVED_FD
)
2855 if (last_warn
+ 15 < squid_curtime
) {
2856 debug(33, 0) ("WARNING! Your cache is running out of filedescriptors\n");
2857 last_warn
= squid_curtime
;
2862 /* Handle a new connection on HTTP socket. */
2864 httpAccept(int sock
, void *data
)
2868 ConnStateData
*connState
= NULL
;
2869 struct sockaddr_in peer
;
2870 struct sockaddr_in me
;
2871 int max
= INCOMING_HTTP_MAX
;
2873 static aclCheck_t identChecklist
;
2875 commSetSelect(sock
, COMM_SELECT_READ
, httpAccept
, NULL
, 0);
2876 while (max
-- && !httpAcceptDefer(sock
, NULL
)) {
2877 memset(&peer
, '\0', sizeof(struct sockaddr_in
));
2878 memset(&me
, '\0', sizeof(struct sockaddr_in
));
2879 if ((fd
= comm_accept(sock
, &peer
, &me
)) < 0) {
2880 if (!ignoreErrno(errno
))
2881 debug(50, 1) ("httpAccept: FD %d: accept failure: %s\n",
2885 debug(33, 4) ("httpAccept: FD %d: accepted\n", fd
);
2886 connState
= memAllocate(MEM_CONNSTATEDATA
);
2887 cbdataAdd(connState
, memFree
, MEM_CONNSTATEDATA
);
2888 connState
->peer
= peer
;
2889 connState
->log_addr
= peer
.sin_addr
;
2890 connState
->log_addr
.s_addr
&= Config
.Addrs
.client_netmask
.s_addr
;
2893 connState
->in
.size
= CLIENT_REQ_BUF_SZ
;
2894 connState
->in
.buf
= memAllocate(MEM_CLIENT_REQ_BUF
);
2895 /* XXX account connState->in.buf */
2896 comm_add_close_handler(fd
, connStateFree
, connState
);
2897 if (Config
.onoff
.log_fqdn
)
2898 fqdncache_gethostbyaddr(peer
.sin_addr
, FQDN_LOOKUP_IF_MISS
);
2899 commSetTimeout(fd
, Config
.Timeout
.request
, requestTimeout
, connState
);
2901 identChecklist
.src_addr
= peer
.sin_addr
;
2902 identChecklist
.my_addr
= me
.sin_addr
;
2903 identChecklist
.my_port
= ntohs(me
.sin_port
);
2904 if (aclCheckFast(Config
.accessList
.identLookup
, &identChecklist
))
2905 identStart(&me
, &peer
, clientIdentDone
, connState
);
2907 commSetSelect(fd
, COMM_SELECT_READ
, clientReadRequest
, connState
, 0);
2908 commSetDefer(fd
, clientReadDefer
, connState
);
2909 clientdbEstablished(peer
.sin_addr
, 1);
2915 #define SENDING_BODY 0
2916 #define SENDING_HDRSONLY 1
2918 clientCheckTransferDone(clientHttpRequest
* http
)
2920 int sending
= SENDING_BODY
;
2921 StoreEntry
*entry
= http
->entry
;
2928 * For now, 'done_copying' is used for special cases like
2929 * Range and HEAD requests.
2931 if (http
->flags
.done_copying
)
2934 * Handle STORE_OK objects.
2935 * objectLen(entry) will be set proprely.
2937 if (entry
->store_status
== STORE_OK
) {
2938 if (http
->out
.offset
>= objectLen(entry
))
2944 * Now, handle STORE_PENDING objects
2946 mem
= entry
->mem_obj
;
2947 assert(mem
!= NULL
);
2948 assert(http
->request
!= NULL
);
2950 if (reply
->hdr_sz
== 0)
2951 return 0; /* haven't found end of headers yet */
2952 else if (reply
->sline
.status
== HTTP_OK
)
2953 sending
= SENDING_BODY
;
2954 else if (reply
->sline
.status
== HTTP_NO_CONTENT
)
2955 sending
= SENDING_HDRSONLY
;
2956 else if (reply
->sline
.status
== HTTP_NOT_MODIFIED
)
2957 sending
= SENDING_HDRSONLY
;
2958 else if (reply
->sline
.status
< HTTP_OK
)
2959 sending
= SENDING_HDRSONLY
;
2960 else if (http
->request
->method
== METHOD_HEAD
)
2961 sending
= SENDING_HDRSONLY
;
2963 sending
= SENDING_BODY
;
2965 * Figure out how much data we are supposed to send.
2966 * If we are sending a body and we don't have a content-length,
2967 * then we must wait for the object to become STORE_OK.
2969 if (sending
== SENDING_HDRSONLY
)
2970 sendlen
= reply
->hdr_sz
;
2971 else if (reply
->content_length
< 0)
2974 sendlen
= reply
->content_length
+ reply
->hdr_sz
;
2976 * Now that we have the expected length, did we send it all?
2978 if (http
->out
.offset
< sendlen
)
2985 clientGotNotEnough(clientHttpRequest
* http
)
2987 int cl
= httpReplyBodySize(http
->request
->method
, http
->entry
->mem_obj
->reply
);
2988 int hs
= http
->entry
->mem_obj
->reply
->hdr_sz
;
2990 if (http
->out
.offset
< cl
+ hs
)
2996 * This function is designed to serve a fairly specific purpose.
2997 * Occasionally our vBNS-connected caches can talk to each other, but not
2998 * the rest of the world. Here we try to detect frequent failures which
2999 * make the cache unusable (e.g. DNS lookup and connect() failures). If
3000 * the failure:success ratio goes above 1.0 then we go into "hit only"
3001 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors
3002 * will only fetch HITs from us if they are using the ICP protocol. We
3003 * stay in this mode for 5 minutes.
3005 * Duane W., Sept 16, 1996
3009 checkFailureRatio(err_type etype
, hier_code hcode
)
3011 static double magic_factor
= 100.0;
3014 if (hcode
== HIER_NONE
)
3016 n_good
= magic_factor
/ (1.0 + request_failure_ratio
);
3017 n_bad
= magic_factor
- n_good
;
3020 case ERR_CONNECT_FAIL
:
3021 case ERR_READ_ERROR
:
3027 request_failure_ratio
= n_bad
/ n_good
;
3028 if (hit_only_mode_until
> squid_curtime
)
3030 if (request_failure_ratio
< 1.0)
3032 debug(33, 0) ("Failure Ratio at %4.2f\n", request_failure_ratio
);
3033 debug(33, 0) ("Going into hit-only-mode for %d minutes...\n",
3034 FAILURE_MODE_TIME
/ 60);
3035 hit_only_mode_until
= squid_curtime
+ FAILURE_MODE_TIME
;
3036 request_failure_ratio
= 0.8; /* reset to something less than 1.0 */
3040 clientHttpConnectionsOpen(void)
3042 sockaddr_in_list
*s
;
3044 for (s
= Config
.Sockaddr
.http
; s
; s
= s
->next
) {
3045 if (MAXHTTPPORTS
== NHttpSockets
) {
3046 debug(1, 1) ("WARNING: You have too many 'http_port' lines.\n");
3047 debug(1, 1) (" The limit is %d\n", MAXHTTPPORTS
);
3051 fd
= comm_open(SOCK_STREAM
,
3054 ntohs(s
->s
.sin_port
),
3061 commSetSelect(fd
, COMM_SELECT_READ
, httpAccept
, NULL
, 0);
3063 * We need to set a defer handler here so that we don't
3064 * peg the CPU with select() when we hit the FD limit.
3066 commSetDefer(fd
, httpAcceptDefer
, NULL
);
3067 debug(1, 1) ("Accepting HTTP connections at %s, port %d, FD %d.\n",
3068 inet_ntoa(s
->s
.sin_addr
),
3069 (int) ntohs(s
->s
.sin_port
),
3071 HttpSockets
[NHttpSockets
++] = fd
;
3073 if (NHttpSockets
< 1)
3074 fatal("Cannot open HTTP Port");
3078 clientHttpConnectionsClose(void)
3081 for (i
= 0; i
< NHttpSockets
; i
++) {
3082 if (HttpSockets
[i
] >= 0) {
3083 debug(1, 1) ("FD %d Closing HTTP connection\n", HttpSockets
[i
]);
3084 comm_close(HttpSockets
[i
]);
3085 HttpSockets
[i
] = -1;