3 * $Id: client_side.cc,v 1.592 2002/09/15 06:40:57 robertc Exp $
5 * DEBUG: section 33 Client-side Routines
6 * AUTHOR: Duane Wessels
8 * SQUID Web Proxy Cache http://www.squid-cache.org/
9 * ----------------------------------------------------------
11 * Squid is the result of efforts by numerous individuals from
12 * the Internet community; see the CONTRIBUTORS file for full
13 * details. Many organizations have provided support for Squid's
14 * development; see the SPONSORS file for full details. Squid is
15 * Copyrighted (C) 2001 by the Regents of the University of
16 * California; see the COPYRIGHT file for full details. Squid
17 * incorporates software developed and/or copyrighted by other
18 * sources; see the CREDITS file for full details.
20 * This program is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
30 * You should have received a copy of the GNU General Public License
31 * along with this program; if not, write to the Free Software
32 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
36 /* Errors and client side
38 * Problem the first: the store entry is no longer authoritative on the
39 * reply status. EBITTEST (E_ABORT) is no longer a valid test outside
40 * of client_side_reply.c.
41 * Problem the second: resources are wasted if we delay in cleaning up.
42 * Problem the third we can't depend on a connection close to clean up.
44 * Nice thing the first: Any step in the stream can callback with data
45 * representing an error.
46 * Nice thing the second: once you stop requesting reads from upstream,
47 * upstream can be stopped too.
49 * Solution #1: Error has a callback mechanism to hand over a membuf
50 * with the error content. The failing node pushes that back as the
51 * reply. Can this be generalised to reduce duplicate efforts?
52 * A: Possibly. For now, only one location uses this.
53 * How to deal with pre-stream errors?
54 * Tell client_side_reply that we *want* an error page before any
55 * stream calls occur. Then we simply read as normal.
62 #include <sys/ioctl.h>
64 #include <netinet/tcp.h>
66 #if HAVE_IP_FIL_COMPAT_H
67 #include <ip_fil_compat.h>
68 #elif HAVE_NETINET_IP_FIL_COMPAT_H
69 #include <netinet/ip_fil_compat.h>
70 #elif HAVE_IP_COMPAT_H
71 #include <ip_compat.h>
72 #elif HAVE_NETINET_IP_COMPAT_H
73 #include <netinet/ip_compat.h>
77 #elif HAVE_NETINET_IP_FIL_H
78 #include <netinet/ip_fil.h>
82 #elif HAVE_NETINET_IP_NAT_H
83 #include <netinet/ip_nat.h>
88 #include <sys/types.h>
89 #include <sys/socket.h>
90 #include <sys/ioctl.h>
91 #include <sys/fcntl.h>
93 #include <netinet/in.h>
94 #include <net/pfvar.h>
98 #include <linux/netfilter_ipv4.h>
103 #define comm_close comm_lingering_close
106 static const char *const crlf
= "\r\n";
108 #define FAILURE_MODE_TIME 300
110 /* Persistent connection logic:
112 * requests (httpClientRequest structs) get added to the connection
113 * list, with the current one being chr
115 * The request is *immediately* kicked off, and data flows through
116 * to clientSocketRecipient.
118 * If the data that arrives at clientSocketRecipient is not for the current
119 * request, clientSocketRecipient simply returns, without requesting more
120 * data, or sending it.
122 * ClientKeepAliveNextRequest will then detect the presence of data in
123 * the next clientHttpRequest, and will send it, restablishing the
127 /* our socket-related context */
128 typedef struct _clientSocketContext
{
129 clientHttpRequest
*http
; /* we own this */
130 char reqbuf
[HTTP_REQBUF_SZ
];
131 struct _clientSocketContext
*next
;
133 int deferred
:1; /* This is a pipelined request waiting for the
134 * current object to complete */
137 clientStreamNode
*node
;
139 const char *body_data
;
142 } clientSocketContext
;
144 CBDATA_TYPE(clientSocketContext
);
146 /* Local functions */
147 /* clientSocketContext */
148 static FREE clientSocketContextFree
;
149 static clientSocketContext
*clientSocketContextNew(clientHttpRequest
*);
151 static CWCB clientWriteComplete
;
152 static CWCB clientWriteBodyComplete
;
153 static PF clientReadRequest
;
154 static PF connStateFree
;
155 static PF requestTimeout
;
156 static PF clientLifetimeTimeout
;
157 static void checkFailureRatio(err_type
, hier_code
);
158 static clientSocketContext
*parseHttpRequestAbort(ConnStateData
* conn
,
160 static clientSocketContext
*parseHttpRequest(ConnStateData
*, method_t
*, int *,
163 static IDCB clientIdentDone
;
165 static CSCB clientSocketRecipient
;
166 static CSD clientSocketDetach
;
167 static void clientSetKeepaliveFlag(clientHttpRequest
*);
168 static int clientCheckContentLength(request_t
* r
);
169 static DEFER httpAcceptDefer
;
170 static int clientRequestBodyTooLarge(int clen
);
171 static void clientProcessBody(ConnStateData
* conn
);
174 clientSocketContextFree(void *data
)
176 clientSocketContext
*context
= data
;
177 ConnStateData
*conn
= context
->http
->conn
;
178 clientStreamNode
*node
= context
->http
->client_stream
.tail
->data
;
179 /* We are *always* the tail - prevent recursive free */
180 assert(context
== node
->data
);
182 httpRequestFree(context
->http
);
183 /* clean up connection links to us */
184 assert(context
!= context
->next
);
187 clientSocketContext
**S
;
188 assert(conn
->currentobject
!= NULL
);
189 /* Unlink us from the connection request list */
190 p
= &conn
->currentobject
;
191 S
= (clientSocketContext
**) p
;
199 context
->next
= NULL
;
203 clientSocketContext
*
204 clientSocketContextNew(clientHttpRequest
* http
)
206 clientSocketContext
*rv
;
207 assert(http
!= NULL
);
208 CBDATA_INIT_TYPE_FREECB(clientSocketContext
, clientSocketContextFree
);
209 rv
= cbdataAlloc(clientSocketContext
);
216 clientIdentDone(const char *ident
, void *data
)
218 ConnStateData
*conn
= data
;
219 xstrncpy(conn
->rfc931
, ident
? ident
: dash_str
, USER_IDENT_SZ
);
225 clientUpdateCounters(clientHttpRequest
* http
)
227 int svc_time
= tvSubMsec(http
->start
, current_time
);
229 HierarchyLogEntry
*H
;
230 statCounter
.client_http
.requests
++;
231 if (isTcpHit(http
->logType
))
232 statCounter
.client_http
.hits
++;
233 if (http
->logType
== LOG_TCP_HIT
)
234 statCounter
.client_http
.disk_hits
++;
235 else if (http
->logType
== LOG_TCP_MEM_HIT
)
236 statCounter
.client_http
.mem_hits
++;
237 if (http
->request
->errType
!= ERR_NONE
)
238 statCounter
.client_http
.errors
++;
239 statHistCount(&statCounter
.client_http
.all_svc_time
, svc_time
);
241 * The idea here is not to be complete, but to get service times
242 * for only well-defined types. For example, we don't include
243 * LOG_TCP_REFRESH_FAIL_HIT because its not really a cache hit
244 * (we *tried* to validate it, but failed).
246 switch (http
->logType
) {
247 case LOG_TCP_REFRESH_HIT
:
248 statHistCount(&statCounter
.client_http
.nh_svc_time
, svc_time
);
250 case LOG_TCP_IMS_HIT
:
251 statHistCount(&statCounter
.client_http
.nm_svc_time
, svc_time
);
254 case LOG_TCP_MEM_HIT
:
255 case LOG_TCP_OFFLINE_HIT
:
256 statHistCount(&statCounter
.client_http
.hit_svc_time
, svc_time
);
259 case LOG_TCP_CLIENT_REFRESH_MISS
:
260 statHistCount(&statCounter
.client_http
.miss_svc_time
, svc_time
);
263 /* make compiler warnings go away */
266 H
= &http
->request
->hier
;
269 statCounter
.cd
.times_used
++;
272 statCounter
.icp
.times_used
++;
274 if (0 != i
->stop
.tv_sec
&& 0 != i
->start
.tv_sec
)
275 statHistCount(&statCounter
.icp
.query_svc_time
,
276 tvSubUsec(i
->start
, i
->stop
));
278 statCounter
.icp
.query_timeouts
++;
281 statCounter
.netdb
.times_used
++;
289 httpRequestFree(void *data
)
291 clientHttpRequest
*http
= data
;
293 request_t
*request
= NULL
;
294 MemObject
*mem
= NULL
;
295 assert(http
!= NULL
);
297 request
= http
->request
;
298 debug(33, 3) ("httpRequestFree: %s\n", http
->uri
);
299 /* FIXME: This needs to use the stream */
300 if (!clientCheckTransferDone(http
)) {
301 if (request
&& request
->body_connection
)
302 clientAbortBody(request
); /* abort body transter */
303 /* the ICP check here was erroneous - storeReleaseRequest was always called if entry was valid
306 assert(http
->logType
< LOG_TYPE_MAX
);
308 mem
= http
->entry
->mem_obj
;
309 if (http
->out
.size
|| http
->logType
) {
310 http
->al
.icp
.opcode
= ICP_INVALID
;
311 http
->al
.url
= http
->log_uri
;
312 debug(33, 9) ("httpRequestFree: al.url='%s'\n", http
->al
.url
);
314 http
->al
.http
.code
= mem
->reply
->sline
.status
;
315 http
->al
.http
.content_type
= strBuf(mem
->reply
->content_type
);
317 http
->al
.cache
.caddr
= conn
? conn
->log_addr
: no_addr
;
318 http
->al
.cache
.size
= http
->out
.size
;
319 http
->al
.cache
.code
= http
->logType
;
320 http
->al
.cache
.msec
= tvSubMsec(http
->start
, current_time
);
325 packerToMemInit(&p
, &mb
);
326 httpHeaderPackInto(&request
->header
, &p
);
327 http
->al
.http
.method
= request
->method
;
328 http
->al
.http
.version
= request
->http_ver
;
329 http
->al
.headers
.request
= xstrdup(mb
.buf
);
330 http
->al
.hier
= request
->hier
;
331 if (request
->auth_user_request
) {
332 http
->al
.cache
.authuser
=
333 xstrdup(authenticateUserRequestUsername(request
->
335 authenticateAuthUserRequestUnlock(request
->auth_user_request
);
336 request
->auth_user_request
= NULL
;
338 if (conn
&& conn
->rfc931
[0])
339 http
->al
.cache
.rfc931
= conn
->rfc931
;
343 accessLogLog(&http
->al
);
344 clientUpdateCounters(http
);
346 clientdbUpdate(conn
->peer
.sin_addr
, http
->logType
, PROTO_HTTP
,
350 checkFailureRatio(request
->errType
, http
->al
.hier
.code
);
351 safe_free(http
->uri
);
352 safe_free(http
->log_uri
);
353 safe_free(http
->al
.headers
.request
);
354 safe_free(http
->al
.headers
.reply
);
355 safe_free(http
->al
.cache
.authuser
);
356 safe_free(http
->redirect
.location
);
357 requestUnlink(http
->request
);
358 if (http
->client_stream
.tail
)
359 clientStreamAbort(http
->client_stream
.tail
->data
, http
);
360 /* moving to the next connection is handled by the context free */
361 dlinkDelete(&http
->active
, &ClientActiveRequests
);
365 /* This is a handler normally called by comm_close() */
367 connStateFree(int fd
, void *data
)
369 ConnStateData
*connState
= data
;
370 clientSocketContext
*context
;
371 debug(33, 3) ("connStateFree: FD %d\n", fd
);
372 assert(connState
!= NULL
);
373 clientdbEstablished(connState
->peer
.sin_addr
, -1); /* decrement */
374 while ((context
= connState
->currentobject
) != NULL
) {
375 assert(context
->http
->conn
== connState
);
376 assert(connState
->currentobject
!=
377 ((clientSocketContext
*) connState
->currentobject
)->next
);
380 if (connState
->auth_user_request
)
381 authenticateAuthUserRequestUnlock(connState
->auth_user_request
);
382 connState
->auth_user_request
= NULL
;
383 authenticateOnCloseConnection(connState
);
384 memFreeBuf(connState
->in
.size
, connState
->in
.buf
);
385 pconnHistCount(0, connState
->nrequests
);
386 cbdataFree(connState
);
388 /* prevent those nasty RST packets */
390 char buf
[SQUID_TCP_SO_RCVBUF
];
391 while (FD_READ_METHOD(fd
, buf
, SQUID_TCP_SO_RCVBUF
) > 0);
397 * clientSetKeepaliveFlag() sets request->flags.proxy_keepalive.
398 * This is the client-side persistent connection flag. We need
399 * to set this relatively early in the request processing
400 * to handle hacks for broken servers and clients.
403 clientSetKeepaliveFlag(clientHttpRequest
* http
)
405 request_t
*request
= http
->request
;
406 const HttpHeader
*req_hdr
= &request
->header
;
408 debug(33, 3) ("clientSetKeepaliveFlag: http_ver = %d.%d\n",
409 request
->http_ver
.major
, request
->http_ver
.minor
);
410 debug(33, 3) ("clientSetKeepaliveFlag: method = %s\n",
411 RequestMethodStr
[request
->method
]);
412 if (!Config
.onoff
.client_pconns
)
413 request
->flags
.proxy_keepalive
= 0;
415 http_version_t http_ver
;
416 httpBuildVersion(&http_ver
, 1, 0);
417 /* we are HTTP/1.0, no matter what the client requests... */
418 if (httpMsgIsPersistent(http_ver
, req_hdr
))
419 request
->flags
.proxy_keepalive
= 1;
424 clientCheckContentLength(request_t
* r
)
429 /* PUT/POST requires a request entity */
430 return (r
->content_length
>= 0);
433 /* We do not want to see a request entity on GET/HEAD requests */
434 return (r
->content_length
<= 0);
436 /* For other types of requests we don't care */
443 isTcpHit(log_type code
)
445 /* this should be a bitmap for better optimization */
446 if (code
== LOG_TCP_HIT
)
448 if (code
== LOG_TCP_IMS_HIT
)
450 if (code
== LOG_TCP_REFRESH_FAIL_HIT
)
452 if (code
== LOG_TCP_REFRESH_HIT
)
454 if (code
== LOG_TCP_NEGATIVE_HIT
)
456 if (code
== LOG_TCP_MEM_HIT
)
458 if (code
== LOG_TCP_OFFLINE_HIT
)
464 clientRequestBodyTooLarge(int clen
)
466 if (0 == Config
.maxRequestBodySize
)
467 return 0; /* disabled */
469 return 0; /* unknown, bug? */
470 if (clen
> Config
.maxRequestBodySize
)
471 return 1; /* too large */
476 * Write a chunk of data to a client socket. If the reply is present, send the reply headers down the wire too,
477 * and clean them up when finished.
479 * The request is one backed by a connection, not an internal request.
480 * data context is not NULL
481 * There are no more entries in the stream chain.
484 clientSocketRecipient(clientStreamNode
* node
, clientHttpRequest
* http
,
485 HttpReply
* rep
, const char *body_data
, ssize_t body_size
)
488 clientSocketContext
*context
;
489 /* Test preconditions */
490 assert(node
!= NULL
);
491 /* TODO: handle this rather than asserting - it should only ever happen if we cause an abort and
492 * the callback chain loops back to here, so we can simply return.
493 * However, that itself shouldn't happen, so it stays as an assert for now.
495 assert(cbdataReferenceValid(node
));
496 assert(node
->data
!= NULL
);
497 assert(node
->node
.next
== NULL
);
498 context
= node
->data
;
499 assert(http
->conn
&& http
->conn
->fd
!= -1);
501 if (http
->conn
->currentobject
!= context
) {
502 /* there is another object in progress, defer this one */
503 debug(33, 2) ("clientSocketRecipient: Deferring %s\n", http
->uri
);
504 context
->flags
.deferred
= 1;
505 context
->deferredparams
.node
= node
;
506 context
->deferredparams
.rep
= rep
;
507 context
->deferredparams
.body_data
= body_data
;
508 context
->deferredparams
.body_size
= body_size
;
511 /* EOF / Read error / aborted entry */
512 if (rep
== NULL
&& body_data
== NULL
&& body_size
== 0) {
513 clientWriteComplete(fd
, NULL
, 0, COMM_OK
, context
);
517 if (http
->out
.offset
!= 0) {
519 /* Avoid copying to MemBuf if we know "rep" is NULL, and we only have a body */
520 http
->out
.offset
+= body_size
;
521 comm_write(fd
, body_data
, body_size
, clientWriteBodyComplete
, context
,
523 /* NULL because its a static buffer */
527 /* write headers and/or body if any */
528 assert(rep
|| (body_data
&& body_size
));
529 /* init mb; put status line and headers if any */
531 mb
= httpReplyPack(rep
);
532 /* http->out.offset += rep->hdr_sz; */
534 headersLog(0, 0, http
->request
->method
, rep
);
536 httpReplyDestroy(rep
);
541 if (body_data
&& body_size
) {
542 http
->out
.offset
+= body_size
;
543 memBufAppend(&mb
, body_data
, body_size
);
546 comm_write_mbuf(fd
, mb
, clientWriteComplete
, context
);
547 /* if we don't do it, who will? */
551 /* Called when a downstream node is no longer interested in
552 * our data. As we are a terminal node, this means on aborts
556 clientSocketDetach(clientStreamNode
* node
, clientHttpRequest
* http
)
558 clientSocketContext
*context
;
559 /* Test preconditions */
560 assert(node
!= NULL
);
561 /* TODO: handle this rather than asserting - it should only ever happen if we cause an abort and
562 * the callback chain loops back to here, so we can simply return.
563 * However, that itself shouldn't happen, so it stays as an assert for now.
565 assert(cbdataReferenceValid(node
));
566 /* Set null by ContextFree */
567 assert(node
->data
== NULL
);
568 assert(node
->node
.next
== NULL
);
569 context
= node
->data
;
570 /* We are only called when the client socket shutsdown.
571 * Tell the prev pipeline member we're finished
573 clientStreamDetach(node
, http
);
577 * clientWriteBodyComplete is called for MEM_CLIENT_SOCK_BUF's
578 * written directly to the client socket, versus copying to a MemBuf
579 * and going through comm_write_mbuf. Most non-range responses after
580 * the headers probably go through here.
583 clientWriteBodyComplete(int fd
, char *buf
, size_t size
, int errflag
, void *data
)
586 * NOTE: clientWriteComplete doesn't currently use its "buf"
587 * (second) argument, so we pass in NULL.
589 clientWriteComplete(fd
, NULL
, size
, errflag
, data
);
593 clientKeepaliveNextRequest(clientSocketContext
* context
)
595 clientHttpRequest
*http
= context
->http
;
596 ConnStateData
*conn
= http
->conn
;
598 debug(33, 3) ("clientKeepaliveNextRequest: FD %d\n", conn
->fd
);
599 conn
->defer
.until
= 0; /* Kick it to read a new request */
601 if ((context
= conn
->currentobject
) == NULL
) {
602 debug(33, 5) ("clientKeepaliveNextRequest: FD %d reading next req\n",
604 fd_note(conn
->fd
, "Waiting for next request");
606 * Set the timeout BEFORE calling clientReadRequest().
608 commSetTimeout(conn
->fd
, Config
.Timeout
.persistent_request
,
609 requestTimeout
, conn
);
611 * CYGWIN has a problem and is blocking on read() requests when there
612 * is no data present.
613 * This hack may hit performance a little, but it's better than
616 #ifdef _SQUID_CYGWIN_
617 commSetSelect(conn
->fd
, COMM_SELECT_READ
, clientReadRequest
, conn
, 0);
619 clientReadRequest(conn
->fd
, conn
); /* Read next request */
622 * Note, the FD may be closed at this point.
625 debug(33, 2) ("clientKeepaliveNextRequest: FD %d Sending next\n",
627 /* If the client stream is waiting on a socket write to occur, then */
628 if (context
->flags
.deferred
) {
629 /* NO data is allowed to have been sent */
630 assert(http
->out
.size
== 0);
631 clientSocketRecipient(context
->deferredparams
.node
, http
,
632 context
->deferredparams
.rep
,
633 context
->deferredparams
.body_data
,
634 context
->deferredparams
.body_size
);
636 /* otherwise, the request is still active in a callbacksomewhere,
643 /* A write has just completed to the client, or we have just realised there is
644 * no more data to send.
647 clientWriteComplete(int fd
, char *bufnotused
, size_t size
, int errflag
, void *data
)
649 clientSocketContext
*context
= data
;
650 clientHttpRequest
*http
= context
->http
;
651 StoreEntry
*entry
= http
->entry
;
652 /* cheating: we are always the tail */
653 clientStreamNode
*node
= http
->client_stream
.tail
->data
;
654 http
->out
.size
+= size
;
655 debug(33, 5) ("clientWriteComplete: FD %d, sz %ld, err %d, off %ld, len %d\n",
656 fd
, (long int) size
, errflag
, (long int) http
->out
.size
, entry
? objectLen(entry
) : 0);
657 if (size
> 0 && fd
> -1) {
658 kb_incr(&statCounter
.client_http
.kbytes_out
, size
);
659 if (isTcpHit(http
->logType
))
660 kb_incr(&statCounter
.client_http
.hit_kbytes_out
, size
);
664 * just close the socket, httpRequestFree will abort if needed.
665 * errflag is only EVER set by the comms callbacks
671 if (clientHttpRequestStatus(fd
, http
)) {
674 /* Do we leak here ? */
677 switch (clientStreamStatus(node
, http
)) {
679 /* More data will be coming from the stream. */
680 clientStreamRead(http
->client_stream
.tail
->data
, http
, http
->out
.offset
,
681 HTTP_REQBUF_SZ
, context
->reqbuf
);
683 case STREAM_COMPLETE
:
684 debug(33, 5) ("clientWriteComplete: FD %d Keeping Alive\n", fd
);
685 clientKeepaliveNextRequest(context
);
687 case STREAM_UNPLANNED_COMPLETE
:
694 fatal("Hit unreachable code in clientWriteComplete\n");
698 extern CSR clientGetMoreData
;
699 extern CSS clientReplyStatus
;
700 extern CSD clientReplyDetach
;
702 static clientSocketContext
*
703 parseHttpRequestAbort(ConnStateData
* conn
, const char *uri
)
705 clientHttpRequest
*http
;
706 clientSocketContext
*context
;
707 http
= cbdataAlloc(clientHttpRequest
);
709 http
->start
= current_time
;
710 http
->req_sz
= conn
->in
.offset
;
711 http
->uri
= xstrdup(uri
);
712 http
->log_uri
= xstrndup(uri
, MAX_URL
);
713 context
= clientSocketContextNew(http
);
714 clientStreamInit(&http
->client_stream
, clientGetMoreData
, clientReplyDetach
,
715 clientReplyStatus
, clientReplyNewContext(http
), clientSocketRecipient
,
716 clientSocketDetach
, context
, context
->reqbuf
, HTTP_REQBUF_SZ
);
717 dlinkAdd(http
, &http
->active
, &ClientActiveRequests
);
721 /* Utility function to perform part of request parsing */
722 static clientSocketContext
*
723 clientParseHttpRequestLine(char *inbuf
, size_t req_sz
, ConnStateData
* conn
,
724 method_t
* method_p
, char **url_p
, http_version_t
* http_ver_p
)
730 /* Barf on NULL characters in the headers */
731 if (strlen(inbuf
) != req_sz
) {
732 debug(33, 1) ("parseHttpRequest: Requestheader contains NULL characters\n");
733 return parseHttpRequestAbort(conn
, "error:invalid-request");
735 /* Look for request method */
736 if ((mstr
= strtok(inbuf
, "\t ")) == NULL
) {
737 debug(33, 1) ("parseHttpRequest: Can't get request method\n");
738 return parseHttpRequestAbort(conn
, "error:invalid-request-method");
740 *method_p
= urlParseMethod(mstr
);
741 if (*method_p
== METHOD_NONE
) {
742 debug(33, 1) ("parseHttpRequest: Unsupported method '%s'\n", mstr
);
743 return parseHttpRequestAbort(conn
, "error:unsupported-request-method");
745 debug(33, 5) ("parseHttpRequest: Method is '%s'\n", mstr
);
747 /* look for URL+HTTP/x.x */
748 if ((url
= strtok(NULL
, "\n")) == NULL
) {
749 debug(33, 1) ("parseHttpRequest: Missing URL\n");
750 return parseHttpRequestAbort(conn
, "error:missing-url");
752 while (xisspace(*url
))
754 t
= url
+ strlen(url
);
758 if (xisspace(*t
) && !strncmp(t
+ 1, "HTTP/", 5)) {
763 while (t
> url
&& xisspace(*t
))
765 debug(33, 5) ("parseHttpRequest: URI is '%s'\n", url
);
768 debug(33, 3) ("parseHttpRequest: Missing HTTP identifier\n");
769 #if RELAXED_HTTP_PARSER
770 httpBuildVersion(http_ver_p
, 0, 9); /* wild guess */
772 return parseHttpRequestAbort(conn
, "error:missing-http-ident");
775 if (sscanf(token
+ 5, "%d.%d", &http_ver_p
->major
,
776 &http_ver_p
->minor
) != 2) {
777 debug(33, 3) ("parseHttpRequest: Invalid HTTP identifier.\n");
778 return parseHttpRequestAbort(conn
, "error: invalid HTTP-ident");
780 debug(33, 6) ("parseHttpRequest: Client HTTP version %d.%d.\n",
781 http_ver_p
->major
, http_ver_p
->minor
);
784 /* everything was ok */
792 * NULL on error or incomplete request
793 * a clientHttpRequest structure on success
795 static clientSocketContext
*
796 parseHttpRequest(ConnStateData
* conn
, method_t
* method_p
, int *status
,
797 char **prefix_p
, size_t * req_line_sz_p
)
801 char *req_hdr
= NULL
;
802 http_version_t http_ver
;
805 size_t header_sz
; /* size of headers, not including first line */
806 size_t prefix_sz
; /* size of whole request (req-line + headers) */
809 clientHttpRequest
*http
;
810 clientSocketContext
*context
;
812 struct natlookup natLookup
;
813 static int natfd
= -1;
814 static int siocgnatl_cmd
= SIOCGNATL
& 0xff;
818 struct pfioc_natlook nl
;
819 static int pffd
= -1;
822 size_t sock_sz
= sizeof(conn
->me
);
825 /* pre-set these values to make aborting simpler */
827 *method_p
= METHOD_NONE
;
830 if ((req_sz
= headersEnd(conn
->in
.buf
, conn
->in
.offset
)) == 0) {
831 debug(33, 5) ("Incomplete request, waiting for end of headers\n");
835 assert(req_sz
<= conn
->in
.offset
);
836 /* Use memcpy, not strdup! */
837 inbuf
= xmalloc(req_sz
+ 1);
838 xmemcpy(inbuf
, conn
->in
.buf
, req_sz
);
839 *(inbuf
+ req_sz
) = '\0';
841 /* Is there a legitimate first line to the headers ? */
843 clientParseHttpRequestLine(inbuf
, req_sz
, conn
, method_p
, &url
,
845 /* something wrong, abort */
850 * Process headers after request line
852 req_hdr
= strtok(NULL
, null_string
);
853 header_sz
= req_sz
- (req_hdr
- inbuf
);
854 if (0 == header_sz
) {
855 debug(33, 3) ("parseHttpRequest: header_sz == 0\n");
860 assert(header_sz
> 0);
861 debug(33, 3) ("parseHttpRequest: req_hdr = {%s}\n", req_hdr
);
862 end
= req_hdr
+ header_sz
;
863 debug(33, 3) ("parseHttpRequest: end = {%s}\n", end
);
865 prefix_sz
= end
- inbuf
;
866 *req_line_sz_p
= req_hdr
- inbuf
;
867 debug(33, 3) ("parseHttpRequest: prefix_sz = %d, req_line_sz = %d\n",
868 (int) prefix_sz
, (int) *req_line_sz_p
);
869 assert(prefix_sz
<= conn
->in
.offset
);
871 /* Ok, all headers are received */
872 http
= cbdataAlloc(clientHttpRequest
);
873 http
->http_ver
= http_ver
;
875 http
->start
= current_time
;
876 http
->req_sz
= prefix_sz
;
877 context
= clientSocketContextNew(http
);
878 clientStreamInit(&http
->client_stream
, clientGetMoreData
, clientReplyDetach
,
879 clientReplyStatus
, clientReplyNewContext(http
), clientSocketRecipient
,
880 clientSocketDetach
, context
, context
->reqbuf
, HTTP_REQBUF_SZ
);
881 *prefix_p
= xmalloc(prefix_sz
+ 1);
882 xmemcpy(*prefix_p
, conn
->in
.buf
, prefix_sz
);
883 *(*prefix_p
+ prefix_sz
) = '\0';
884 dlinkAdd(http
, &http
->active
, &ClientActiveRequests
);
886 /* XXX this function is still way to long. here is a natural point for further simplification */
888 debug(33, 5) ("parseHttpRequest: Request Header is\n%s\n",
889 (*prefix_p
) + *req_line_sz_p
);
890 #if THIS_VIOLATES_HTTP_SPECS_ON_URL_TRANSFORMATION
891 if ((t
= strchr(url
, '#'))) /* remove HTML anchors */
895 /* handle internal objects */
896 if (internalCheck(url
)) {
897 /* prepend our name & port */
898 http
->uri
= xstrdup(internalLocalUri(NULL
, url
));
899 http
->flags
.internal
= 1;
900 http
->flags
.accel
= 1;
902 /* see if we running in Config2.Accel.on, if so got to convert it to URL */
903 else if (Config2
.Accel
.on
&& *url
== '/') {
904 /* prepend the accel prefix */
905 if (opt_accel_uses_host
&& (t
= mime_get_header(req_hdr
, "Host"))) {
908 const char *protocol_name
= "http";
910 vport
= (int) ntohs(http
->conn
->me
.sin_port
);
912 vport
= (int) Config
.Accel
.port
;
913 /* If a Host: header was specified, use it to build the URL
914 * instead of the one in the Config file. */
916 * XXX Use of the Host: header here opens a potential
917 * security hole. There are no checks that the Host: value
918 * corresponds to one of your servers. It might, for example,
919 * refer to www.playboy.com. The 'dst' and/or 'dst_domain' ACL
920 * types should be used to prevent httpd-accelerators
921 * handling requests for non-local servers */
923 if ((q
= strchr(t
, ':'))) {
928 url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
+ strlen(t
);
929 http
->uri
= xcalloc(url_sz
, 1);
931 #if SSL_FORWARDING_NOT_YET_DONE
932 if (Config
.Sockaddr
.https
->s
.sin_port
== http
->conn
->me
.sin_port
) {
933 protocol_name
= "https";
934 vport
= ntohs(http
->conn
->me
.sin_port
);
937 snprintf(http
->uri
, url_sz
, "%s://%s:%d%s",
938 protocol_name
, t
, vport
, url
);
939 } else if (vhost_mode
) {
941 /* Put the local socket IP address as the hostname */
942 url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
;
943 http
->uri
= xcalloc(url_sz
, 1);
945 vport
= (int) ntohs(http
->conn
->me
.sin_port
);
947 vport
= (int) Config
.Accel
.port
;
949 natLookup
.nl_inport
= http
->conn
->me
.sin_port
;
950 natLookup
.nl_outport
= http
->conn
->peer
.sin_port
;
951 natLookup
.nl_inip
= http
->conn
->me
.sin_addr
;
952 natLookup
.nl_outip
= http
->conn
->peer
.sin_addr
;
953 natLookup
.nl_flags
= IPN_TCP
;
957 natfd
= open(IPL_NAT
, O_RDONLY
, 0);
963 debug(50, 1) ("parseHttpRequest: NAT open failed: %s\n",
967 return parseHttpRequestAbort(conn
, "error:nat-open-failed");
970 * IP-Filter changed the type for SIOCGNATL between
971 * 3.3 and 3.4. It also changed the cmd value for
972 * SIOCGNATL, so at least we can detect it. We could
973 * put something in configure and use ifdefs here, but
974 * this seems simpler.
976 if (63 == siocgnatl_cmd
) {
977 struct natlookup
*nlp
= &natLookup
;
978 x
= ioctl(natfd
, SIOCGNATL
, &nlp
);
980 x
= ioctl(natfd
, SIOCGNATL
, &natLookup
);
983 if (errno
!= ESRCH
) {
984 debug(50, 1) ("parseHttpRequest: NAT lookup failed: ioctl(SIOCGNATL)\n");
989 return parseHttpRequestAbort(conn
,
990 "error:nat-lookup-failed");
992 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
993 inet_ntoa(http
->conn
->me
.sin_addr
), vport
, url
);
996 vport
= ntohs(natLookup
.nl_realport
);
997 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
998 inet_ntoa(natLookup
.nl_realip
), vport
, url
);
1000 #elif PF_TRANSPARENT
1002 pffd
= open("/dev/pf", O_RDWR
);
1004 debug(50, 1) ("parseHttpRequest: PF open failed: %s\n",
1006 cbdataFree(context
);
1008 return parseHttpRequestAbort(conn
, "error:pf-open-failed");
1010 memset(&nl
, 0, sizeof(struct pfioc_natlook
));
1011 nl
.saddr
.v4
.s_addr
= http
->conn
->peer
.sin_addr
.s_addr
;
1012 nl
.sport
= http
->conn
->peer
.sin_port
;
1013 nl
.daddr
.v4
.s_addr
= http
->conn
->me
.sin_addr
.s_addr
;
1014 nl
.dport
= http
->conn
->me
.sin_port
;
1016 nl
.proto
= IPPROTO_TCP
;
1017 nl
.direction
= PF_OUT
;
1018 if (ioctl(pffd
, DIOCNATLOOK
, &nl
)) {
1019 if (errno
!= ENOENT
) {
1020 debug(50, 1) ("parseHttpRequest: PF lookup failed: ioctl(DIOCNATLOOK)\n");
1023 cbdataFree(context
);
1025 return parseHttpRequestAbort(conn
,
1026 "error:pf-lookup-failed");
1028 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
1029 inet_ntoa(http
->conn
->me
.sin_addr
), vport
, url
);
1031 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
1032 inet_ntoa(nl
.rdaddr
.v4
), ntohs(nl
.rdport
), url
);
1035 /* If the call fails the address structure will be unchanged */
1036 getsockopt(conn
->fd
, SOL_IP
, SO_ORIGINAL_DST
, &conn
->me
, &sock_sz
);
1037 debug(33, 5) ("parseHttpRequest: addr = %s",
1038 inet_ntoa(conn
->me
.sin_addr
));
1040 vport
= (int) ntohs(http
->conn
->me
.sin_port
);
1042 snprintf(http
->uri
, url_sz
, "http://%s:%d%s",
1043 inet_ntoa(http
->conn
->me
.sin_addr
), vport
, url
);
1045 debug(33, 5) ("VHOST REWRITE: '%s'\n", http
->uri
);
1047 url_sz
= strlen(Config2
.Accel
.prefix
) + strlen(url
) +
1048 Config
.appendDomainLen
+ 1;
1049 http
->uri
= xcalloc(url_sz
, 1);
1050 snprintf(http
->uri
, url_sz
, "%s%s", Config2
.Accel
.prefix
, url
);
1052 http
->flags
.accel
= 1;
1054 /* URL may be rewritten later, so make extra room */
1055 url_sz
= strlen(url
) + Config
.appendDomainLen
+ 5;
1056 http
->uri
= xcalloc(url_sz
, 1);
1057 strcpy(http
->uri
, url
);
1058 http
->flags
.accel
= 0;
1060 if (!stringHasCntl(http
->uri
))
1061 http
->log_uri
= xstrndup(http
->uri
, MAX_URL
);
1063 http
->log_uri
= xstrndup(rfc1738_escape_unescaped(http
->uri
), MAX_URL
);
1064 debug(33, 5) ("parseHttpRequest: Complete request received\n");
1071 clientReadDefer(int fdnotused
, void *data
)
1073 ConnStateData
*conn
= data
;
1074 if (conn
->body
.size_left
)
1075 return conn
->in
.offset
>= conn
->in
.size
- 1;
1077 return conn
->defer
.until
> squid_curtime
;
1081 clientReadRequest(int fd
, void *data
)
1083 ConnStateData
*conn
= data
;
1084 int parser_return_code
= 0;
1085 request_t
*request
= NULL
;
1088 char *prefix
= NULL
;
1089 fde
*F
= &fd_table
[fd
];
1090 int len
= conn
->in
.size
- conn
->in
.offset
- 1;
1091 clientSocketContext
*context
;
1092 debug(33, 4) ("clientReadRequest: FD %d: reading request...\n", fd
);
1093 commSetSelect(fd
, COMM_SELECT_READ
, clientReadRequest
, conn
, 0);
1095 /* Grow the request memory area to accomodate for a large request */
1097 memReallocBuf(conn
->in
.buf
, conn
->in
.size
* 2, &conn
->in
.size
);
1098 debug(33, 2) ("growing request buffer: offset=%ld size=%ld\n",
1099 (long) conn
->in
.offset
, (long) conn
->in
.size
);
1100 len
= conn
->in
.size
- conn
->in
.offset
- 1;
1102 statCounter
.syscalls
.sock
.reads
++;
1103 size
= FD_READ_METHOD(fd
, conn
->in
.buf
+ conn
->in
.offset
, len
);
1105 fd_bytes(fd
, size
, FD_READ
);
1106 kb_incr(&statCounter
.client_http
.kbytes_in
, size
);
1109 * Don't reset the timeout value here. The timeout value will be
1110 * set to Config.Timeout.request by httpAccept() and
1111 * clientWriteComplete(), and should apply to the request as a
1112 * whole, not individual read() calls. Plus, it breaks our
1113 * lame half-close detection
1116 conn
->in
.offset
+= size
;
1117 conn
->in
.buf
[conn
->in
.offset
] = '\0'; /* Terminate the string */
1118 } else if (size
== 0) {
1119 if (conn
->currentobject
== NULL
&& conn
->in
.offset
== 0) {
1120 /* no current or pending requests */
1121 debug(33, 4) ("clientReadRequest: FD %d closed\n", fd
);
1124 } else if (!Config
.onoff
.half_closed_clients
) {
1125 /* admin doesn't want to support half-closed client sockets */
1126 debug(33, 3) ("clientReadRequest: FD %d aborted (half_closed_clients disabled)\n",
1131 /* It might be half-closed, we can't tell */
1132 debug(33, 5) ("clientReadRequest: FD %d closed?\n", fd
);
1133 F
->flags
.socket_eof
= 1;
1134 conn
->defer
.until
= squid_curtime
+ 1;
1136 fd_note(fd
, "half-closed");
1137 /* There is one more close check at the end, to detect aborted
1138 * (partial) requests. At this point we can't tell if the request
1141 /* Continue to process previously read data */
1142 } else if (size
< 0) {
1143 if (!ignoreErrno(errno
)) {
1144 debug(50, 2) ("clientReadRequest: FD %d: %s\n", fd
, xstrerror());
1147 } else if (conn
->in
.offset
== 0) {
1148 debug(50, 2) ("clientReadRequest: FD %d: no data to process (%s)\n",
1151 /* Continue to process previously read data */
1153 /* Process request body if any */
1154 if (conn
->in
.offset
> 0 && conn
->body
.callback
!= NULL
)
1155 clientProcessBody(conn
);
1156 /* Process next request */
1157 while (conn
->in
.offset
> 0 && conn
->body
.size_left
== 0) {
1158 clientSocketContext
**S
;
1161 /* Skip leading (and trailing) whitespace */
1162 while (conn
->in
.offset
> 0 && xisspace(conn
->in
.buf
[0])) {
1163 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ 1, conn
->in
.offset
- 1);
1166 conn
->in
.buf
[conn
->in
.offset
] = '\0'; /* Terminate the string */
1167 if (conn
->in
.offset
== 0)
1169 /* Limit the number of concurrent requests to 2 */
1170 for (S
= (clientSocketContext
**) & conn
->currentobject
, nrequests
= 0;
1171 *S
; S
= &(*S
)->next
, nrequests
++);
1172 if (nrequests
>= (Config
.onoff
.pipeline_prefetch
? 2 : 1)) {
1173 debug(33, 3) ("clientReadRequest: FD %d max concurrent requests reached\n",
1175 debug(33, 5) ("clientReadRequest: FD %d defering new request until one is done\n",
1177 conn
->defer
.until
= squid_curtime
+ 100; /* Reset when a request is complete */
1181 conn
->in
.buf
[conn
->in
.offset
] = '\0'; /* Terminate the string */
1183 fd_note(conn
->fd
, "Reading next request");
1184 /* Process request */
1185 context
= parseHttpRequest(conn
,
1186 &method
, &parser_return_code
, &prefix
, &req_line_sz
);
1190 clientHttpRequest
*http
= context
->http
;
1191 /* We have an initial client stream in place should it be needed */
1192 /* setup our private context */
1193 assert(http
->req_sz
> 0);
1194 conn
->in
.offset
-= http
->req_sz
;
1195 assert(conn
->in
.offset
>= 0);
1196 debug(33, 5) ("conn->in.offset = %d\n", (int) conn
->in
.offset
);
1198 * If we read past the end of this request, move the remaining
1199 * data to the beginning
1201 if (conn
->in
.offset
> 0)
1202 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ http
->req_sz
,
1204 /* add to the client request queue */
1205 for (S
= (clientSocketContext
**) & conn
->currentobject
; *S
;
1209 commSetTimeout(fd
, Config
.Timeout
.lifetime
, clientLifetimeTimeout
,
1211 if (parser_return_code
< 0) {
1212 clientStreamNode
*node
= http
->client_stream
.tail
->prev
->data
;
1213 debug(33, 1) ("clientReadRequest: FD %d Invalid Request\n", fd
);
1214 clientSetReplyToError(node
->data
,
1215 ERR_INVALID_REQ
, HTTP_BAD_REQUEST
, method
, NULL
,
1216 &conn
->peer
.sin_addr
, NULL
, conn
->in
.buf
, NULL
);
1217 clientStreamRead(http
->client_stream
.tail
->data
, http
, 0,
1218 HTTP_REQBUF_SZ
, context
->reqbuf
);
1222 if ((request
= urlParse(method
, http
->uri
)) == NULL
) {
1223 clientStreamNode
*node
= http
->client_stream
.tail
->prev
->data
;
1224 debug(33, 5) ("Invalid URL: %s\n", http
->uri
);
1225 clientSetReplyToError(node
->data
,
1226 ERR_INVALID_URL
, HTTP_BAD_REQUEST
, method
, http
->uri
,
1227 &conn
->peer
.sin_addr
, NULL
, NULL
, NULL
);
1228 clientStreamRead(http
->client_stream
.tail
->data
, http
, 0,
1229 HTTP_REQBUF_SZ
, context
->reqbuf
);
1233 /* compile headers */
1234 /* we should skip request line! */
1235 if (!httpRequestParseHeader(request
, prefix
+ req_line_sz
))
1236 debug(33, 1) ("Failed to parse request headers: %s\n%s\n",
1238 /* continue anyway? */
1240 request
->flags
.accelerated
= http
->flags
.accel
;
1241 if (!http
->flags
.internal
) {
1242 if (internalCheck(strBuf(request
->urlpath
))) {
1243 if (internalHostnameIs(request
->host
) &&
1244 request
->port
== getMyPort()) {
1245 http
->flags
.internal
= 1;
1246 } else if (internalStaticCheck(strBuf(request
->urlpath
))) {
1247 xstrncpy(request
->host
, internalHostname(),
1249 request
->port
= getMyPort();
1250 http
->flags
.internal
= 1;
1255 * cache the Content-length value in request_t.
1257 request
->content_length
= httpHeaderGetInt(&request
->header
,
1258 HDR_CONTENT_LENGTH
);
1259 request
->flags
.internal
= http
->flags
.internal
;
1261 safe_free(http
->log_uri
);
1262 http
->log_uri
= xstrdup(urlCanonicalClean(request
));
1263 request
->client_addr
= conn
->peer
.sin_addr
;
1264 request
->my_addr
= conn
->me
.sin_addr
;
1265 request
->my_port
= ntohs(conn
->me
.sin_port
);
1266 request
->http_ver
= http
->http_ver
;
1267 if (!urlCheckRequest(request
) ||
1268 httpHeaderHas(&request
->header
, HDR_TRANSFER_ENCODING
)) {
1269 clientStreamNode
*node
= http
->client_stream
.tail
->prev
->data
;
1270 clientSetReplyToError(node
->data
, ERR_UNSUP_REQ
,
1271 HTTP_NOT_IMPLEMENTED
, request
->method
, NULL
,
1272 &conn
->peer
.sin_addr
, request
, NULL
, NULL
);
1273 clientStreamRead(http
->client_stream
.tail
->data
, http
, 0,
1274 HTTP_REQBUF_SZ
, context
->reqbuf
);
1277 if (!clientCheckContentLength(request
)) {
1278 clientStreamNode
*node
= http
->client_stream
.tail
->prev
->data
;
1279 clientSetReplyToError(node
->data
, ERR_INVALID_REQ
,
1280 HTTP_LENGTH_REQUIRED
, request
->method
, NULL
,
1281 &conn
->peer
.sin_addr
, request
, NULL
, NULL
);
1282 clientStreamRead(http
->client_stream
.tail
->data
, http
, 0,
1283 HTTP_REQBUF_SZ
, context
->reqbuf
);
1286 http
->request
= requestLink(request
);
1287 clientSetKeepaliveFlag(http
);
1288 /* Do we expect a request-body? */
1289 if (request
->content_length
> 0) {
1290 conn
->body
.size_left
= request
->content_length
;
1291 request
->body_connection
= conn
;
1292 /* Is it too large? */
1293 if (clientRequestBodyTooLarge(request
->content_length
)) {
1294 clientStreamNode
*node
=
1295 http
->client_stream
.tail
->prev
->data
;
1296 clientSetReplyToError(node
->data
, ERR_TOO_BIG
,
1297 HTTP_REQUEST_ENTITY_TOO_LARGE
, METHOD_NONE
, NULL
,
1298 &conn
->peer
.sin_addr
, http
->request
, NULL
, NULL
);
1299 clientStreamRead(http
->client_stream
.tail
->data
, http
, 0,
1300 HTTP_REQBUF_SZ
, context
->reqbuf
);
1304 clientAccessCheck(http
);
1305 continue; /* while offset > 0 && body.size_left == 0 */
1306 } else if (parser_return_code
== 0) {
1308 * Partial request received; reschedule until parseHttpRequest()
1309 * is happy with the input
1311 if (conn
->in
.offset
>= Config
.maxRequestHeaderSize
) {
1312 /* The request is too large to handle */
1313 clientStreamNode
*node
;
1315 parseHttpRequestAbort(conn
, "error:request-too-large");
1316 node
= context
->http
->client_stream
.tail
->prev
->data
;
1317 debug(33, 1) ("Request header is too large (%d bytes)\n",
1318 (int) conn
->in
.offset
);
1319 debug(33, 1) ("Config 'request_header_max_size'= %ld bytes.\n",
1320 (long int) Config
.maxRequestHeaderSize
);
1321 clientSetReplyToError(node
->data
, ERR_TOO_BIG
,
1322 HTTP_REQUEST_ENTITY_TOO_LARGE
, METHOD_NONE
, NULL
,
1323 &conn
->peer
.sin_addr
, NULL
, NULL
, NULL
);
1324 /* add to the client request queue */
1325 for (S
= (clientSocketContext
**) & conn
->currentobject
; *S
;
1328 clientStreamRead(context
->http
->client_stream
.tail
->data
,
1329 context
->http
, 0, HTTP_REQBUF_SZ
, context
->reqbuf
);
1334 } /* while offset > 0 && conn->body.size_left == 0 */
1335 /* Check if a half-closed connection was aborted in the middle */
1336 if (F
->flags
.socket_eof
) {
1337 if (conn
->in
.offset
!= conn
->body
.size_left
) { /* != 0 when no request body */
1338 /* Partial request received. Abort client connection! */
1339 debug(33, 3) ("clientReadRequest: FD %d aborted, partial request\n",
1347 /* file_read like function, for reading body content */
1349 clientReadBody(request_t
* request
, char *buf
, size_t size
, CBCB
* callback
,
1352 ConnStateData
*conn
= request
->body_connection
;
1354 debug(33, 5) ("clientReadBody: no body to read, request=%p\n", request
);
1355 callback(buf
, 0, cbdata
); /* Signal end of body */
1358 debug(33, 2) ("clientReadBody: start fd=%d body_size=%lu in.offset=%ld cb=%p req=%p\n",
1359 conn
->fd
, (unsigned long int) conn
->body
.size_left
,
1360 (long int) conn
->in
.offset
, callback
, request
);
1361 conn
->body
.callback
= callback
;
1362 conn
->body
.cbdata
= cbdata
;
1363 conn
->body
.buf
= buf
;
1364 conn
->body
.bufsize
= size
;
1365 conn
->body
.request
= requestLink(request
);
1366 clientProcessBody(conn
);
1369 /* Called by clientReadRequest to process body content */
1371 clientProcessBody(ConnStateData
* conn
)
1374 char *buf
= conn
->body
.buf
;
1375 void *cbdata
= conn
->body
.cbdata
;
1376 CBCB
*callback
= conn
->body
.callback
;
1377 request_t
*request
= conn
->body
.request
;
1378 /* Note: request is null while eating "aborted" transfers */
1379 debug(33, 2) ("clientProcessBody: start fd=%d body_size=%lu in.offset=%ld cb=%p req=%p\n",
1380 conn
->fd
, (unsigned long int) conn
->body
.size_left
,
1381 (long int) conn
->in
.offset
, callback
, request
);
1382 if (conn
->in
.offset
) {
1383 /* Some sanity checks... */
1384 assert(conn
->body
.size_left
> 0);
1385 assert(conn
->in
.offset
> 0);
1386 assert(callback
!= NULL
);
1387 assert(buf
!= NULL
);
1388 /* How much do we have to process? */
1389 size
= conn
->in
.offset
;
1390 if (size
> conn
->body
.size_left
) /* only process the body part */
1391 size
= conn
->body
.size_left
;
1392 if (size
> conn
->body
.bufsize
) /* don't copy more than requested */
1393 size
= conn
->body
.bufsize
;
1394 xmemcpy(buf
, conn
->in
.buf
, size
);
1395 conn
->body
.size_left
-= size
;
1396 /* Move any remaining data */
1397 conn
->in
.offset
-= size
;
1398 if (conn
->in
.offset
> 0)
1399 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ size
, conn
->in
.offset
);
1400 /* Remove request link if this is the last part of the body, as
1401 * clientReadRequest automatically continues to process next request */
1402 if (conn
->body
.size_left
<= 0 && request
!= NULL
)
1403 request
->body_connection
= NULL
;
1404 /* Remove clientReadBody arguments (the call is completed) */
1405 conn
->body
.request
= NULL
;
1406 conn
->body
.callback
= NULL
;
1407 conn
->body
.buf
= NULL
;
1408 conn
->body
.bufsize
= 0;
1409 /* Remember that we have touched the body, not restartable */
1410 if (request
!= NULL
)
1411 request
->flags
.body_sent
= 1;
1412 /* Invoke callback function */
1413 callback(buf
, size
, cbdata
);
1414 if (request
!= NULL
)
1415 requestUnlink(request
); /* Linked in clientReadBody */
1416 debug(33, 2) ("clientProcessBody: end fd=%d size=%d body_size=%lu in.offset=%ld cb=%p req=%p\n",
1417 conn
->fd
, size
, (unsigned long int) conn
->body
.size_left
,
1418 (long int) conn
->in
.offset
, callback
, request
);
1422 /* A dummy handler that throws away a request-body */
1423 static char bodyAbortBuf
[SQUID_TCP_SO_RCVBUF
];
1425 clientReadBodyAbortHandler(char *buf
, size_t size
, void *data
)
1427 ConnStateData
*conn
= (ConnStateData
*) data
;
1428 debug(33, 2) ("clientReadBodyAbortHandler: fd=%d body_size=%lu in.offset=%ld\n",
1429 conn
->fd
, (unsigned long int) conn
->body
.size_left
,
1430 (long int) conn
->in
.offset
);
1431 if (size
!= 0 && conn
->body
.size_left
!= 0) {
1432 debug(33, 3) ("clientReadBodyAbortHandler: fd=%d shedule next read\n",
1434 conn
->body
.callback
= clientReadBodyAbortHandler
;
1435 conn
->body
.buf
= bodyAbortBuf
;
1436 conn
->body
.bufsize
= sizeof(bodyAbortBuf
);
1437 conn
->body
.cbdata
= data
;
1441 /* Abort a body request */
1443 clientAbortBody(request_t
* request
)
1445 ConnStateData
*conn
= request
->body_connection
;
1449 request
->body_connection
= NULL
;
1450 if (!conn
|| conn
->body
.size_left
<= 0)
1451 return 0; /* No body to abort */
1452 if (conn
->body
.callback
!= NULL
) {
1453 buf
= conn
->body
.buf
;
1454 callback
= conn
->body
.callback
;
1455 cbdata
= conn
->body
.cbdata
;
1456 assert(request
== conn
->body
.request
);
1457 conn
->body
.buf
= NULL
;
1458 conn
->body
.callback
= NULL
;
1459 conn
->body
.cbdata
= NULL
;
1460 conn
->body
.request
= NULL
;
1461 callback(buf
, -1, cbdata
); /* Signal abort to clientReadBody caller */
1462 requestUnlink(request
);
1464 clientReadBodyAbortHandler(NULL
, -1, conn
); /* Install abort handler */
1465 /* clientProcessBody() */
1466 return 1; /* Aborted */
1469 /* general lifetime handler for HTTP requests */
1471 requestTimeout(int fd
, void *data
)
1473 #if THIS_CONFUSES_PERSISTENT_CONNECTION_AWARE_BROWSERS_AND_USERS
1474 ConnStateData
*conn
= data
;
1475 debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd
);
1476 if (fd_table
[fd
].rwstate
) {
1478 * Some data has been sent to the client, just close the FD
1481 } else if (conn
->nrequests
) {
1483 * assume its a persistent connection; just close it
1490 clientHttpRequest
**H
;
1491 clientStreamNode
*node
;
1492 clientHttpRequest
*http
=
1493 parseHttpRequestAbort(conn
,
1494 "error:Connection%20lifetime%20expired");
1495 node
= http
->client_stream
.tail
->prev
->data
;
1496 clientSetReplyToError(node
->data
, ERR_LIFETIME_EXP
,
1497 HTTP_REQUEST_TIMEOUT
, METHOD_NONE
, "N/A", &conn
->peer
.sin_addr
,
1499 /* No requests can be outstanded */
1500 assert(conn
->chr
== NULL
);
1501 /* add to the client request queue */
1502 for (H
= &conn
->chr
; *H
; H
= &(*H
)->next
);
1504 clientStreamRead(http
->client_stream
.tail
->data
, http
, 0,
1505 HTTP_REQBUF_SZ
, context
->reqbuf
);
1507 * if we don't close() here, we still need a timeout handler!
1509 commSetTimeout(fd
, 30, requestTimeout
, conn
);
1511 * Aha, but we don't want a read handler!
1513 commSetSelect(fd
, COMM_SELECT_READ
, NULL
, NULL
, 0);
1517 * Just close the connection to not confuse browsers
1518 * using persistent connections. Some browsers opens
1519 * an connection and then does not use it until much
1520 * later (presumeably because the request triggering
1521 * the open has already been completed on another
1524 debug(33, 3) ("requestTimeout: FD %d: lifetime is expired.\n", fd
);
1530 clientLifetimeTimeout(int fd
, void *data
)
1532 clientHttpRequest
*http
= data
;
1533 ConnStateData
*conn
= http
->conn
;
1535 1) ("WARNING: Closing client %s connection due to lifetime timeout\n",
1536 inet_ntoa(conn
->peer
.sin_addr
));
1537 debug(33, 1) ("\t%s\n", http
->uri
);
1542 httpAcceptDefer(int fdunused
, void *dataunused
)
1544 static time_t last_warn
= 0;
1545 if (fdNFree() >= RESERVED_FD
)
1547 if (last_warn
+ 15 < squid_curtime
) {
1548 debug(33, 0) ("WARNING! Your cache is running out of filedescriptors\n");
1549 last_warn
= squid_curtime
;
1554 /* Handle a new connection on HTTP socket. */
1556 httpAccept(int sock
, void *data
)
1558 int *N
= &incoming_sockets_accepted
;
1560 ConnStateData
*connState
= NULL
;
1561 struct sockaddr_in peer
;
1562 struct sockaddr_in me
;
1563 int max
= INCOMING_HTTP_MAX
;
1565 static aclCheck_t identChecklist
;
1567 commSetSelect(sock
, COMM_SELECT_READ
, httpAccept
, NULL
, 0);
1568 while (max
-- && !httpAcceptDefer(sock
, NULL
)) {
1569 memset(&peer
, '\0', sizeof(struct sockaddr_in
));
1570 memset(&me
, '\0', sizeof(struct sockaddr_in
));
1571 if ((fd
= comm_accept(sock
, &peer
, &me
)) < 0) {
1572 if (!ignoreErrno(errno
))
1573 debug(50, 1) ("httpAccept: FD %d: accept failure: %s\n",
1577 debug(33, 4) ("httpAccept: FD %d: accepted\n", fd
);
1578 connState
= cbdataAlloc(ConnStateData
);
1579 connState
->peer
= peer
;
1580 connState
->log_addr
= peer
.sin_addr
;
1581 connState
->log_addr
.s_addr
&= Config
.Addrs
.client_netmask
.s_addr
;
1584 connState
->in
.buf
= memAllocBuf(CLIENT_REQ_BUF_SZ
, &connState
->in
.size
);
1585 comm_add_close_handler(fd
, connStateFree
, connState
);
1586 if (Config
.onoff
.log_fqdn
)
1587 fqdncache_gethostbyaddr(peer
.sin_addr
, FQDN_LOOKUP_IF_MISS
);
1588 commSetTimeout(fd
, Config
.Timeout
.request
, requestTimeout
, connState
);
1590 identChecklist
.src_addr
= peer
.sin_addr
;
1591 identChecklist
.my_addr
= me
.sin_addr
;
1592 identChecklist
.my_port
= ntohs(me
.sin_port
);
1593 if (aclCheckFast(Config
.accessList
.identLookup
, &identChecklist
))
1594 identStart(&me
, &peer
, clientIdentDone
, connState
);
1596 commSetSelect(fd
, COMM_SELECT_READ
, clientReadRequest
, connState
, 0);
1597 commSetDefer(fd
, clientReadDefer
, connState
);
1598 clientdbEstablished(peer
.sin_addr
, 1);
1606 /* negotiate an SSL connection */
1608 clientNegotiateSSL(int fd
, void *data
)
1610 ConnStateData
*conn
= data
;
1614 if ((ret
= SSL_accept(fd_table
[fd
].ssl
)) <= 0) {
1615 if (BIO_sock_should_retry(ret
)) {
1616 commSetSelect(fd
, COMM_SELECT_READ
, clientNegotiateSSL
, conn
, 0);
1619 ret
= ERR_get_error();
1622 ("clientNegotiateSSL: Error negotiating SSL connection on FD %d: %s\n",
1623 fd
, ERR_error_string(ret
, NULL
));
1628 debug(83, 5) ("clientNegotiateSSL: FD %d negotiated cipher %s\n", fd
,
1629 SSL_get_cipher(fd_table
[fd
].ssl
));
1631 client_cert
= SSL_get_peer_certificate(fd_table
[fd
].ssl
);
1632 if (client_cert
!= NULL
) {
1633 debug(83, 5) ("clientNegotiateSSL: FD %d client certificate: subject: %s\n",
1634 fd
, X509_NAME_oneline(X509_get_subject_name(client_cert
), 0, 0));
1636 debug(83, 5) ("clientNegotiateSSL: FD %d client certificate: issuer: %s\n",
1637 fd
, X509_NAME_oneline(X509_get_issuer_name(client_cert
), 0, 0));
1639 X509_free(client_cert
);
1641 debug(83, 5) ("clientNegotiateSSL: FD %d has no certificate.\n", fd
);
1644 commSetSelect(fd
, COMM_SELECT_READ
, clientReadRequest
, conn
, 0);
1647 struct _https_port_data
{
1648 SSL_CTX
*sslContext
;
1650 typedef struct _https_port_data https_port_data
;
1651 CBDATA_TYPE(https_port_data
);
1653 /* handle a new HTTPS connection */
1655 httpsAccept(int sock
, void *data
)
1657 int *N
= &incoming_sockets_accepted
;
1658 https_port_data
*https_port
= data
;
1659 SSL_CTX
*sslContext
= https_port
->sslContext
;
1661 ConnStateData
*connState
= NULL
;
1662 struct sockaddr_in peer
;
1663 struct sockaddr_in me
;
1664 int max
= INCOMING_HTTP_MAX
;
1668 static aclCheck_t identChecklist
;
1670 commSetSelect(sock
, COMM_SELECT_READ
, httpsAccept
, https_port
, 0);
1671 while (max
-- && !httpAcceptDefer(sock
, NULL
)) {
1672 memset(&peer
, '\0', sizeof(struct sockaddr_in
));
1673 memset(&me
, '\0', sizeof(struct sockaddr_in
));
1674 if ((fd
= comm_accept(sock
, &peer
, &me
)) < 0) {
1675 if (!ignoreErrno(errno
))
1676 debug(50, 1) ("httpsAccept: FD %d: accept failure: %s\n",
1680 if ((ssl
= SSL_new(sslContext
)) == NULL
) {
1681 ssl_error
= ERR_get_error();
1682 debug(83, 1) ("httpsAccept: Error allocating handle: %s\n",
1683 ERR_error_string(ssl_error
, NULL
));
1686 SSL_set_fd(ssl
, fd
);
1687 fd_table
[fd
].ssl
= ssl
;
1688 fd_table
[fd
].read_method
= &ssl_read_method
;
1689 fd_table
[fd
].write_method
= &ssl_write_method
;
1690 debug(50, 5) ("httpsAccept: FD %d accepted, starting SSL negotiation.\n", fd
);
1692 connState
= cbdataAlloc(ConnStateData
);
1693 connState
->peer
= peer
;
1694 connState
->log_addr
= peer
.sin_addr
;
1695 connState
->log_addr
.s_addr
&= Config
.Addrs
.client_netmask
.s_addr
;
1698 connState
->in
.buf
= memAllocBuf(CLIENT_REQ_BUF_SZ
, &connState
->in
.size
);
1699 /* XXX account connState->in.buf */
1700 comm_add_close_handler(fd
, connStateFree
, connState
);
1701 if (Config
.onoff
.log_fqdn
)
1702 fqdncache_gethostbyaddr(peer
.sin_addr
, FQDN_LOOKUP_IF_MISS
);
1703 commSetTimeout(fd
, Config
.Timeout
.request
, requestTimeout
, connState
);
1705 identChecklist
.src_addr
= peer
.sin_addr
;
1706 identChecklist
.my_addr
= me
.sin_addr
;
1707 identChecklist
.my_port
= ntohs(me
.sin_port
);
1708 if (aclCheckFast(Config
.accessList
.identLookup
, &identChecklist
))
1709 identStart(&me
, &peer
, clientIdentDone
, connState
);
1711 commSetSelect(fd
, COMM_SELECT_READ
, clientNegotiateSSL
, connState
, 0);
1712 commSetDefer(fd
, clientReadDefer
, connState
);
1713 clientdbEstablished(peer
.sin_addr
, 1);
1718 #endif /* USE_SSL */
1721 * This function is designed to serve a fairly specific purpose.
1722 * Occasionally our vBNS-connected caches can talk to each other, but not
1723 * the rest of the world. Here we try to detect frequent failures which
1724 * make the cache unusable (e.g. DNS lookup and connect() failures). If
1725 * the failure:success ratio goes above 1.0 then we go into "hit only"
1726 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH. Neighbors
1727 * will only fetch HITs from us if they are using the ICP protocol. We
1728 * stay in this mode for 5 minutes.
1730 * Duane W., Sept 16, 1996
1734 checkFailureRatio(err_type etype
, hier_code hcode
)
1736 static double magic_factor
= 100.0;
1739 if (hcode
== HIER_NONE
)
1741 n_good
= magic_factor
/ (1.0 + request_failure_ratio
);
1742 n_bad
= magic_factor
- n_good
;
1745 case ERR_CONNECT_FAIL
:
1746 case ERR_READ_ERROR
:
1752 request_failure_ratio
= n_bad
/ n_good
;
1753 if (hit_only_mode_until
> squid_curtime
)
1755 if (request_failure_ratio
< 1.0)
1757 debug(33, 0) ("Failure Ratio at %4.2f\n", request_failure_ratio
);
1758 debug(33, 0) ("Going into hit-only-mode for %d minutes...\n",
1759 FAILURE_MODE_TIME
/ 60);
1760 hit_only_mode_until
= squid_curtime
+ FAILURE_MODE_TIME
;
1761 request_failure_ratio
= 0.8; /* reset to something less than 1.0 */
1765 clientHttpConnectionsOpen(void)
1767 sockaddr_in_list
*s
;
1769 for (s
= Config
.Sockaddr
.http
; s
; s
= s
->next
) {
1770 if (MAXHTTPPORTS
== NHttpSockets
) {
1771 debug(1, 1) ("WARNING: You have too many 'http_port' lines.\n");
1772 debug(1, 1) (" The limit is %d\n", MAXHTTPPORTS
);
1776 fd
= comm_open(SOCK_STREAM
,
1779 ntohs(s
->s
.sin_port
), COMM_NONBLOCKING
, "HTTP Socket");
1784 commSetSelect(fd
, COMM_SELECT_READ
, httpAccept
, NULL
, 0);
1786 * We need to set a defer handler here so that we don't
1787 * peg the CPU with select() when we hit the FD limit.
1789 commSetDefer(fd
, httpAcceptDefer
, NULL
);
1790 debug(1, 1) ("Accepting HTTP connections at %s, port %d, FD %d.\n",
1791 inet_ntoa(s
->s
.sin_addr
), (int) ntohs(s
->s
.sin_port
), fd
);
1792 HttpSockets
[NHttpSockets
++] = fd
;
1798 clientHttpsConnectionsOpen(void)
1801 https_port_data
*https_port
;
1803 for (s
= Config
.Sockaddr
.https
; s
; s
= s
->next
) {
1804 if (MAXHTTPPORTS
== NHttpSockets
) {
1805 debug(1, 1) ("WARNING: You have too many 'https_port' lines.\n");
1806 debug(1, 1) (" The limit is %d\n", MAXHTTPPORTS
);
1810 fd
= comm_open(SOCK_STREAM
,
1813 ntohs(s
->s
.sin_port
), COMM_NONBLOCKING
, "HTTPS Socket");
1817 CBDATA_INIT_TYPE(https_port_data
);
1818 https_port
= cbdataAlloc(https_port_data
);
1819 https_port
->sslContext
=
1820 sslCreateContext(s
->cert
, s
->key
, s
->version
, s
->cipher
,
1823 commSetSelect(fd
, COMM_SELECT_READ
, httpsAccept
, https_port
, 0);
1824 commSetDefer(fd
, httpAcceptDefer
, NULL
);
1825 debug(1, 1) ("Accepting HTTPS connections at %s, port %d, FD %d.\n",
1826 inet_ntoa(s
->s
.sin_addr
), (int) ntohs(s
->s
.sin_port
), fd
);
1827 HttpSockets
[NHttpSockets
++] = fd
;
1834 clientOpenListenSockets(void)
1836 clientHttpConnectionsOpen();
1838 clientHttpsConnectionsOpen();
1840 if (NHttpSockets
< 1)
1841 fatal("Cannot open HTTP Port");
1845 clientHttpConnectionsClose(void)
1848 for (i
= 0; i
< NHttpSockets
; i
++) {
1849 if (HttpSockets
[i
] >= 0) {
1850 debug(1, 1) ("FD %d Closing HTTP connection\n", HttpSockets
[i
]);
1851 comm_close(HttpSockets
[i
]);
1852 HttpSockets
[i
] = -1;
1859 varyEvaluateMatch(StoreEntry
* entry
, request_t
* request
)
1861 const char *vary
= request
->vary_headers
;
1862 int has_vary
= httpHeaderHas(&entry
->mem_obj
->reply
->header
, HDR_VARY
);
1863 #if X_ACCELERATOR_VARY
1865 httpHeaderHas(&entry
->mem_obj
->reply
->header
, HDR_X_ACCELERATOR_VARY
);
1867 if (!has_vary
|| !entry
->mem_obj
->vary_headers
) {
1869 /* Oops... something odd is going on here.. */
1872 ("varyEvaluateMatch: Oops. Not a Vary object on second attempt, '%s' '%s'\n",
1873 entry
->mem_obj
->url
, vary
);
1874 safe_free(request
->vary_headers
);
1878 /* This is not a varying object */
1881 /* virtual "vary" object found. Calculate the vary key and
1882 * continue the search
1884 vary
= httpMakeVaryMark(request
, entry
->mem_obj
->reply
);
1886 request
->vary_headers
= xstrdup(vary
);
1889 /* Ouch.. we cannot handle this kind of variance */
1890 /* XXX This cannot really happen, but just to be complete */
1895 vary
= httpMakeVaryMark(request
, entry
->mem_obj
->reply
);
1897 request
->vary_headers
= xstrdup(vary
);
1900 /* Ouch.. we cannot handle this kind of variance */
1901 /* XXX This cannot really happen, but just to be complete */
1903 } else if (strcmp(vary
, entry
->mem_obj
->vary_headers
) == 0) {
1906 /* Oops.. we have already been here and still haven't
1907 * found the requested variant. Bail out
1909 debug(33, 1) ("varyEvaluateMatch: Oops. Not a Vary match on second attempt, '%s' '%s'\n",
1910 entry
->mem_obj
->url
, vary
);